feat: 大屏后端（FastAPI+MQTT+sim_engine）
- REST :10085 / MQTT 控制 / SSE 回退 / AI+语音 - 数据源规划：接核心服务端 + 算力引擎真实数据
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,447 @@
|
||||
/* =========================================================
|
||||
Admin 管理后台前端逻辑（Jinja 页面配套）
|
||||
事件采用 document 委托绑定，保证按钮点击稳定响应；
|
||||
连接状态每 3s 轮询 /api/display/state 刷新
|
||||
========================================================= */
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
var init = window.__DPM_INIT || { fullscreen: true, autostart: false };
|
||||
var fullscreen = !!init.fullscreen;
|
||||
var autostart = !!init.autostart;
|
||||
|
||||
/* 目标屏幕（按 MQTT 客户端精准控制）：client_id + role */
|
||||
var screenId = '';
|
||||
var screenRole = '';
|
||||
|
||||
function applyScreenTarget(sel) {
|
||||
sel = sel || document.getElementById('screenTarget');
|
||||
if (!sel) return;
|
||||
var v = sel.value || '';
|
||||
screenId = v;
|
||||
var opt = null;
|
||||
if (v) { for (var i = 0; i < sel.options.length; i++) { if (sel.options[i].value === v) { opt = sel.options[i]; break; } } }
|
||||
screenRole = opt ? (opt.getAttribute('data-role') || '') : '';
|
||||
}
|
||||
|
||||
function loadScreens() {
|
||||
fetch('/api/display/state', { cache: 'no-store' })
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
var sel = document.getElementById('screenTarget');
|
||||
var cnt = document.getElementById('screenCount');
|
||||
if (!sel) return;
|
||||
var devices = (d && d.devices) || [];
|
||||
if (cnt) cnt.textContent = '在线设备：' + (devices.length || 0) + ' 台';
|
||||
var cur = sel.value;
|
||||
sel.innerHTML = '<option value="">全部屏幕</option>';
|
||||
devices.forEach(function (dev) {
|
||||
var role = dev.role || '';
|
||||
var name = role === 'main' ? '主屏' : role === 'secondary' ? '副屏' : (role || '屏幕');
|
||||
var parent = (dev.role === 'secondary' && dev.parent_id) ? '·属于:' + String(dev.parent_id).slice(0, 8) : '';
|
||||
var opt = document.createElement('option');
|
||||
opt.value = dev.device_id;
|
||||
opt.setAttribute('data-role', role);
|
||||
opt.textContent = name + parent + ' · ' + String(dev.device_id).slice(0, 16);
|
||||
sel.appendChild(opt);
|
||||
});
|
||||
if (cur) sel.value = cur;
|
||||
applyScreenTarget(sel);
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
|
||||
/* ---------- Toast ---------- */
|
||||
var toastContainer = document.getElementById('toastContainer');
|
||||
function toast(msg, type) {
|
||||
if (!toastContainer) return;
|
||||
var el = document.createElement('div');
|
||||
el.className = 'toast ' + (type || 'success');
|
||||
el.textContent = msg;
|
||||
toastContainer.appendChild(el);
|
||||
setTimeout(function () { el.remove(); }, 2800);
|
||||
}
|
||||
|
||||
/* ---------- 通用请求 ---------- */
|
||||
function post(url, body, cb) {
|
||||
fetch(url, {
|
||||
method: 'POST',
|
||||
headers: body ? { 'Content-Type': 'application/json' } : undefined,
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
})
|
||||
.then(function (r) { return r.json().catch(function () { return {}; }); })
|
||||
.then(function (d) {
|
||||
if (d && d.ok === false) { toast(d.error || '操作失败', 'error'); return; }
|
||||
if (cb) cb(d);
|
||||
})
|
||||
.catch(function () { toast('网络错误', 'error'); });
|
||||
}
|
||||
function reload(msg, delay) {
|
||||
if (msg) toast(msg);
|
||||
setTimeout(function () { location.reload(); }, delay || 600);
|
||||
}
|
||||
|
||||
/* ---------- 播放状态（SSE，仅更新状态文字，不自动重载页面） ---------- */
|
||||
var statusText = document.getElementById('statusText');
|
||||
try {
|
||||
var es = new EventSource('/api/events');
|
||||
es.onmessage = function (e) {
|
||||
var msg; try { msg = JSON.parse(e.data); } catch (err) { return; }
|
||||
if (msg.action === 'state_update' && msg.state && statusText) {
|
||||
var s = msg.state;
|
||||
if (!s.status || s.status === 'idle' || !s.name) statusText.textContent = '等待大屏连接...';
|
||||
else if (s.status === 'playing') statusText.textContent = '正在播放 — ' + s.name;
|
||||
else if (s.status === 'paused') statusText.textContent = '已暂停 — ' + s.name;
|
||||
}
|
||||
};
|
||||
} catch (err) { /* ignore */ }
|
||||
|
||||
/* =========================================================
|
||||
连接状态轮询（/api/display/state）
|
||||
========================================================= */
|
||||
var connEls = {
|
||||
api: document.getElementById('connApi'),
|
||||
mqtt: document.getElementById('connMqtt'),
|
||||
screens: document.getElementById('connScreens'),
|
||||
lastCmd: document.getElementById('connLastCmd'),
|
||||
};
|
||||
function setConn(id, ok, value) {
|
||||
var el = connEls[id];
|
||||
if (!el) return;
|
||||
var dot = el.querySelector('.conn-dot');
|
||||
var val = el.querySelector('.conn-value');
|
||||
if (dot) dot.className = 'conn-dot ' + (ok ? 'on' : 'off');
|
||||
if (val) val.textContent = value;
|
||||
}
|
||||
function pollStatus() {
|
||||
fetch('/api/display/state', { cache: 'no-store' })
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
setConn('api', true, '在线');
|
||||
setConn('mqtt', !!d.mqtt_connected, d.mqtt_connected ? '已连接（' + (d.mqtt_host || '') + '）' : '未连接');
|
||||
setConn('screens', (d.screens_online || 0) > 0, (d.screens_online || 0) + ' 台');
|
||||
loadScreens(); // 刷新目标屏幕下拉（按 MQTT 客户端）
|
||||
var lc = d.last_command;
|
||||
if (lc) {
|
||||
var names = { navigate: '页面切换', play: '播放', pause: '暂停', next: '下一项', prev: '上一项',
|
||||
'play_target': '指定播放', alert: '通知', show_card: '卡片', set_mode: '播放模式' };
|
||||
var when = new Date(lc.ts).toLocaleTimeString('zh-CN', { hour12: false });
|
||||
setConn('lastCmd', !!lc.published,
|
||||
(names[lc.action] || lc.action) + ' · ' + (lc.published ? '已广播' : '未广播') + ' · ' + when);
|
||||
}
|
||||
})
|
||||
.catch(function () {
|
||||
setConn('api', false, '离线');
|
||||
setConn('mqtt', false, '未知');
|
||||
setConn('screens', false, '未知');
|
||||
});
|
||||
}
|
||||
if (connEls.api) { pollStatus(); setInterval(pollStatus, 3000); }
|
||||
// 目标屏幕下拉：变更时记录目标 client_id + role；初始加载一次
|
||||
var screenSel = document.getElementById('screenTarget');
|
||||
if (screenSel) {
|
||||
screenSel.addEventListener('change', function () { applyScreenTarget(screenSel); });
|
||||
loadScreens();
|
||||
}
|
||||
|
||||
/* =========================================================
|
||||
事件委托：页面切换 / 播放控制 / 媒体操作
|
||||
========================================================= */
|
||||
// 输入框回车提交
|
||||
document.addEventListener('keydown', function (e) {
|
||||
if (e.key !== 'Enter') return;
|
||||
if (e.target && e.target.id === 'mqttAiInput') { document.getElementById('mqttAiSend').click(); }
|
||||
else if (e.target && e.target.id === 'mqttAlertText') { document.getElementById('mqttAlertSend').click(); }
|
||||
});
|
||||
|
||||
document.addEventListener('click', function (e) {
|
||||
var btn = e.target.closest ? e.target.closest('[data-nav], [data-mqtt], [data-control], [data-action], .copy-btn, [data-preview], #mqttAiSend, #mqttAlertSend') : null;
|
||||
if (!btn) return;
|
||||
e.stopPropagation();
|
||||
|
||||
// 预览
|
||||
if (btn.hasAttribute('data-preview')) { openPreview(btn.closest('.media-card')); return; }
|
||||
|
||||
// 复制 URL
|
||||
if (btn.classList.contains('copy-btn')) {
|
||||
var url = btn.closest('.media-url').getAttribute('data-url') || '';
|
||||
var full = /^https?:\/\//.test(url) ? url : window.location.origin + url;
|
||||
var done = function () { toast('链接已复制'); };
|
||||
if (navigator.clipboard && navigator.clipboard.writeText) {
|
||||
navigator.clipboard.writeText(full).then(done).catch(function () { fallbackCopy(full); done(); });
|
||||
} else { fallbackCopy(full); done(); }
|
||||
return;
|
||||
}
|
||||
|
||||
// 大屏 MQTT 指令（data-mqtt 携带完整 {action, params} JSON）
|
||||
if (btn.hasAttribute('data-mqtt')) {
|
||||
var cmd = null;
|
||||
try { cmd = JSON.parse(btn.getAttribute('data-mqtt')); } catch (err) { toast('指令 JSON 错误', 'error'); return; }
|
||||
// 屏幕终端精准控制：除「双屏开关」外，注入目标 client_id + role；全空=全局广播
|
||||
if (cmd.action !== 'dual_screen') { cmd.screen_id = screenId; cmd.screen_role = screenRole; }
|
||||
var tgtTxt = screenId
|
||||
? (cmd.screen_role === 'main' ? '主屏' : cmd.screen_role === 'secondary' ? '副屏' : '屏幕') + '·' + String(screenId).slice(0, 10)
|
||||
: '全部';
|
||||
post('/api/display/command', cmd, function (d) {
|
||||
toast(d && d.ok !== false ? ('指令已发送：' + (cmd.action || '') + ' → ' + tgtTxt) : '发送失败（MQTT 未连接？）',
|
||||
d && d.ok !== false ? 'ok' : 'error');
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// AI 提问输入框
|
||||
if (btn.id === 'mqttAiSend') {
|
||||
var text = (document.getElementById('mqttAiInput').value || '').trim();
|
||||
if (!text) { toast('请输入问题', 'error'); return; }
|
||||
post('/api/display/command', { action: 'ai_input', params: { text: text }, screen_id: screenId, screen_role: screenRole }, function () {
|
||||
toast('已发送问题到大屏 AI 页');
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 通知输入框
|
||||
if (btn.id === 'mqttAlertSend') {
|
||||
var alertText = (document.getElementById('mqttAlertText').value || '').trim();
|
||||
if (!alertText) { toast('请输入通知内容', 'error'); return; }
|
||||
post('/api/display/command', { action: 'alert', params: { text: alertText }, screen_id: screenId, screen_role: screenRole }, function () {
|
||||
toast('通知已发送');
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 页面切换（绑定目标屏幕：screen_id 精确某台；空=全部）
|
||||
if (btn.hasAttribute('data-nav')) {
|
||||
var page = btn.getAttribute('data-nav');
|
||||
var tgtTxt = screenId
|
||||
? (screenRole === 'main' ? '主屏' : screenRole === 'secondary' ? '副屏' : '屏幕') + '·' + String(screenId).slice(0, 10)
|
||||
: '全部';
|
||||
post('/api/display/command', { action: 'navigate', params: { page: page }, screen_id: screenId, screen_role: screenRole }, function () {
|
||||
toast('已切换大屏至「' + btn.textContent.trim() + '」 → ' + tgtTxt);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 播放控制
|
||||
if (btn.hasAttribute('data-control')) {
|
||||
var action = btn.getAttribute('data-control');
|
||||
if (action === 'minimize') post('/api/display/command', { action: 'minimize', params: {} });
|
||||
else post('/api/control', { action: action });
|
||||
return;
|
||||
}
|
||||
|
||||
// 媒体操作
|
||||
if (btn.hasAttribute('data-action')) {
|
||||
var card = btn.closest('.media-card');
|
||||
var path = card.getAttribute('data-path');
|
||||
var act = btn.getAttribute('data-action');
|
||||
if (act === 'play-now') {
|
||||
post('/api/playlist/play', { path: path }, function () { toast('已发送播放指令，大屏即将播放'); });
|
||||
} else if (act === 'add-playlist') {
|
||||
post('/api/playlist/add', { path: path }, function () { reload('已加入播放列表'); });
|
||||
} else if (act === 'remove-playlist') {
|
||||
post('/api/playlist/remove', { path: path }, function () { reload('已移出播放列表'); });
|
||||
} else if (act === 'delete') {
|
||||
if (!window.confirm('确定删除该文件？')) return;
|
||||
post('/api/delete', { path: path }, function () { reload('已删除'); });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function fallbackCopy(text) {
|
||||
var ta = document.createElement('textarea');
|
||||
ta.value = text; ta.style.position = 'fixed'; ta.style.opacity = '0';
|
||||
document.body.appendChild(ta); ta.select();
|
||||
try { document.execCommand('copy'); } catch (err) { /* ignore */ }
|
||||
document.body.removeChild(ta);
|
||||
}
|
||||
|
||||
/* =========================================================
|
||||
播放设置（change/input 事件）
|
||||
========================================================= */
|
||||
var playMode = document.getElementById('playMode');
|
||||
if (playMode) playMode.addEventListener('change', function () {
|
||||
post('/api/settings', { play_mode: playMode.value }, function () { toast('播放模式已更新'); });
|
||||
});
|
||||
|
||||
var imageDuration = document.getElementById('imageDuration');
|
||||
if (imageDuration) imageDuration.addEventListener('change', function () {
|
||||
var v = Number(imageDuration.value);
|
||||
if (v < 1 || v > 300) { toast('时长需在 1-300 秒', 'error'); imageDuration.value = 5; return; }
|
||||
post('/api/settings', { image_duration: v }, function () { toast('图片时长已更新'); });
|
||||
});
|
||||
|
||||
var volumeRange = document.getElementById('volumeRange');
|
||||
var volumeLabel = document.getElementById('volumeLabel');
|
||||
var volTimer = null;
|
||||
if (volumeRange && volumeLabel) {
|
||||
volumeRange.addEventListener('input', function () {
|
||||
volumeLabel.textContent = volumeRange.value + '%';
|
||||
if (volTimer) clearTimeout(volTimer);
|
||||
volTimer = setTimeout(function () {
|
||||
post('/api/settings', { volume: Number(volumeRange.value) }, function () { toast('媒体音量已更新'); });
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
|
||||
var sfxVolumeRange = document.getElementById('sfxVolumeRange');
|
||||
var sfxVolumeLabel = document.getElementById('sfxVolumeLabel');
|
||||
var sfxVolTimer = null;
|
||||
if (sfxVolumeRange && sfxVolumeLabel) {
|
||||
sfxVolumeRange.addEventListener('input', function () {
|
||||
sfxVolumeLabel.textContent = sfxVolumeRange.value + '%';
|
||||
if (sfxVolTimer) clearTimeout(sfxVolTimer);
|
||||
sfxVolTimer = setTimeout(function () {
|
||||
post('/api/settings', { sfx_volume: Number(sfxVolumeRange.value) }, function () { toast('音效音量已更新'); });
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
|
||||
/* ---------- 显示控制 ---------- */
|
||||
var btnFullscreen = document.getElementById('btnFullscreen');
|
||||
var btnAutostart = document.getElementById('btnAutostart');
|
||||
function renderDisplay() {
|
||||
if (btnFullscreen) btnFullscreen.textContent = fullscreen ? '切换窗口模式' : '切换大屏模式';
|
||||
if (btnAutostart) { btnAutostart.textContent = autostart ? '已开启' : '已关闭'; btnAutostart.classList.toggle('active', autostart); }
|
||||
}
|
||||
if (btnFullscreen) btnFullscreen.addEventListener('click', function () {
|
||||
fullscreen = !fullscreen; renderDisplay();
|
||||
post('/api/settings', { fullscreen: fullscreen });
|
||||
});
|
||||
if (btnAutostart) btnAutostart.addEventListener('click', function () {
|
||||
autostart = !autostart; renderDisplay();
|
||||
post('/api/settings', { autostart: autostart });
|
||||
});
|
||||
renderDisplay();
|
||||
|
||||
/* ---------- 上传（带进度） ---------- */
|
||||
var btnUpload = document.getElementById('btnUpload');
|
||||
if (btnUpload) btnUpload.addEventListener('click', function () {
|
||||
var inp = document.getElementById('fileInput');
|
||||
var fileList = Array.from(inp.files || []);
|
||||
if (fileList.length === 0) { toast('请选择文件', 'error'); return; }
|
||||
var prog = document.getElementById('uploadProgress');
|
||||
var fill = document.getElementById('progressFill');
|
||||
var text = document.getElementById('progressText');
|
||||
prog.style.display = 'block';
|
||||
var done = 0;
|
||||
fileList.forEach(function (file) {
|
||||
var fd = new FormData();
|
||||
fd.append('file', file);
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/upload');
|
||||
xhr.upload.onprogress = function (e) {
|
||||
if (e.lengthComputable) {
|
||||
var pct = Math.round((e.loaded / e.total) * 100);
|
||||
var overall = Math.round(((done + e.loaded / e.total) / fileList.length) * 100);
|
||||
fill.style.width = overall + '%';
|
||||
text.textContent = file.name + ' (' + (done + 1) + '/' + fileList.length + ') — ' + pct + '%';
|
||||
}
|
||||
};
|
||||
xhr.onload = function () {
|
||||
done += 1;
|
||||
if (done === fileList.length) reload('上传完成');
|
||||
};
|
||||
xhr.onerror = function () { toast(file.name + ' 上传失败', 'error'); done += 1; };
|
||||
xhr.send(fd);
|
||||
});
|
||||
});
|
||||
|
||||
/* ---------- 远程 URL ---------- */
|
||||
var btnAddUrl = document.getElementById('btnAddUrl');
|
||||
var btnAddUrlPlay = document.getElementById('btnAddUrlPlay');
|
||||
function addUrlThen(play) {
|
||||
var inp = document.getElementById('urlInput');
|
||||
var url = (inp.value || '').trim();
|
||||
if (!url) { toast('请输入 URL', 'error'); return; }
|
||||
post('/api/media/add-url', { url: url }, function (d) {
|
||||
if (play) {
|
||||
post('/api/playlist/play', { path: url }, function () { toast('已加入并发送播放指令'); });
|
||||
} else {
|
||||
reload(d && d.duplicate ? 'URL 已存在于媒体库' : '已添加到媒体库');
|
||||
}
|
||||
});
|
||||
}
|
||||
if (btnAddUrl) btnAddUrl.addEventListener('click', function () { addUrlThen(false); });
|
||||
if (btnAddUrlPlay) btnAddUrlPlay.addEventListener('click', function () { addUrlThen(true); });
|
||||
|
||||
/* =========================================================
|
||||
预览弹窗
|
||||
========================================================= */
|
||||
var modal = document.getElementById('previewModal');
|
||||
var modalBody = document.getElementById('previewBody');
|
||||
var current = null;
|
||||
|
||||
function openPreview(card) {
|
||||
if (!card || !modal) return;
|
||||
var path = card.getAttribute('data-path');
|
||||
var type = card.getAttribute('data-type');
|
||||
var source = card.getAttribute('data-source');
|
||||
var name = card.getAttribute('data-name');
|
||||
var url = source === 'url' ? path : '/file/' + encodeURI(path);
|
||||
current = { path: path, url: url, type: type, name: name };
|
||||
document.getElementById('previewTitle').textContent = name;
|
||||
modalBody.innerHTML = type === 'image'
|
||||
? '<img src="' + url + '" alt="' + name + '">'
|
||||
: '<video src="' + url + '" controls autoplay playsinline></video>';
|
||||
document.getElementById('previewMeta').innerHTML =
|
||||
'<span class="badge ' + (source === 'url' ? 'badge-url' : 'badge-local') + '">' + (source === 'url' ? '远程' : '本地') + '</span>' +
|
||||
'<span class="badge ' + (type === 'video' ? 'badge-video' : 'badge-image') + '">' + (type === 'video' ? '视频' : '图片') + '</span>';
|
||||
modal.style.display = 'flex';
|
||||
}
|
||||
function closePreview() {
|
||||
var v = modalBody.querySelector('video');
|
||||
if (v) { v.pause(); v.removeAttribute('src'); v.load(); }
|
||||
modalBody.innerHTML = '';
|
||||
modal.style.display = 'none';
|
||||
current = null;
|
||||
}
|
||||
document.getElementById('previewClose').addEventListener('click', closePreview);
|
||||
if (modal) modal.addEventListener('click', function (e) {
|
||||
if (e.target === modal) closePreview();
|
||||
});
|
||||
document.addEventListener('keydown', function (e) { if (e.key === 'Escape') closePreview(); });
|
||||
document.getElementById('previewAdd').addEventListener('click', function () {
|
||||
if (!current) return;
|
||||
post('/api/playlist/add', { path: current.path }, function () { closePreview(); reload('已加入播放列表'); });
|
||||
});
|
||||
document.getElementById('previewPlay').addEventListener('click', function () {
|
||||
if (!current) return;
|
||||
post('/api/playlist/play', { path: current.path }, function () { closePreview(); toast('已发送播放指令'); });
|
||||
});
|
||||
document.getElementById('previewDelete').addEventListener('click', function () {
|
||||
if (!current) return;
|
||||
if (!window.confirm('确定删除该文件？')) return;
|
||||
post('/api/delete', { path: current.path }, function () { closePreview(); reload('已删除'); });
|
||||
});
|
||||
})();
|
||||
/* ---------- 语音 AI 测试（主动问候） ---------- */
|
||||
(function () {
|
||||
var greetBtn = document.getElementById('btnVoiceGreet');
|
||||
var greetInput = document.getElementById('voiceGreetText');
|
||||
var greetHint = document.getElementById('voiceGreetHint');
|
||||
if (!greetBtn || !greetInput) return;
|
||||
greetBtn.addEventListener('click', function () {
|
||||
var text = (greetInput.value || '').trim();
|
||||
if (!text) { toast('请输入问候语'); return; }
|
||||
greetBtn.disabled = true;
|
||||
greetBtn.textContent = '发送中…';
|
||||
fetch('/api/display/command', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'voice_start', params: { text: text } }),
|
||||
})
|
||||
.then(function (r) { return r.json().catch(function () { return {}; }); })
|
||||
.then(function (d) {
|
||||
greetBtn.disabled = false;
|
||||
greetBtn.textContent = '播报问候';
|
||||
if (d && d.ok === false) { toast('发送失败：MQTT 未发布（' + (d.error || '未知') + '）', 'error'); return; }
|
||||
if (greetHint) greetHint.textContent = '✅ 已发送：展播端将进入语音对话页并主动播报「' + text + '」，请留意展播端声音。';
|
||||
toast('语音问候已发送');
|
||||
})
|
||||
.catch(function () {
|
||||
greetBtn.disabled = false;
|
||||
greetBtn.textContent = '播报问候';
|
||||
toast('发送失败：后端不可达', 'error');
|
||||
});
|
||||
});
|
||||
})();
|
||||
@@ -0,0 +1,211 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* AudioWorkletProcessor that plays back Float32 mono samples received from
|
||||
* the main thread, upsampling whatever incoming rate the server uses
|
||||
* (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz).
|
||||
*
|
||||
* Lifecycle / messaging:
|
||||
*
|
||||
* main -> worklet:
|
||||
* { kind: "config", inputRate: 16000, muted: false } one-shot at startup
|
||||
* { kind: "audio", samples: Float32Array } per chunk
|
||||
* { kind: "clear" } wipe queue (barge-in)
|
||||
*
|
||||
* worklet -> main:
|
||||
* { kind: "stats", queuedMs, played } every ~250 ms
|
||||
* { kind: "underrun" } every time the queue
|
||||
* runs dry mid-playback
|
||||
*
|
||||
* `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the
|
||||
* same audio, so the page mutes its own playback to avoid double audio. When
|
||||
* the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page
|
||||
* plays the backend audio itself.
|
||||
*
|
||||
* IMPORTANT: the read position only advances while audio is actually being
|
||||
* played (`_playing`). Advancing during idle would drift `_readIdx` far past
|
||||
* the next chunk's length and turn every read into NaN (silence) once audio
|
||||
* arrives.
|
||||
*/
|
||||
|
||||
const STATS_INTERVAL_FRAMES = 12000;
|
||||
const FADE_FRAMES = 32;
|
||||
|
||||
class AudioPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this._inputRate = 16000;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
this._muted = false;
|
||||
this._queue = [];
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._framesSinceStats = 0;
|
||||
this._totalPlayed = 0;
|
||||
this._dbgReceived = 0;
|
||||
this._dbgOutPeak = 0;
|
||||
this._fadeIn = 0;
|
||||
this._fadeOut = 0;
|
||||
|
||||
this.port.onmessage = (e) => {
|
||||
const data = e.data;
|
||||
if (!data || typeof data !== "object") return;
|
||||
switch (data.kind) {
|
||||
case "config":
|
||||
if (typeof data.inputRate === "number" && data.inputRate > 0) {
|
||||
this._inputRate = data.inputRate;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
}
|
||||
if (typeof data.muted === "boolean") this._muted = data.muted;
|
||||
break;
|
||||
case "audio": {
|
||||
// Accept Float32Array, a transferred ArrayBuffer, or a plain
|
||||
// {buffer, byteOffset, length} descriptor: cross-realm structured
|
||||
// cloning can deliver the samples in any of these shapes.
|
||||
let arr = data.samples;
|
||||
let f32 = null;
|
||||
if (arr instanceof Float32Array) {
|
||||
f32 = arr;
|
||||
} else if (arr instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr);
|
||||
} else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2);
|
||||
}
|
||||
if (f32 && f32.length > 0) {
|
||||
this._queue.push(f32);
|
||||
this._dbgReceived += 1;
|
||||
if (!this._playing) {
|
||||
this._playing = true;
|
||||
this._fadeIn = FADE_FRAMES;
|
||||
this._fadeOut = 0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "clear":
|
||||
this._queue.length = 0;
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._fadeOut = FADE_FRAMES;
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
_queuedSamples() {
|
||||
let total = -this._readIdx;
|
||||
for (const buf of this._queue) total += buf.length;
|
||||
return Math.max(0, total);
|
||||
}
|
||||
|
||||
/** Linear-interp read at the current fractional position. */
|
||||
_readInterpolated() {
|
||||
if (this._queue.length === 0) return null;
|
||||
const head = this._queue[0];
|
||||
const idx = this._readIdx;
|
||||
const frac = this._fracPos;
|
||||
|
||||
let a = head[idx];
|
||||
let b;
|
||||
if (idx + 1 < head.length) {
|
||||
b = head[idx + 1];
|
||||
} else if (this._queue.length > 1) {
|
||||
b = this._queue[1][0];
|
||||
} else {
|
||||
b = a;
|
||||
}
|
||||
return a + (b - a) * frac;
|
||||
}
|
||||
|
||||
/** Advance the read position by `stepRatio`; pop consumed buffers. */
|
||||
_advance() {
|
||||
this._fracPos += this._stepRatio;
|
||||
while (this._fracPos >= 1) {
|
||||
this._fracPos -= 1;
|
||||
this._readIdx += 1;
|
||||
}
|
||||
while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
|
||||
this._readIdx -= this._queue[0].length;
|
||||
this._queue.shift();
|
||||
}
|
||||
}
|
||||
|
||||
process(_, outputs) {
|
||||
const channels = outputs[0];
|
||||
if (!channels || channels.length === 0) return true;
|
||||
const out = channels[0];
|
||||
const stereo = channels.length > 1 ? channels[1] : null;
|
||||
|
||||
// Muted mode (Live Avatar WebRTC carries the audio): output silence.
|
||||
if (this._muted) {
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
out[i] = 0;
|
||||
if (stereo) stereo[i] = 0;
|
||||
}
|
||||
this._framesSinceStats += out.length;
|
||||
return true;
|
||||
}
|
||||
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
let sample = 0;
|
||||
|
||||
if (this._playing) {
|
||||
const v = this._readInterpolated();
|
||||
if (v === null) {
|
||||
// Underrun: ramp out cleanly to avoid clicks.
|
||||
sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
|
||||
this._lastSample = sample;
|
||||
if (Math.abs(sample) < 1e-4) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this.port.postMessage({ kind: "underrun" });
|
||||
}
|
||||
} else {
|
||||
sample = v;
|
||||
this._lastSample = v;
|
||||
this._advance();
|
||||
}
|
||||
|
||||
if (this._fadeIn > 0) {
|
||||
const gain = 1 - this._fadeIn / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeIn -= 1;
|
||||
}
|
||||
if (this._fadeOut > 0) {
|
||||
const gain = this._fadeOut / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeOut -= 1;
|
||||
if (this._fadeOut === 0) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this._totalPlayed += 1;
|
||||
const abs = sample < 0 ? -sample : sample;
|
||||
if (abs > this._dbgOutPeak) this._dbgOutPeak = abs;
|
||||
}
|
||||
|
||||
out[i] = sample;
|
||||
if (stereo) stereo[i] = sample;
|
||||
}
|
||||
|
||||
this._framesSinceStats += out.length;
|
||||
if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
|
||||
this.port.postMessage({
|
||||
kind: "stats",
|
||||
queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000),
|
||||
played: this._totalPlayed,
|
||||
received: this._dbgReceived,
|
||||
outPeak: this._dbgOutPeak,
|
||||
});
|
||||
this._framesSinceStats = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("audio-playback", AudioPlaybackProcessor);
|
||||
@@ -0,0 +1,159 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz)
|
||||
* down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it
|
||||
* back to the main thread in fixed-size chunks.
|
||||
*
|
||||
* The Hugging Face speech-to-speech WebSocket route expects the
|
||||
* `input_audio_buffer.append` payload at 16 kHz PCM16 mono.
|
||||
*
|
||||
* Design notes:
|
||||
* - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a
|
||||
* cheap low-pass before decimating. Good enough for voice STT; we lose
|
||||
* a tiny bit of >8 kHz content which the pipeline discards anyway.
|
||||
* - Output frames are emitted at the cadence dictated by `chunkMs`
|
||||
* (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime
|
||||
* server batches incoming audio so the cadence is flexible; 20-100 ms
|
||||
* is the sweet spot.
|
||||
* - Float -> Int16 saturates to [-1, 1] before scaling.
|
||||
* - Optional noise gate: per-chunk RMS decides open/closed against a
|
||||
* threshold; the gain ramps (fast attack, hold, slow release) so word
|
||||
* onsets aren't clipped and quiet tails don't click. The gate only
|
||||
* affects the audio we SEND; the main-thread visualiser taps the raw
|
||||
* mic separately. We post the chunk RMS up every frame so the Settings
|
||||
* mic meter can show the live level against the threshold.
|
||||
*/
|
||||
|
||||
const TARGET_RATE = 16000;
|
||||
const DEFAULT_CHUNK_MS = 40;
|
||||
// Gate envelope timing (fixed; only the threshold is user-tunable).
|
||||
const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive
|
||||
const GATE_HOLD_MS = 250; // stay open this long after the level drops back under
|
||||
const GATE_RELEASE_MS = 80; // then fade closed over this long (no click)
|
||||
|
||||
class MicCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor(options) {
|
||||
super();
|
||||
const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS;
|
||||
this._inputRate = sampleRate;
|
||||
this._ratio = this._inputRate / TARGET_RATE;
|
||||
this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000);
|
||||
this._scratch = new Float32Array(0);
|
||||
this._decimated = new Float32Array(this._chunkSamples16k);
|
||||
this._enabled = true;
|
||||
|
||||
// Noise gate state. Disabled by default (pure passthrough).
|
||||
this._gateEnabled = false;
|
||||
this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open
|
||||
this._gateGain = 1; // smoothed gain currently applied
|
||||
this._holdRemaining = 0; // samples left before the gate may start closing
|
||||
this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE));
|
||||
this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE));
|
||||
this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE);
|
||||
|
||||
this.port.onmessage = (e) => {
|
||||
const data = e.data;
|
||||
if (data?.kind === "enable") this._enabled = !!data.value;
|
||||
else if (data?.kind === "gate") {
|
||||
this._gateEnabled = !!data.enabled;
|
||||
// dB -> linear amplitude. When off, threshold 0 keeps the gate open.
|
||||
this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Append `incoming` to the internal scratch buffer, then emit as many
|
||||
* full output chunks as we have material for.
|
||||
* @param {Float32Array} incoming
|
||||
*/
|
||||
_ingest(incoming) {
|
||||
if (incoming.length === 0) return;
|
||||
const next = new Float32Array(this._scratch.length + incoming.length);
|
||||
next.set(this._scratch, 0);
|
||||
next.set(incoming, this._scratch.length);
|
||||
this._scratch = next;
|
||||
this._maybeEmit();
|
||||
}
|
||||
|
||||
_maybeEmit() {
|
||||
const r = this._ratio;
|
||||
const n = this._chunkSamples16k;
|
||||
const needIn = Math.ceil(n * r);
|
||||
const dec = this._decimated;
|
||||
while (this._scratch.length >= needIn) {
|
||||
// 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter.
|
||||
let sumSq = 0;
|
||||
if (Math.abs(r - 3) < 1e-6) {
|
||||
// 48 kHz -> 16 kHz fast path with boxcar lowpass.
|
||||
for (let i = 0; i < n; i++) {
|
||||
const idx = i * 3;
|
||||
const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3;
|
||||
dec[i] = s;
|
||||
sumSq += s * s;
|
||||
}
|
||||
} else {
|
||||
// Generic path: linear interpolation. Slower but works at any rate
|
||||
// (e.g. some Windows boxes report sampleRate=44100).
|
||||
for (let i = 0; i < n; i++) {
|
||||
const srcPos = i * r;
|
||||
const idx = Math.floor(srcPos);
|
||||
const frac = srcPos - idx;
|
||||
const a = this._scratch[idx];
|
||||
const b = this._scratch[idx + 1] ?? a;
|
||||
const s = a + (b - a) * frac;
|
||||
dec[i] = s;
|
||||
sumSq += s * s;
|
||||
}
|
||||
}
|
||||
const rms = Math.sqrt(sumSq / n);
|
||||
|
||||
// 2. Decide the gate target for this chunk, then ramp sample-by-sample.
|
||||
let target = 1;
|
||||
if (this._gateEnabled) {
|
||||
if (rms >= this._thresholdLin) {
|
||||
this._holdRemaining = this._holdSamples; // re-arm the hold
|
||||
} else if (this._holdRemaining > 0) {
|
||||
this._holdRemaining -= n; // coasting through the hold window
|
||||
} else {
|
||||
target = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Apply the (smoothed) gain and pack to Int16.
|
||||
const out = new Int16Array(n);
|
||||
let gain = this._gateGain;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const coef = target > gain ? this._attackCoef : this._releaseCoef;
|
||||
gain = target + (gain - target) * coef;
|
||||
const s = dec[i] * gain;
|
||||
const clamped = s < -1 ? -1 : s > 1 ? 1 : s;
|
||||
out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
|
||||
}
|
||||
this._gateGain = gain;
|
||||
|
||||
// Shift the scratch buffer to keep only the trailing unused samples.
|
||||
const consumed = Math.floor(n * r);
|
||||
this._scratch = this._scratch.slice(consumed);
|
||||
|
||||
// Live input level for the Settings meter (raw RMS, pre-gate).
|
||||
this.port.postMessage({ kind: "level", rms });
|
||||
|
||||
if (this._enabled) {
|
||||
this.port.postMessage(out.buffer, [out.buffer]);
|
||||
}
|
||||
// When disabled (mic muted) we silently consume input so the worklet
|
||||
// stays alive and the buffer never grows unbounded.
|
||||
}
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0 || !input[0]) return true;
|
||||
const mono = input[0];
|
||||
if (mono.length > 0) this._ingest(mono);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("mic-capture", MicCaptureProcessor);
|
||||
Reference in New Issue
Block a user