Files
moloapp/example/assets/voice.js
T
retoorandClaude Sonnet 5.5 837de58136 feat(clients): google cloud voice: generic client, TTS, STT, Gemini audio, live voice page
src/google: GoogleClient over molo::clients (key in x-goog-api-key, never in
URLs), Service base, TextToSpeech, SpeechToText, Generative (replies, audio
in, audio out), Recognizer/Synthesizer seams with cloud, gemini and vertex
backends, VoicePipeline with bounded history. Example: /voice live
conversation page, /voice/{say,hear,turn,reset}, /google/status. Keys load
from env, .env.json.local, .env.json. Paid routes are excluded from locust.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 04:05:41 +02:00

245 lines
8.6 KiB
JavaScript

// Voice conversation page: hands-free loop in the browser.
// listen -> detect speech (mic level) -> record one utterance -> POST
// /voice/turn -> play the spoken answer -> listen again. Speaking over the
// answer interrupts it. All limits are constants below.
(function () {
"use strict";
var LEVEL_START = 0.04; // RMS that counts as speech
var LEVEL_BARGE = 0.09; // RMS that interrupts the assistant's voice
var ONSET_MS = 80; // speech must last this long to start recording
var BARGE_MS = 300; // ... and this long to interrupt playback
var SILENCE_MS = 900; // quiet this long ends the utterance
var MIN_SPEECH_MS = 300; // shorter utterances are noise, discarded
var MAX_UTTERANCE_MS = 30000;
var TICK_MS = 40;
var $ = function (id) { return document.getElementById(id); };
var toggle = $("v-toggle"), langSel = $("v-lang"), stateEl = $("v-state");
var levelEl = $("v-level"), errorEl = $("v-error"), logEl = $("v-log");
var form = $("v-form"), textEl = $("v-text"), resetBtn = $("v-reset");
var cid = newId();
var stream = null, ctx = null, analyser = null, buf = null, timer = null;
var recorder = null, chunks = [], recStart = 0, voicedMs = 0, quietMs = 0;
var onsetMs = 0, bargeMs = 0;
var player = null;
var state = "idle"; // idle | listening | hearing | thinking | speaking
function newId() {
var a = new Uint8Array(12);
crypto.getRandomValues(a);
return Array.prototype.map.call(a, function (b) {
return ("0" + b.toString(16)).slice(-2);
}).join("");
}
function setState(s) {
state = s;
stateEl.textContent = s;
stateEl.className = "pill " + s;
}
function showError(msg) { errorEl.textContent = msg || ""; }
function addLog(who, text) {
var li = document.createElement("li");
li.className = who;
var b = document.createElement("b");
b.textContent = who === "you" ? "You" : "Assistant";
li.appendChild(b);
li.appendChild(document.createTextNode(": " + text));
logEl.appendChild(li);
li.scrollIntoView({ block: "nearest" });
}
function errorText(res, body) {
try {
var j = JSON.parse(body);
return j.message || j.error || ("HTTP " + res.status);
} catch (e) { return "HTTP " + res.status; }
}
function rms() {
analyser.getByteTimeDomainData(buf);
var sum = 0;
for (var i = 0; i < buf.length; i++) {
var v = (buf[i] - 128) / 128;
sum += v * v;
}
return Math.sqrt(sum / buf.length);
}
function pickMime() {
var list = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"];
for (var i = 0; i < list.length; i++)
if (window.MediaRecorder && MediaRecorder.isTypeSupported(list[i])) return list[i];
return "";
}
function startRecording() {
chunks = [];
var mime = pickMime();
recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
recorder.ondataavailable = function (e) { if (e.data.size) chunks.push(e.data); };
recorder.start(250);
recStart = Date.now();
voicedMs = 0; quietMs = 0;
setState("hearing");
}
function stopRecording(send) {
var r = recorder, voiced = voicedMs;
recorder = null;
r.onstop = function () {
var blob = new Blob(chunks, { type: r.mimeType || "audio/webm" });
chunks = [];
if (send && voiced >= MIN_SPEECH_MS && state !== "idle") sendAudio(blob);
else if (state !== "idle") setState("listening");
};
r.stop();
}
function turnUrl() {
return "/voice/turn?cid=" + encodeURIComponent(cid) +
"&lang=" + encodeURIComponent(langSel.value);
}
function sendAudio(blob) {
setState("thinking");
post(turnUrl(), blob, blob.type || "audio/webm");
}
function post(url, body, type) {
showError("");
fetch(url, { method: "POST", headers: { "Content-Type": type }, body: body })
.then(function (res) { return res.text().then(function (t) { return [res, t]; }); })
.then(function (pair) {
var res = pair[0], text = pair[1];
if (!res.ok) {
showError(errorText(res, text));
if (res.status === 503) { stop(); return; }
if (state !== "idle") setState(stream ? "listening" : "idle");
return;
}
handleTurn(JSON.parse(text));
})
.catch(function (err) {
showError("Network error: " + err.message);
if (state !== "idle") setState(stream ? "listening" : "idle");
});
}
function handleTurn(turn) {
if (!turn.transcript) { // nothing intelligible was heard
if (state !== "idle") setState(stream ? "listening" : "idle");
return;
}
addLog("you", turn.transcript);
addLog("bot", turn.reply);
if (state === "idle" && !stream) return;
play(turn);
}
function play(turn) {
stopPlayback();
player = new Audio("data:" + turn.mime + ";base64," + turn.audio_b64);
var mine = player;
mine.onended = function () {
if (player === mine) { player = null; if (stream) setState("listening"); else setState("idle"); }
};
mine.onerror = function () {
if (player === mine) { player = null; showError("Could not play audio"); if (stream) setState("listening"); }
};
setState(stream ? "speaking" : "idle");
mine.play().catch(function (e) { showError("Playback blocked: " + e.message); if (stream) setState("listening"); });
}
function stopPlayback() {
if (player) { var p = player; player = null; p.onended = null; p.pause(); }
}
function tick() {
var level = rms();
levelEl.style.width = Math.min(100, Math.round(level * 400)) + "%";
if (state === "listening") {
onsetMs = level > LEVEL_START ? onsetMs + TICK_MS : 0;
if (onsetMs >= ONSET_MS) { onsetMs = 0; startRecording(); }
} else if (state === "speaking") {
bargeMs = level > LEVEL_BARGE ? bargeMs + TICK_MS : 0;
if (bargeMs >= BARGE_MS) { bargeMs = 0; stopPlayback(); startRecording(); }
} else if (state === "hearing") {
if (level > LEVEL_START) { voicedMs += TICK_MS; quietMs = 0; }
else quietMs += TICK_MS;
if (quietMs >= SILENCE_MS || Date.now() - recStart >= MAX_UTTERANCE_MS)
stopRecording(true);
}
}
function start() {
showError("");
if (!navigator.mediaDevices || !window.MediaRecorder) {
showError("This browser has no microphone recording support.");
return;
}
navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
}).then(function (s) {
stream = s;
ctx = new (window.AudioContext || window.webkitAudioContext)();
var src = ctx.createMediaStreamSource(stream);
analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
buf = new Uint8Array(analyser.fftSize);
src.connect(analyser);
onsetMs = 0; bargeMs = 0;
timer = setInterval(tick, TICK_MS);
toggle.textContent = "Stop conversation";
toggle.classList.add("live");
setState("listening");
}).catch(function (err) {
showError("Microphone unavailable: " + err.message);
});
}
function stop() {
if (timer) { clearInterval(timer); timer = null; }
if (recorder) { var r = recorder; recorder = null; r.onstop = null; try { r.stop(); } catch (e) {} }
stopPlayback();
if (stream) { stream.getTracks().forEach(function (t) { t.stop(); }); stream = null; }
if (ctx) { ctx.close(); ctx = null; }
levelEl.style.width = "0";
toggle.textContent = "Start conversation";
toggle.classList.remove("live");
setState("idle");
}
toggle.addEventListener("click", function () { if (stream) stop(); else start(); });
form.addEventListener("submit", function (ev) {
ev.preventDefault();
var text = textEl.value.trim();
if (!text || state === "thinking") return;
textEl.value = "";
stopPlayback();
setState("thinking");
post(turnUrl(), JSON.stringify({ text: text }), "application/json");
});
resetBtn.addEventListener("click", function () {
fetch("/voice/reset?cid=" + encodeURIComponent(cid), { method: "POST" });
cid = newId();
logEl.innerHTML = "";
showError("");
});
fetch("/google/status").then(function (r) { return r.json(); }).then(function (j) {
var cfg = $("v-config");
if (j.configured) {
cfg.textContent = "Google credentials found. Model: " + j.model + ".";
cfg.className = "ok";
toggle.disabled = false;
} else {
cfg.textContent = "No Google API key. Start the server with GOOGLE_API_KEY set.";
cfg.className = "error";
}
}).catch(function () { $("v-config").textContent = "Status unavailable."; });
setState("idle");
})();