// Voice conversation page: hands-free loop in the browser. // listen -> detect speech (mic level) -> record one utterance -> POST // /voice/turn -> play the spoken answer -> listen again. Speaking over the // answer interrupts it. All limits are constants below. (function () { "use strict"; var LEVEL_START = 0.04; // RMS that counts as speech var LEVEL_BARGE = 0.09; // RMS that interrupts the assistant's voice var ONSET_MS = 80; // speech must last this long to start recording var BARGE_MS = 300; // ... and this long to interrupt playback var SILENCE_MS = 900; // quiet this long ends the utterance var MIN_SPEECH_MS = 300; // shorter utterances are noise, discarded var MAX_UTTERANCE_MS = 30000; var TICK_MS = 40; var $ = function (id) { return document.getElementById(id); }; var toggle = $("v-toggle"), langSel = $("v-lang"), stateEl = $("v-state"); var levelEl = $("v-level"), errorEl = $("v-error"), logEl = $("v-log"); var form = $("v-form"), textEl = $("v-text"), resetBtn = $("v-reset"); var cid = newId(); var stream = null, ctx = null, analyser = null, buf = null, timer = null; var recorder = null, chunks = [], recStart = 0, voicedMs = 0, quietMs = 0; var onsetMs = 0, bargeMs = 0; var player = null; var state = "idle"; // idle | listening | hearing | thinking | speaking function newId() { var a = new Uint8Array(12); crypto.getRandomValues(a); return Array.prototype.map.call(a, function (b) { return ("0" + b.toString(16)).slice(-2); }).join(""); } function setState(s) { state = s; stateEl.textContent = s; stateEl.className = "pill " + s; } function showError(msg) { errorEl.textContent = msg || ""; } function addLog(who, text) { var li = document.createElement("li"); li.className = who; var b = document.createElement("b"); b.textContent = who === "you" ? "You" : "Assistant"; li.appendChild(b); li.appendChild(document.createTextNode(": " + text)); logEl.appendChild(li); li.scrollIntoView({ block: "nearest" }); } function errorText(res, body) { try { var j = JSON.parse(body); return j.message || j.error || ("HTTP " + res.status); } catch (e) { return "HTTP " + res.status; } } function rms() { analyser.getByteTimeDomainData(buf); var sum = 0; for (var i = 0; i < buf.length; i++) { var v = (buf[i] - 128) / 128; sum += v * v; } return Math.sqrt(sum / buf.length); } function pickMime() { var list = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"]; for (var i = 0; i < list.length; i++) if (window.MediaRecorder && MediaRecorder.isTypeSupported(list[i])) return list[i]; return ""; } function startRecording() { chunks = []; var mime = pickMime(); recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream); recorder.ondataavailable = function (e) { if (e.data.size) chunks.push(e.data); }; recorder.start(250); recStart = Date.now(); voicedMs = 0; quietMs = 0; setState("hearing"); } function stopRecording(send) { var r = recorder, voiced = voicedMs; recorder = null; r.onstop = function () { var blob = new Blob(chunks, { type: r.mimeType || "audio/webm" }); chunks = []; if (send && voiced >= MIN_SPEECH_MS && state !== "idle") sendAudio(blob); else if (state !== "idle") setState("listening"); }; r.stop(); } function turnUrl() { return "/voice/turn?cid=" + encodeURIComponent(cid) + "&lang=" + encodeURIComponent(langSel.value); } function sendAudio(blob) { setState("thinking"); post(turnUrl(), blob, blob.type || "audio/webm"); } function post(url, body, type) { showError(""); fetch(url, { method: "POST", headers: { "Content-Type": type }, body: body }) .then(function (res) { return res.text().then(function (t) { return [res, t]; }); }) .then(function (pair) { var res = pair[0], text = pair[1]; if (!res.ok) { showError(errorText(res, text)); if (res.status === 503) { stop(); return; } if (state !== "idle") setState(stream ? "listening" : "idle"); return; } handleTurn(JSON.parse(text)); }) .catch(function (err) { showError("Network error: " + err.message); if (state !== "idle") setState(stream ? "listening" : "idle"); }); } function handleTurn(turn) { if (!turn.transcript) { // nothing intelligible was heard if (state !== "idle") setState(stream ? "listening" : "idle"); return; } addLog("you", turn.transcript); addLog("bot", turn.reply); if (state === "idle" && !stream) return; play(turn); } function play(turn) { stopPlayback(); player = new Audio("data:" + turn.mime + ";base64," + turn.audio_b64); var mine = player; mine.onended = function () { if (player === mine) { player = null; if (stream) setState("listening"); else setState("idle"); } }; mine.onerror = function () { if (player === mine) { player = null; showError("Could not play audio"); if (stream) setState("listening"); } }; setState(stream ? "speaking" : "idle"); mine.play().catch(function (e) { showError("Playback blocked: " + e.message); if (stream) setState("listening"); }); } function stopPlayback() { if (player) { var p = player; player = null; p.onended = null; p.pause(); } } function tick() { var level = rms(); levelEl.style.width = Math.min(100, Math.round(level * 400)) + "%"; if (state === "listening") { onsetMs = level > LEVEL_START ? onsetMs + TICK_MS : 0; if (onsetMs >= ONSET_MS) { onsetMs = 0; startRecording(); } } else if (state === "speaking") { bargeMs = level > LEVEL_BARGE ? bargeMs + TICK_MS : 0; if (bargeMs >= BARGE_MS) { bargeMs = 0; stopPlayback(); startRecording(); } } else if (state === "hearing") { if (level > LEVEL_START) { voicedMs += TICK_MS; quietMs = 0; } else quietMs += TICK_MS; if (quietMs >= SILENCE_MS || Date.now() - recStart >= MAX_UTTERANCE_MS) stopRecording(true); } } function start() { showError(""); if (!navigator.mediaDevices || !window.MediaRecorder) { showError("This browser has no microphone recording support."); return; } navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } }).then(function (s) { stream = s; ctx = new (window.AudioContext || window.webkitAudioContext)(); var src = ctx.createMediaStreamSource(stream); analyser = ctx.createAnalyser(); analyser.fftSize = 1024; buf = new Uint8Array(analyser.fftSize); src.connect(analyser); onsetMs = 0; bargeMs = 0; timer = setInterval(tick, TICK_MS); toggle.textContent = "Stop conversation"; toggle.classList.add("live"); setState("listening"); }).catch(function (err) { showError("Microphone unavailable: " + err.message); }); } function stop() { if (timer) { clearInterval(timer); timer = null; } if (recorder) { var r = recorder; recorder = null; r.onstop = null; try { r.stop(); } catch (e) {} } stopPlayback(); if (stream) { stream.getTracks().forEach(function (t) { t.stop(); }); stream = null; } if (ctx) { ctx.close(); ctx = null; } levelEl.style.width = "0"; toggle.textContent = "Start conversation"; toggle.classList.remove("live"); setState("idle"); } toggle.addEventListener("click", function () { if (stream) stop(); else start(); }); form.addEventListener("submit", function (ev) { ev.preventDefault(); var text = textEl.value.trim(); if (!text || state === "thinking") return; textEl.value = ""; stopPlayback(); setState("thinking"); post(turnUrl(), JSON.stringify({ text: text }), "application/json"); }); resetBtn.addEventListener("click", function () { fetch("/voice/reset?cid=" + encodeURIComponent(cid), { method: "POST" }); cid = newId(); logEl.innerHTML = ""; showError(""); }); fetch("/google/status").then(function (r) { return r.json(); }).then(function (j) { var cfg = $("v-config"); if (j.configured) { cfg.textContent = "Google credentials found. Model: " + j.model + "."; cfg.className = "ok"; toggle.disabled = false; } else { cfg.textContent = "No Google API key. Start the server with GOOGLE_API_KEY set."; cfg.className = "error"; } }).catch(function () { $("v-config").textContent = "Status unavailable."; }); setState("idle"); })();