245 lines
8.6 KiB
JavaScript
245 lines
8.6 KiB
JavaScript
// Voice conversation page: hands-free loop in the browser.
|
|||
|
|
// listen -> detect speech (mic level) -> record one utterance -> POST
|
||
|
|
// /voice/turn -> play the spoken answer -> listen again. Speaking over the
|
||
|
|
// answer interrupts it. All limits are constants below.
|
||
|
|
(function () {
|
||
|
|
"use strict";
|
||
|
|
var LEVEL_START = 0.04; // RMS that counts as speech
|
||
|
|
var LEVEL_BARGE = 0.09; // RMS that interrupts the assistant's voice
|
||
|
|
var ONSET_MS = 80; // speech must last this long to start recording
|
||
|
|
var BARGE_MS = 300; // ... and this long to interrupt playback
|
||
|
|
var SILENCE_MS = 900; // quiet this long ends the utterance
|
||
|
|
var MIN_SPEECH_MS = 300; // shorter utterances are noise, discarded
|
||
|
|
var MAX_UTTERANCE_MS = 30000;
|
||
|
|
var TICK_MS = 40;
|
||
|
|
|
||
|
|
var $ = function (id) { return document.getElementById(id); };
|
||
|
|
var toggle = $("v-toggle"), langSel = $("v-lang"), stateEl = $("v-state");
|
||
|
|
var levelEl = $("v-level"), errorEl = $("v-error"), logEl = $("v-log");
|
||
|
|
var form = $("v-form"), textEl = $("v-text"), resetBtn = $("v-reset");
|
||
|
|
|
||
|
|
var cid = newId();
|
||
|
|
var stream = null, ctx = null, analyser = null, buf = null, timer = null;
|
||
|
|
var recorder = null, chunks = [], recStart = 0, voicedMs = 0, quietMs = 0;
|
||
|
|
var onsetMs = 0, bargeMs = 0;
|
||
|
|
var player = null;
|
||
|
|
var state = "idle"; // idle | listening | hearing | thinking | speaking
|
||
|
|
|
||
|
|
function newId() {
|
||
|
|
var a = new Uint8Array(12);
|
||
|
|
crypto.getRandomValues(a);
|
||
|
|
return Array.prototype.map.call(a, function (b) {
|
||
|
|
return ("0" + b.toString(16)).slice(-2);
|
||
|
|
}).join("");
|
||
|
|
}
|
||
|
|
function setState(s) {
|
||
|
|
state = s;
|
||
|
|
stateEl.textContent = s;
|
||
|
|
stateEl.className = "pill " + s;
|
||
|
|
}
|
||
|
|
function showError(msg) { errorEl.textContent = msg || ""; }
|
||
|
|
function addLog(who, text) {
|
||
|
|
var li = document.createElement("li");
|
||
|
|
li.className = who;
|
||
|
|
var b = document.createElement("b");
|
||
|
|
b.textContent = who === "you" ? "You" : "Assistant";
|
||
|
|
li.appendChild(b);
|
||
|
|
li.appendChild(document.createTextNode(": " + text));
|
||
|
|
logEl.appendChild(li);
|
||
|
|
li.scrollIntoView({ block: "nearest" });
|
||
|
|
}
|
||
|
|
function errorText(res, body) {
|
||
|
|
try {
|
||
|
|
var j = JSON.parse(body);
|
||
|
|
return j.message || j.error || ("HTTP " + res.status);
|
||
|
|
} catch (e) { return "HTTP " + res.status; }
|
||
|
|
}
|
||
|
|
|
||
|
|
function rms() {
|
||
|
|
analyser.getByteTimeDomainData(buf);
|
||
|
|
var sum = 0;
|
||
|
|
for (var i = 0; i < buf.length; i++) {
|
||
|
|
var v = (buf[i] - 128) / 128;
|
||
|
|
sum += v * v;
|
||
|
|
}
|
||
|
|
return Math.sqrt(sum / buf.length);
|
||
|
|
}
|
||
|
|
|
||
|
|
function pickMime() {
|
||
|
|
var list = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"];
|
||
|
|
for (var i = 0; i < list.length; i++)
|
||
|
|
if (window.MediaRecorder && MediaRecorder.isTypeSupported(list[i])) return list[i];
|
||
|
|
return "";
|
||
|
|
}
|
||
|
|
|
||
|
|
function startRecording() {
|
||
|
|
chunks = [];
|
||
|
|
var mime = pickMime();
|
||
|
|
recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
||
|
|
recorder.ondataavailable = function (e) { if (e.data.size) chunks.push(e.data); };
|
||
|
|
recorder.start(250);
|
||
|
|
recStart = Date.now();
|
||
|
|
voicedMs = 0; quietMs = 0;
|
||
|
|
setState("hearing");
|
||
|
|
}
|
||
|
|
|
||
|
|
function stopRecording(send) {
|
||
|
|
var r = recorder, voiced = voicedMs;
|
||
|
|
recorder = null;
|
||
|
|
r.onstop = function () {
|
||
|
|
var blob = new Blob(chunks, { type: r.mimeType || "audio/webm" });
|
||
|
|
chunks = [];
|
||
|
|
if (send && voiced >= MIN_SPEECH_MS && state !== "idle") sendAudio(blob);
|
||
|
|
else if (state !== "idle") setState("listening");
|
||
|
|
};
|
||
|
|
r.stop();
|
||
|
|
}
|
||
|
|
|
||
|
|
function turnUrl() {
|
||
|
|
return "/voice/turn?cid=" + encodeURIComponent(cid) +
|
||
|
|
"&lang=" + encodeURIComponent(langSel.value);
|
||
|
|
}
|
||
|
|
|
||
|
|
function sendAudio(blob) {
|
||
|
|
setState("thinking");
|
||
|
|
post(turnUrl(), blob, blob.type || "audio/webm");
|
||
|
|
}
|
||
|
|
|
||
|
|
function post(url, body, type) {
|
||
|
|
showError("");
|
||
|
|
fetch(url, { method: "POST", headers: { "Content-Type": type }, body: body })
|
||
|
|
.then(function (res) { return res.text().then(function (t) { return [res, t]; }); })
|
||
|
|
.then(function (pair) {
|
||
|
|
var res = pair[0], text = pair[1];
|
||
|
|
if (!res.ok) {
|
||
|
|
showError(errorText(res, text));
|
||
|
|
if (res.status === 503) { stop(); return; }
|
||
|
|
if (state !== "idle") setState(stream ? "listening" : "idle");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
handleTurn(JSON.parse(text));
|
||
|
|
})
|
||
|
|
.catch(function (err) {
|
||
|
|
showError("Network error: " + err.message);
|
||
|
|
if (state !== "idle") setState(stream ? "listening" : "idle");
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function handleTurn(turn) {
|
||
|
|
if (!turn.transcript) { // nothing intelligible was heard
|
||
|
|
if (state !== "idle") setState(stream ? "listening" : "idle");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
addLog("you", turn.transcript);
|
||
|
|
addLog("bot", turn.reply);
|
||
|
|
if (state === "idle" && !stream) return;
|
||
|
|
play(turn);
|
||
|
|
}
|
||
|
|
|
||
|
|
function play(turn) {
|
||
|
|
stopPlayback();
|
||
|
|
player = new Audio("data:" + turn.mime + ";base64," + turn.audio_b64);
|
||
|
|
var mine = player;
|
||
|
|
mine.onended = function () {
|
||
|
|
if (player === mine) { player = null; if (stream) setState("listening"); else setState("idle"); }
|
||
|
|
};
|
||
|
|
mine.onerror = function () {
|
||
|
|
if (player === mine) { player = null; showError("Could not play audio"); if (stream) setState("listening"); }
|
||
|
|
};
|
||
|
|
setState(stream ? "speaking" : "idle");
|
||
|
|
mine.play().catch(function (e) { showError("Playback blocked: " + e.message); if (stream) setState("listening"); });
|
||
|
|
}
|
||
|
|
|
||
|
|
function stopPlayback() {
|
||
|
|
if (player) { var p = player; player = null; p.onended = null; p.pause(); }
|
||
|
|
}
|
||
|
|
|
||
|
|
function tick() {
|
||
|
|
var level = rms();
|
||
|
|
levelEl.style.width = Math.min(100, Math.round(level * 400)) + "%";
|
||
|
|
if (state === "listening") {
|
||
|
|
onsetMs = level > LEVEL_START ? onsetMs + TICK_MS : 0;
|
||
|
|
if (onsetMs >= ONSET_MS) { onsetMs = 0; startRecording(); }
|
||
|
|
} else if (state === "speaking") {
|
||
|
|
bargeMs = level > LEVEL_BARGE ? bargeMs + TICK_MS : 0;
|
||
|
|
if (bargeMs >= BARGE_MS) { bargeMs = 0; stopPlayback(); startRecording(); }
|
||
|
|
} else if (state === "hearing") {
|
||
|
|
if (level > LEVEL_START) { voicedMs += TICK_MS; quietMs = 0; }
|
||
|
|
else quietMs += TICK_MS;
|
||
|
|
if (quietMs >= SILENCE_MS || Date.now() - recStart >= MAX_UTTERANCE_MS)
|
||
|
|
stopRecording(true);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function start() {
|
||
|
|
showError("");
|
||
|
|
if (!navigator.mediaDevices || !window.MediaRecorder) {
|
||
|
|
showError("This browser has no microphone recording support.");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
navigator.mediaDevices.getUserMedia({
|
||
|
|
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
|
||
|
|
}).then(function (s) {
|
||
|
|
stream = s;
|
||
|
|
ctx = new (window.AudioContext || window.webkitAudioContext)();
|
||
|
|
var src = ctx.createMediaStreamSource(stream);
|
||
|
|
analyser = ctx.createAnalyser();
|
||
|
|
analyser.fftSize = 1024;
|
||
|
|
buf = new Uint8Array(analyser.fftSize);
|
||
|
|
src.connect(analyser);
|
||
|
|
onsetMs = 0; bargeMs = 0;
|
||
|
|
timer = setInterval(tick, TICK_MS);
|
||
|
|
toggle.textContent = "Stop conversation";
|
||
|
|
toggle.classList.add("live");
|
||
|
|
setState("listening");
|
||
|
|
}).catch(function (err) {
|
||
|
|
showError("Microphone unavailable: " + err.message);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function stop() {
|
||
|
|
if (timer) { clearInterval(timer); timer = null; }
|
||
|
|
if (recorder) { var r = recorder; recorder = null; r.onstop = null; try { r.stop(); } catch (e) {} }
|
||
|
|
stopPlayback();
|
||
|
|
if (stream) { stream.getTracks().forEach(function (t) { t.stop(); }); stream = null; }
|
||
|
|
if (ctx) { ctx.close(); ctx = null; }
|
||
|
|
levelEl.style.width = "0";
|
||
|
|
toggle.textContent = "Start conversation";
|
||
|
|
toggle.classList.remove("live");
|
||
|
|
setState("idle");
|
||
|
|
}
|
||
|
|
|
||
|
|
toggle.addEventListener("click", function () { if (stream) stop(); else start(); });
|
||
|
|
|
||
|
|
form.addEventListener("submit", function (ev) {
|
||
|
|
ev.preventDefault();
|
||
|
|
var text = textEl.value.trim();
|
||
|
|
if (!text || state === "thinking") return;
|
||
|
|
textEl.value = "";
|
||
|
|
stopPlayback();
|
||
|
|
setState("thinking");
|
||
|
|
post(turnUrl(), JSON.stringify({ text: text }), "application/json");
|
||
|
|
});
|
||
|
|
|
||
|
|
resetBtn.addEventListener("click", function () {
|
||
|
|
fetch("/voice/reset?cid=" + encodeURIComponent(cid), { method: "POST" });
|
||
|
|
cid = newId();
|
||
|
|
logEl.innerHTML = "";
|
||
|
|
showError("");
|
||
|
|
});
|
||
|
|
|
||
|
|
fetch("/google/status").then(function (r) { return r.json(); }).then(function (j) {
|
||
|
|
var cfg = $("v-config");
|
||
|
|
if (j.configured) {
|
||
|
|
cfg.textContent = "Google credentials found. Model: " + j.model + ".";
|
||
|
|
cfg.className = "ok";
|
||
|
|
toggle.disabled = false;
|
||
|
|
} else {
|
||
|
|
cfg.textContent = "No Google API key. Start the server with GOOGLE_API_KEY set.";
|
||
|
|
cfg.className = "error";
|
||
|
|
}
|
||
|
|
}).catch(function () { $("v-config").textContent = "Status unavailable."; });
|
||
|
|
|
||
|
|
setState("idle");
|
||
|
|
})();
|