feat: add deepsearch research system with CLI prune/clear and database schema

Implement a multi-agent deep web research subsystem including CLI commands for pruning expired jobs and clearing all artifacts, database tables for sessions/messages/URL cache with indexes, config paths for chroma storage, and internal embed URL for vector operations.
This commit is contained in:
2026-06-14 01:34:21 +00:00
parent c7770ee21a
commit 4ae0b0db5d
53 changed files with 3956 additions and 18 deletions
+417
View File
@@ -0,0 +1,417 @@
/* retoor <retoor@molodetz.nl> */
.ds-layout {
display: grid;
grid-template-columns: var(--sidebar-width) 1fr;
gap: var(--space-xl);
align-items: start;
}
.ds-main {
min-width: 0;
}
.ds-side-list {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
gap: var(--space-sm);
font-size: 0.8125rem;
color: var(--text-secondary);
line-height: 1.5;
}
.ds-side-list li {
padding-left: var(--space-md);
position: relative;
}
.ds-side-list li::before {
content: "";
position: absolute;
left: 0;
top: 0.55em;
width: 5px;
height: 5px;
border-radius: 50%;
background: var(--accent);
}
.ds-header h1 {
font-size: 1.75rem;
margin: 0 0 0.5rem;
}
.ds-header p {
color: var(--text-secondary);
margin: 0 0 var(--space-lg);
}
.ds-form-card,
.ds-card {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--radius-lg);
box-shadow: var(--shadow-sm);
padding: var(--space-xl);
margin-bottom: var(--space-xl);
}
.ds-input {
width: 100%;
background: var(--bg-input);
border: 1px solid var(--border);
border-radius: var(--radius-input);
color: var(--text-primary);
padding: var(--space-md);
font-family: inherit;
resize: vertical;
}
.ds-form-row {
display: flex;
flex-wrap: wrap;
align-items: flex-end;
gap: var(--space-md);
margin-top: var(--space-md);
}
.ds-field {
display: flex;
flex-direction: column;
gap: 4px;
font-size: 0.8125rem;
color: var(--text-secondary);
}
.ds-input-num {
width: 5rem;
background: var(--bg-input);
border: 1px solid var(--border);
border-radius: var(--radius-input);
color: var(--text-primary);
padding: var(--space-sm);
}
.ds-run-btn {
margin-left: auto;
}
.ds-form-error {
color: var(--danger, #e5484d);
margin: var(--space-md) 0 0;
font-size: 0.875rem;
}
.ds-live {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--radius-lg);
padding: var(--space-lg);
margin-bottom: var(--space-xl);
}
.ds-live-head {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: var(--space-md);
margin-bottom: var(--space-md);
}
.ds-live-status {
font-weight: 600;
color: var(--text-primary);
}
.ds-live-count {
color: var(--text-muted);
font-size: 0.875rem;
}
.ds-live-controls {
margin-left: auto;
display: flex;
gap: var(--space-sm);
}
.ds-ctl {
background: var(--bg-card-hover);
border: 1px solid var(--border);
color: var(--text-secondary);
padding: 4px 12px;
font-size: 0.8125rem;
}
.ds-progress {
height: 6px;
border-radius: 3px;
background: var(--border);
overflow: hidden;
}
.ds-progress-bar {
height: 100%;
width: 0;
background: var(--accent-gradient, var(--accent));
transition: width 0.3s ease;
}
.ds-timeline {
list-style: none;
margin: var(--space-md) 0 0;
padding: 0;
max-height: 300px;
overflow-y: auto;
font-size: 0.8125rem;
color: var(--text-secondary);
}
.ds-timeline li {
padding: 4px 0;
border-bottom: 1px solid var(--border);
}
.ds-done {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--radius-lg);
padding: var(--space-lg);
text-align: center;
}
.ds-session {
display: grid;
grid-template-columns: 1fr 360px;
gap: var(--space-xl);
align-items: start;
}
.ds-session-main {
min-width: 0;
}
.ds-session-head h1 {
font-size: 1.5rem;
margin: 0 0 var(--space-md);
}
.ds-metrics {
display: flex;
flex-wrap: wrap;
gap: var(--space-md);
margin-bottom: var(--space-md);
}
.ds-metric {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 4px 12px;
font-size: 0.8125rem;
color: var(--text-secondary);
}
.ds-metric strong {
color: var(--accent);
}
.ds-exports {
display: flex;
gap: var(--space-sm);
margin-bottom: var(--space-lg);
}
.ds-export {
background: var(--bg-card-hover);
border: 1px solid var(--border);
color: var(--text-secondary);
padding: 6px 14px;
font-size: 0.8125rem;
}
.ds-card h2 {
font-size: 1.125rem;
margin: 0 0 var(--space-md);
}
.ds-findings {
display: flex;
flex-direction: column;
gap: var(--space-sm);
}
.ds-finding {
border: 1px solid var(--border);
border-radius: var(--radius);
padding: var(--space-sm) var(--space-md);
}
.ds-finding summary {
display: flex;
justify-content: space-between;
cursor: pointer;
font-weight: 600;
color: var(--text-primary);
}
.ds-finding-confidence {
color: var(--accent);
font-size: 0.8125rem;
}
.ds-finding-detail {
margin-top: var(--space-sm);
color: var(--text-secondary);
line-height: 1.6;
}
.ds-gaps {
margin: 0;
padding-left: var(--space-lg);
color: var(--text-secondary);
line-height: 1.6;
}
.ds-sources {
margin: 0;
padding-left: var(--space-lg);
color: var(--text-secondary);
line-height: 1.8;
}
.ds-source-tag {
color: var(--text-muted);
font-size: 0.75rem;
margin-left: var(--space-sm);
}
.ds-chat-pane {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--radius-lg);
box-shadow: var(--shadow-sm);
display: flex;
flex-direction: column;
position: sticky;
top: var(--space-lg);
max-height: calc(100vh - 120px);
}
.ds-chat-head {
padding: var(--space-md) var(--space-lg);
border-bottom: 1px solid var(--border);
font-weight: 600;
}
.ds-chat-pending {
padding: var(--space-lg);
color: var(--text-muted);
}
dp-deepsearch-chat {
display: flex;
flex-direction: column;
min-height: 0;
flex: 1;
}
.ds-chat-log {
flex: 1;
overflow-y: auto;
padding: var(--space-md);
display: flex;
flex-direction: column;
gap: var(--space-md);
min-height: 240px;
}
.ds-chat-msg {
display: flex;
}
.ds-chat-msg.user {
justify-content: flex-end;
}
.ds-chat-bubble {
max-width: 90%;
padding: var(--space-sm) var(--space-md);
border-radius: var(--radius);
font-size: 0.875rem;
line-height: 1.5;
}
.ds-chat-msg.user .ds-chat-bubble {
background: var(--accent);
color: var(--white);
}
.ds-chat-msg.agent .ds-chat-bubble {
background: var(--bg-card-hover);
color: var(--text-primary);
}
.ds-chat-msg.error .ds-chat-bubble {
background: var(--bg-card-hover);
color: var(--danger, #e5484d);
}
.ds-chat-citations {
display: flex;
flex-direction: column;
gap: 2px;
margin-top: var(--space-sm);
font-size: 0.75rem;
}
.ds-chat-citations a {
color: var(--accent);
}
.ds-chat-status {
padding: 0 var(--space-md) var(--space-sm);
font-size: 0.75rem;
color: var(--text-muted);
}
.ds-chat-input {
display: flex;
gap: var(--space-sm);
padding: var(--space-md);
border-top: 1px solid var(--border);
}
.ds-chat-field {
flex: 1;
resize: none;
background: var(--bg-input);
border: 1px solid var(--border);
border-radius: var(--radius-input);
color: var(--text-primary);
padding: var(--space-sm);
font-family: inherit;
}
.ds-chat-send {
background: var(--accent);
color: var(--white);
border: none;
border-radius: var(--radius-input);
padding: 0 var(--space-md);
cursor: pointer;
}
@media (max-width: 900px) {
.ds-layout,
.ds-session {
grid-template-columns: 1fr;
}
.ds-chat-pane {
position: static;
max-height: 70vh;
}
}
@@ -0,0 +1,50 @@
// retoor <retoor@molodetz.nl>
const WRONG_WORKER_CODE = 4013;
const WRONG_WORKER_RETRY_MS = 200;
export class DeepsearchProgressSocket {
constructor(uid, handlers) {
this.handlers = handlers || {};
this.ws = null;
this._shouldRun = true;
const scheme = location.protocol === "https:" ? "wss://" : "ws://";
this.url = `${scheme}${location.host}/tools/deepsearch/${encodeURIComponent(uid)}/ws`;
}
connect() {
this._shouldRun = true;
this._open();
}
_open() {
this.ws = new WebSocket(this.url);
this.ws.addEventListener("message", (event) => {
let frame;
try {
frame = JSON.parse(event.data);
} catch {
return;
}
this._emit("onMessage", frame);
});
this.ws.addEventListener("close", (event) => {
if (event.code === WRONG_WORKER_CODE && this._shouldRun) {
window.setTimeout(() => this._open(), WRONG_WORKER_RETRY_MS);
return;
}
this._emit("onClose");
});
this.ws.addEventListener("error", () => this._emit("onError"));
}
close() {
this._shouldRun = false;
if (this.ws) this.ws.close();
}
_emit(name, payload) {
const handler = this.handlers[name];
if (typeof handler === "function") handler(payload);
}
}
+162
View File
@@ -0,0 +1,162 @@
// retoor <retoor@molodetz.nl>
import { Http } from "./Http.js";
import { DeepsearchProgressSocket } from "./DeepsearchProgressSocket.js";
export class DeepsearchTool {
constructor() {
this.root = document.querySelector("[data-deepsearch-tool]");
if (!this.root) return;
this.form = this.root.querySelector("[data-deepsearch-form]");
this.errorBox = this.root.querySelector("[data-deepsearch-error]");
this.live = this.root.querySelector("[data-deepsearch-live]");
this.statusText = this.root.querySelector("[data-deepsearch-status]");
this.countText = this.root.querySelector("[data-deepsearch-count]");
this.bar = this.root.querySelector("[data-deepsearch-bar]");
this.log = this.root.querySelector("[data-deepsearch-log]");
this.done = this.root.querySelector("[data-deepsearch-done]");
this.openLink = this.root.querySelector("[data-deepsearch-open]");
this.pauseBtn = this.root.querySelector("[data-deepsearch-pause]");
this.resumeBtn = this.root.querySelector("[data-deepsearch-resume]");
this.cancelBtn = this.root.querySelector("[data-deepsearch-cancel]");
this.runBtn = this.form.querySelector("[data-deepsearch-run]");
this.socket = null;
this.uid = null;
this._bind();
}
_bind() {
this.form.addEventListener("submit", (event) => {
event.preventDefault();
this._start();
});
this.pauseBtn.addEventListener("click", () => this._control("pause"));
this.resumeBtn.addEventListener("click", () => this._control("resume"));
this.cancelBtn.addEventListener("click", () => this._control("cancel"));
}
async _start() {
this._setError("");
const query = this.form.query.value.trim();
if (!query) return;
const params = {
query,
depth: this.form.depth ? this.form.depth.value : 2,
max_pages: this.form.max_pages ? this.form.max_pages.value : 12,
};
this._resetLive();
try {
const data = await Http.send("/tools/deepsearch/run", params);
this.uid = data.uid;
this._watch(data.uid);
} catch (error) {
this._setError(error.message || "Could not start the research.");
this.live.hidden = true;
this.runBtn.disabled = false;
}
}
_resetLive() {
if (this.socket) this.socket.close();
this.done.hidden = true;
this.live.hidden = false;
this.log.innerHTML = "";
this.bar.style.width = "0%";
this.countText.textContent = "";
this.statusText.textContent = "Starting…";
this.runBtn.disabled = true;
this.pauseBtn.hidden = false;
this.resumeBtn.hidden = true;
this.cancelBtn.hidden = false;
}
async _control(action) {
if (!this.uid) return;
try {
await Http.send(`/tools/deepsearch/${this.uid}/${action}`, {});
} catch (error) {
this._setError(error.message || "Control failed.");
return;
}
if (action === "pause") {
this.pauseBtn.hidden = true;
this.resumeBtn.hidden = false;
this.statusText.textContent = "Paused";
} else if (action === "resume") {
this.pauseBtn.hidden = false;
this.resumeBtn.hidden = true;
} else if (action === "cancel") {
this.pauseBtn.hidden = true;
this.resumeBtn.hidden = true;
this.cancelBtn.hidden = true;
this.statusText.textContent = "Cancelling…";
}
}
_watch(uid) {
this.socket = new DeepsearchProgressSocket(uid, {
onMessage: (frame) => this._frame(frame),
onClose: () => {},
});
this.socket.connect();
}
_frame(frame) {
const type = frame.type;
if (type === "stage") {
this.statusText.textContent = frame.message || "Working…";
this._appendLog(frame.message || frame.stage);
} else if (type === "queries") {
this._appendLog(`Planned ${frame.queries.length} queries`);
} else if (type === "candidates") {
this._appendLog(`Found ${frame.count} candidate sources`);
} else if (type === "progress") {
const total = frame.total || 1;
const done = frame.done || 0;
this.bar.style.width = `${Math.round((done / total) * 100)}%`;
this.countText.textContent = `${done} / ${total} sources`;
if (frame.message) this.statusText.textContent = frame.message;
} else if (type === "page_loaded") {
this.bar.style.width = `${Math.round((frame.done / frame.total) * 100)}%`;
this.countText.textContent = `${frame.done} / ${frame.total} sources`;
this._appendLog(`${frame.source} ${frame.title || frame.url}`);
} else if (type === "agent") {
this.statusText.textContent = frame.message || "Analysing";
this._appendLog(`Agent: ${frame.agent}`);
} else if (type === "report_ready") {
this.statusText.textContent = "Compiling report…";
} else if (type === "done") {
this.bar.style.width = "100%";
this.statusText.textContent = "Done";
this._finish(frame.session_url || `/tools/deepsearch/${this.uid}/session`);
} else if (type === "failed") {
this.statusText.textContent = "Failed";
this._setError(frame.message || frame.error || "The research failed.");
this.runBtn.disabled = false;
this.pauseBtn.hidden = true;
this.cancelBtn.hidden = true;
}
}
_finish(sessionUrl) {
this.runBtn.disabled = false;
this.pauseBtn.hidden = true;
this.resumeBtn.hidden = true;
this.cancelBtn.hidden = true;
this.done.hidden = false;
this.openLink.href = sessionUrl;
}
_appendLog(text) {
if (!text) return;
const item = document.createElement("li");
item.textContent = text;
this.log.prepend(item);
}
_setError(message) {
if (!this.errorBox) return;
this.errorBox.textContent = message;
this.errorBox.hidden = !message;
}
}
@@ -0,0 +1,216 @@
// retoor <retoor@molodetz.nl>
import { Component } from "./Component.js";
import { assetUrl } from "../assetVersion.js";
const CHAT_CSS_ID = "deepsearch-chat-css";
const RETRY_CODE = 4013;
const RETRY_MS = 200;
const RECONNECT_MS = 1500;
export class AppDeepsearchChat extends Component {
constructor() {
super();
this.ws = null;
this._shouldRun = true;
this._busy = false;
}
_ensureCss() {
if (document.getElementById(CHAT_CSS_ID)) return;
const link = document.createElement("link");
link.id = CHAT_CSS_ID;
link.rel = "stylesheet";
link.href = assetUrl("/static/css/deepsearch.css");
document.head.appendChild(link);
}
connectedCallback() {
if (this._built) return;
this._built = true;
this._ensureCss();
this.uid = this.attr("uid");
this.chatPath = this.attr("chat-ws") || `/tools/deepsearch/${this.uid}/chat`;
this._build();
this._open();
}
disconnectedCallback() {
this._shouldRun = false;
if (this.ws) this.ws.close();
}
_build() {
this.innerHTML = "";
this.log = document.createElement("div");
this.log.className = "ds-chat-log";
this.log.setAttribute("role", "log");
this.log.setAttribute("aria-live", "polite");
this.statusEl = document.createElement("div");
this.statusEl.className = "ds-chat-status";
this.statusEl.hidden = true;
this.form = document.createElement("form");
this.form.className = "ds-chat-input";
this.field = document.createElement("textarea");
this.field.className = "ds-chat-field";
this.field.rows = 1;
this.field.placeholder = "Ask about the gathered evidence...";
this.send = document.createElement("button");
this.send.type = "submit";
this.send.className = "ds-chat-send";
this.send.textContent = "Ask";
this.form.append(this.field, this.send);
this.append(this.log, this.statusEl, this.form);
this.form.addEventListener("submit", (event) => {
event.preventDefault();
this._submit(this.field.value);
});
this.field.addEventListener("keydown", (event) => {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
this._submit(this.field.value);
}
});
this.field.addEventListener("input", () => this._autosize());
}
_wsUrl() {
const scheme = location.protocol === "https:" ? "wss://" : "ws://";
return `${scheme}${location.host}${this.chatPath}`;
}
_open() {
this.ws = new WebSocket(this._wsUrl());
this.ws.addEventListener("message", (event) => {
let frame;
try {
frame = JSON.parse(event.data);
} catch {
return;
}
this._onMessage(frame);
});
this.ws.addEventListener("close", (event) => {
if (event.code === RETRY_CODE && this._shouldRun) {
window.setTimeout(() => this._open(), RETRY_MS);
return;
}
if (this._shouldRun) {
this._setStatus("Reconnecting...");
window.setTimeout(() => this._open(), RECONNECT_MS);
}
});
this.ws.addEventListener("error", () => this._setStatus("Connection error."));
}
_onMessage(frame) {
if (frame.type === "ready") {
this._setStatus("");
this._renderHistory(frame.history || []);
} else if (frame.type === "status") {
this._setStatus(frame.text);
} else if (frame.type === "reply") {
this._busy = false;
this._setStatus("");
this._agent(frame.text, frame.citations);
} else if (frame.type === "error") {
this._busy = false;
this._setStatus("");
this._errorLine(frame.text);
}
}
_submit(raw) {
const text = (raw || "").trim();
if (!text || this._busy) return;
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
this._errorLine("Not connected.");
return;
}
this._busy = true;
this._userLine(text);
this.field.value = "";
this._autosize();
this.ws.send(JSON.stringify({ type: "input", text }));
this._setStatus("Searching the research collection");
}
_renderHistory(messages) {
this.log.innerHTML = "";
messages.forEach((message) => {
if (message.role === "user") {
this._userLine(message.content);
} else if (message.role === "assistant" && message.content) {
this._agent(message.content, []);
}
});
}
_userLine(text) {
const row = document.createElement("div");
row.className = "ds-chat-msg user";
const bubble = document.createElement("div");
bubble.className = "ds-chat-bubble";
bubble.textContent = text;
row.appendChild(bubble);
this._append(row);
}
_agent(text, citations) {
const row = document.createElement("div");
row.className = "ds-chat-msg agent";
const bubble = document.createElement("div");
bubble.className = "ds-chat-bubble";
const content = document.createElement("dp-content");
content.setAttribute("data-render", "");
content.textContent = text;
bubble.appendChild(content);
if (citations && citations.length) {
const list = document.createElement("div");
list.className = "ds-chat-citations";
citations.forEach((cite) => {
const link = document.createElement("a");
link.href = cite.url;
link.target = "_blank";
link.rel = "noopener nofollow";
link.textContent = `[${cite.index}] ${cite.title}`;
list.appendChild(link);
});
bubble.appendChild(list);
}
row.appendChild(bubble);
this._append(row);
}
_errorLine(text) {
const row = document.createElement("div");
row.className = "ds-chat-msg error";
const bubble = document.createElement("div");
bubble.className = "ds-chat-bubble";
bubble.textContent = text;
row.appendChild(bubble);
this._append(row);
}
_append(node) {
const pinned = this.log.scrollHeight - this.log.scrollTop - this.log.clientHeight < 80;
this.log.appendChild(node);
if (pinned) this.log.scrollTop = this.log.scrollHeight;
}
_autosize() {
this.field.style.height = "auto";
this.field.style.height = `${Math.min(this.field.scrollHeight, 160)}px`;
}
_setStatus(text) {
this.statusEl.textContent = text || "";
this.statusEl.hidden = !text;
}
}
customElements.define("dp-deepsearch-chat", AppDeepsearchChat);