retoor/tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

detz: M0 scaffold

Browse Source
This commit is contained in:
retoor
2026-10-24 06:40:09 +02:00
parent fc436aeb24
commit 89918cbf0e
32 changed files with 599 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tai.py
+384 -3
View File
@@ -313,7 +313,7 @@ SCHEDULER_INTERVAL = 30
HTTP_TIMEOUT = 60
STREAM_TIMEOUT = 300
APPROVAL_MAX_RETRIES = 3
VERIFY_MAX_ROUNDS = 2
VERIFY_MAX_ROUNDS = 9
VERIFY_TIMEOUT = 120
VERIFY_STEPS = 6
VERIFY_MAX_CHARS = 6000
@@ -334,6 +334,11 @@ EVAL_MAX_GOLDENS = 200
EVAL_MAX_PAIRS = 200
EVAL_MIN_TOKEN = 6
EVAL_PREFIX_MIN = 4
RAG_STATS_EVENTS = 5000
RAG_STATS_TOP = 20
RAG_STATS_QUERIES = 20
RAG_STATS_SAMPLE = 10
RAG_STATS_KINDS = 30
EDGE_HOST = "speech.platform.bing.com"
EDGE_WS_PATH = "/consumer/speech/synthesize/readaloud/edge/v1"
@@ -2599,7 +2604,7 @@ BUNDLED_SKILLS = {
"deep-research": {
"SKILL.md": r"""---
name: deep-research
description: "End-to-end deep research that turns a question into a cited, verifiable report. Use whenever the answer needs many external sources rather than a single lookup: technology/library selection, code and architecture questions, academic or medical literature, market and company facts, policy and standards, and 'what's the latest' or capability/benchmark questions. Routes each sub-question to the most authoritative sources for its domain (GitHub and official docs for code, arXiv/OpenAlex/Semantic Scholar for scholarship, PubMed/Europe PMC for medicine, SEC EDGAR for company facts, vendor and official pages for products), gathers evidence in parallel, triangulates claims, and writes a structured report with dated citations. Trigger on 调研/研究/调查/综述/对比一下 X、帮我查清楚 X、X 的最新进展/评测数据/方案有哪些, and any request for a research report, literature review, landscape survey, or source-backed answer - even when the user does not say the word 'research'."
description: "End-to-end deep research that turns a question into a cited, verifiable report. Reach for it generously whenever external facts could help resolve the request: prefer it over a single lookup for technology/library selection, code and architecture questions, debugging or resolving an issue, academic or medical literature, market and company facts, policy and standards, and 'what's the latest' or capability/benchmark questions. Routes each sub-question to the most authoritative sources for its domain (GitHub and official docs for code, arXiv/OpenAlex/Semantic Scholar for scholarship, PubMed/Europe PMC for medicine, SEC EDGAR for company facts, vendor and official pages for products), gathers evidence in parallel, triangulates claims, and writes a structured report with dated citations. Trigger on 调研/研究/调查/综述/对比一下 X、帮我查清楚 X、X 的最新进展/评测数据/方案有哪些, troubleshooting and debugging questions, and any request for a research report, literature review, landscape survey, or source-backed answer - even when the user does not say the word 'research'."
---
# Deep Research
@@ -2610,9 +2615,10 @@ This skill is **self-contained and host-agnostic**. It defines the *method*; it
## When to use
Use it when the answer must be assembled from **multiple external sources** and needs to be current, comparative, or authoritative:
Use it generously whenever external facts could help resolve the request, especially when the answer needs to be current, comparative, or authoritative:
- technical / library / architecture selection, migration paths, "X vs Y"
- debugging or resolving an issue: errors, unexpected behavior, root-cause analysis
- literature reviews, "what does the research say about X"
- medical, clinical, or life-science questions
- market, company, competitive, or due-diligence facts
@@ -19535,6 +19541,372 @@ class RetrievalEval:
return "\n".join(lines)
class RagStats:
"""RAG usage and effectiveness stats (spec). One read-only rollup over
the vault tables plus the event log, printed by --stats-rag.
Definitions: shown means a record id appeared in a search-query event
hit list; read means a record-read event named the id; context uses is
shown plus read (both inject vault text into the LLM context, snippets
on show, full pages on read); read-through is reads divided by shows.
The events window holds the newest RAG_STATS_EVENTS rows; totals via
COUNT(*) are exact, window-derived shares are labeled as window shares.
Caps: RAG_STATS_* constants bound rows scanned, top lists, and samples."""
@staticmethod
def _one(db, sql, params=()):
try:
row = db.execute(sql, tuple(params)).fetchone()
except sqlite3.Error:
return None
return row[0] if row else None
@staticmethod
def _rows(db, sql, params=()):
try:
return db.execute(sql, tuple(params)).fetchall()
except sqlite3.Error:
return []
@staticmethod
def _parse_hits(raw):
kinds = {}
ranked = []
shown_records = []
shown_chunks = []
for piece in (raw or "").split(","):
piece = piece.strip()
if not piece:
continue
kind, sep, rest = piece.partition(":")
if not sep or kind not in ("record", "event", "audit", "chunk", "log"):
kinds["other"] = kinds.get("other", 0) + 1
ranked.append(piece)
continue
kinds[kind] = kinds.get(kind, 0) + 1
if kind == "record":
ranked.append(rest)
shown_records.append(rest)
elif kind == "chunk":
ranked.append(piece)
shown_chunks.append(rest or piece)
else:
ranked.append(piece)
return kinds, ranked, shown_records, shown_chunks
@staticmethod
def _parse_search_event(text):
try:
parts = dict(part.split("=", 1) for part in str(text or "").split("\t") if "=" in part)
query = (parts.get("q") or "").strip()
if not query:
return None
kinds, ranked, shown_records, shown_chunks = RagStats._parse_hits(parts.get("hits") or "")
try:
limit = int(parts.get("limit") or 0)
except (TypeError, ValueError):
limit = 0
return {"query": query, "limit": limit, "hits": len(ranked), "kinds": kinds,
"ranked": ranked, "shown_records": shown_records, "shown_chunks": shown_chunks}
except Exception:
return None
@staticmethod
def collect(store, profile, include_eval=True):
db = store.db
stats = {"profile": profile, "fts_ok": bool(getattr(store, "fts_ok", False)),
"window": RAG_STATS_EVENTS}
stats["events_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ?", (profile,)) or 0
row = RagStats._rows(db, "SELECT COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ?", (profile,))
stats["records_total"], stats["records_bytes"], stats["records_reads"] = (row[0][0] or 0, row[0][1] or 0, row[0][2] or 0) if row else (0, 0, 0)
stats["records_by_kind"] = [(r[0], r[1], r[2] or 0, r[3] or 0) for r in RagStats._rows(
db, "SELECT kind, COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", (profile,))]
stats["top_reads"] = [(r[0], r[1], r[2] or "", r[3] or 0, r[4] or 0) for r in RagStats._rows(
db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY reads DESC, updated DESC LIMIT ?", (profile, RAG_STATS_TOP))]
stats["top_size"] = [(r[0], r[1], r[2] or "", r[3] or 0, r[4] or 0) for r in RagStats._rows(
db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY size DESC LIMIT ?", (profile, RAG_STATS_TOP))]
stats["zero_read_records"] = RagStats._one(db, "SELECT COUNT(*) FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0", (profile,)) or 0
stats["never_read_sample"] = [r[0] for r in RagStats._rows(
db, "SELECT id FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0 ORDER BY updated DESC LIMIT ?", (profile, RAG_STATS_SAMPLE))]
stats["chunks_total"] = RagStats._one(db, "SELECT COUNT(*) FROM chunks WHERE profile = ?", (profile,)) or 0
stats["chunks_by_parent"] = [(r[0], r[1]) for r in RagStats._rows(
db, "SELECT parent_kind, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind ORDER BY COUNT(*) DESC", (profile,))]
stats["chunk_parents"] = RagStats._one(db, "SELECT COUNT(DISTINCT parent_kind || ':' || parent_id) FROM chunks WHERE profile = ?", (profile,)) or 0
stats["top_chunk_parents"] = [(r[0], r[1], r[2]) for r in RagStats._rows(
db, "SELECT parent_kind, parent_id, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind, parent_id ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_TOP))]
stats["orphan_chunks"] = RagStats._one(db, "SELECT COUNT(*) FROM chunks WHERE profile = ? AND parent_kind = 'record' AND parent_id NOT IN (SELECT id FROM records WHERE profile = ?)", (profile, profile)) or 0
tag_row = RagStats._rows(db, "SELECT COUNT(*), COUNT(DISTINCT tag), COUNT(DISTINCT item) FROM tags WHERE profile = ?", (profile,))
stats["tags_rows"], stats["tags_distinct"], stats["tags_items"] = (tag_row[0][0] or 0, tag_row[0][1] or 0, tag_row[0][2] or 0) if tag_row else (0, 0, 0)
try:
stats["top_tags"] = list(store.tag_counts("", RAG_STATS_TOP, profile))
except Exception:
stats["top_tags"] = []
stats["record_tag_rows"] = RagStats._one(db, "SELECT COUNT(*) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", (profile,)) or 0
stats["tagged_records"] = RagStats._one(db, "SELECT COUNT(DISTINCT item) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", (profile,)) or 0
stats["only_record_tag"] = RagStats._one(db, "SELECT COUNT(*) FROM (SELECT item FROM tags WHERE profile = ? AND item LIKE 'mem:%' GROUP BY item HAVING COUNT(*) = 1 AND MAX(tag) = 'record')", (profile,)) or 0
stats["edges_total"] = RagStats._one(db, "SELECT COUNT(*) FROM edges WHERE profile = ?", (profile,)) or 0
stats["edges_by_relation"] = [(r[0], r[1]) for r in RagStats._rows(
db, "SELECT relation, COUNT(*) FROM edges WHERE profile = ? GROUP BY relation ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_TOP))]
stats["top_degree"] = [(r[0], r[1]) for r in RagStats._rows(
db, "SELECT node, COUNT(*) FROM (SELECT src AS node FROM edges WHERE profile = ? UNION ALL SELECT dst AS node FROM edges WHERE profile = ?) GROUP BY node ORDER BY COUNT(*) DESC LIMIT ?", (profile, profile, RAG_STATS_TOP))]
stats["isolated_records"] = RagStats._one(db, "SELECT COUNT(*) FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?)", (profile, profile, profile)) or 0
stats["isolated_sample"] = [r[0] for r in RagStats._rows(
db, "SELECT id FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?) ORDER BY updated DESC LIMIT ?", (profile, profile, profile, RAG_STATS_SAMPLE))]
stats["fts_by_kind"] = []
if stats["fts_ok"]:
stats["fts_by_kind"] = [(r[0], r[1]) for r in RagStats._rows(db, "SELECT kind, COUNT(*) FROM fts_docs WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", (profile,))]
stats["events_by_kind"] = [(r[0], r[1], r[2]) for r in RagStats._rows(
db, "SELECT role, kind, COUNT(*) FROM events WHERE profile = ? GROUP BY role, kind ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_KINDS))]
span = RagStats._rows(db, "SELECT MIN(ts), MAX(ts) FROM events WHERE profile = ?", (profile,))
stats["events_first"], stats["events_last"] = (span[0][0], span[0][1]) if span else (None, None)
now = datetime.now(timezone.utc)
day_cut = (now - timedelta(days=1)).isoformat(timespec="seconds")
week_cut = (now - timedelta(days=7)).isoformat(timespec="seconds")
stats["searches_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query'", (profile,)) or 0
stats["reads_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read'", (profile,)) or 0
stats["searches_24h"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", (profile, day_cut)) or 0
stats["reads_24h"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", (profile, day_cut)) or 0
stats["searches_7d"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", (profile, week_cut)) or 0
stats["reads_7d"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", (profile, week_cut)) or 0
text_col = "tai_dec(text)" if store.seal.enabled else "text"
window_rows = RagStats._rows(db, "SELECT ts, role, kind, %s FROM events WHERE profile = ? ORDER BY id DESC LIMIT ?" % text_col, (profile, RAG_STATS_EVENTS))
window_rows.reverse()
searches = []
reads = []
for pos, row in enumerate(window_rows):
if row[1] == "tool" and row[2] == "search-query":
parsed = RagStats._parse_search_event(row[3] or "")
if parsed is not None:
parsed["pos"] = pos
parsed["ts"] = row[0]
searches.append(parsed)
elif row[1] == "tool" and row[2] == "record-read" and (row[3] or "").strip():
reads.append({"pos": pos, "id": (row[3] or "").strip()[:200], "ts": row[0]})
stats["window_events"] = len(window_rows)
stats["window_searches"] = len(searches)
stats["window_reads"] = len(reads)
hits_by_kind = {}
shown_counter = collections.Counter()
chunk_counter = collections.Counter()
query_counter = collections.Counter()
query_hits = {}
query_len = 0
query_terms = 0
zero_hit = []
dist = {"0": 0, "1": 0, "2-5": 0, "6+": 0}
for item in searches:
query_counter[item["query"]] += 1
query_hits.setdefault(item["query"], []).append(item["hits"])
query_len += len(item["query"])
query_terms += len(item["query"].split())
for kind, count in item["kinds"].items():
hits_by_kind[kind] = hits_by_kind.get(kind, 0) + count
for record_id in item["shown_records"]:
shown_counter[record_id] += 1
for chunk_id in item["shown_chunks"]:
chunk_counter[chunk_id] += 1
if item["hits"] == 0:
dist["0"] += 1
zero_hit.append(item["query"])
elif item["hits"] == 1:
dist["1"] += 1
elif item["hits"] <= 5:
dist["2-5"] += 1
else:
dist["6+"] += 1
stats["hits_by_kind"] = sorted(hits_by_kind.items(), key=lambda pair: (-pair[1], pair[0]))
stats["hits_dist"] = dist
stats["zero_hit_window"] = dist["0"]
stats["zero_hit_sample"] = zero_hit[:RAG_STATS_SAMPLE]
total_hits = sum(item["hits"] for item in searches)
stats["avg_hits"] = round(total_hits / len(searches), 2) if searches else 0.0
stats["avg_query_chars"] = round(query_len / len(searches), 1) if searches else 0.0
stats["avg_query_terms"] = round(query_terms / len(searches), 1) if searches else 0.0
stats["repeated_queries"] = sum(1 for count in query_counter.values() if count > 1)
stats["queries_distinct"] = len(query_counter)
top_queries = []
for query, count in query_counter.most_common(RAG_STATS_QUERIES):
shots = query_hits.get(query) or [0]
top_queries.append((query, count, round(sum(shots) / len(shots), 1)))
stats["top_queries"] = top_queries
stats["top_shown"] = shown_counter.most_common(RAG_STATS_TOP)
stats["top_shown_chunks"] = chunk_counter.most_common(RAG_STATS_TOP)
stats["shown_distinct"] = len(shown_counter)
read_counter = collections.Counter(item["id"] for item in reads)
stats["top_read_ids"] = read_counter.most_common(RAG_STATS_TOP)
stats["read_distinct"] = len(read_counter)
rank_dist = {"1": 0, "2": 0, "3": 0, "4-5": 0, "6+": 0, "unshown": 0}
read_follow = 0
for idx, item in enumerate(searches):
end = searches[idx + 1]["pos"] if idx + 1 < len(searches) else len(window_rows)
for row in window_rows[item["pos"] + 1:min(end, item["pos"] + 21)]:
if row[1] == "tool" and row[2] == "record-read" and (row[3] or "").strip():
read_follow += 1
rank = RetrievalEval.rank_of((row[3] or "").strip()[:200], item["ranked"])
if rank is None:
rank_dist["unshown"] += 1
elif rank == 1:
rank_dist["1"] += 1
elif rank == 2:
rank_dist["2"] += 1
elif rank == 3:
rank_dist["3"] += 1
elif rank <= 5:
rank_dist["4-5"] += 1
else:
rank_dist["6+"] += 1
break
stats["read_follow"] = read_follow
stats["rank_dist"] = rank_dist
shown_set = set(shown_counter)
read_set = set(read_counter)
stats["direct_reads"] = len(read_set - shown_set)
stats["direct_sample"] = sorted(read_set - shown_set)[:RAG_STATS_SAMPLE]
stats["shown_never_read"] = len(shown_set - read_set)
stats["shown_never_sample"] = sorted(shown_set - read_set)[:RAG_STATS_SAMPLE]
effectiveness = []
for record_id in set(list(shown_set) + list(read_set)):
shown = shown_counter.get(record_id, 0)
read = read_counter.get(record_id, 0)
effectiveness.append((record_id, shown, read, shown + read))
effectiveness.sort(key=lambda row: (-row[3], -row[2], row[0]))
stats["effectiveness"] = effectiveness[:RAG_STATS_TOP]
meta = {}
ids = [row[0] for row in stats["effectiveness"]]
if ids:
marks = ", ".join("?" * len(ids))
for r in RagStats._rows(db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? AND id IN (%s)" % marks, tuple([profile] + ids)):
meta[r[0]] = {"kind": r[1], "title": r[2] or "", "reads_col": r[3] or 0, "size": r[4] or 0}
stats["effectiveness_meta"] = meta
stats["eval"] = None
stats["eval_error"] = ""
if include_eval:
try:
goldens = RetrievalEval.goldens_from_vault(store, profile)
result = RetrievalEval.score(store, goldens, profile)
pairs = RetrievalEval.implicit_pairs(store, profile)
stats["eval"] = {"goldens": result.get("goldens", 0), "k": result.get("k", EVAL_K),
"overall": result.get("overall") or {}, "by_category": result.get("by_category") or {},
"misses": (result.get("misses") or [])[:RAG_STATS_SAMPLE],
"searches": pairs.get("searches", 0), "read_follow": pairs.get("read_follow", 0),
"zero_hit_count": pairs.get("zero_hit_count", 0),
"zero_hit": (pairs.get("zero_hit") or [])[:RAG_STATS_SAMPLE]}
except Exception as exc:
stats["eval_error"] = short_error(exc)
return stats
@staticmethod
def _pct(part, whole):
if not whole:
return "n/a"
return "%.1f%%" % (100.0 * part / whole)
@staticmethod
def format_report(stats):
lines = ["rag stats: profile %s (fts %s, window newest %d of %d events)" % (
stats.get("profile"), "ok" if stats.get("fts_ok") else "OFF", stats.get("window_events", 0), stats.get("events_total", 0))]
lines.append("vault: %d records (%d bytes, %d reads col) | %d chunks (%d parents) | %d tag rows (%d tags on %d items) | %d edges | %d events" % (
stats.get("records_total", 0), stats.get("records_bytes", 0), stats.get("records_reads", 0),
stats.get("chunks_total", 0), stats.get("chunk_parents", 0), stats.get("tags_rows", 0),
stats.get("tags_distinct", 0), stats.get("tags_items", 0), stats.get("edges_total", 0), stats.get("events_total", 0)))
if stats.get("events_first") or stats.get("events_last"):
lines.append("events span: %s .. %s" % (stats.get("events_first") or "?", stats.get("events_last") or "?"))
for kind, count, size, reads in stats.get("records_by_kind") or []:
lines.append("records [%s]: n=%d bytes=%d reads=%d" % (kind, count, size, reads))
for record_id, kind, title, reads, size in stats.get("top_reads") or []:
lines.append("top-read: %d reads %s [%s] %s (%d bytes)" % (reads, record_id, kind, " ".join(title.split())[:60], size))
for record_id, kind, title, reads, size in stats.get("top_size") or []:
lines.append("top-size: %d bytes %s [%s] %s (%d reads)" % (size, record_id, kind, " ".join(title.split())[:60], reads))
lines.append("unread records: %d of %d (%s)" % (stats.get("zero_read_records", 0), stats.get("records_total", 0),
RagStats._pct(stats.get("zero_read_records", 0), stats.get("records_total", 0))))
for record_id in stats.get("never_read_sample") or []:
lines.append("never-read: %s" % record_id)
for parent, count in stats.get("chunks_by_parent") or []:
lines.append("chunks [%s]: n=%d" % (parent, count))
parents = stats.get("chunk_parents", 0)
lines.append("chunks per parent: %.1f avg, %d orphan record-chunks" % (
(stats.get("chunks_total", 0) / parents) if parents else 0.0, stats.get("orphan_chunks", 0)))
for parent_kind, parent_id, count in stats.get("top_chunk_parents") or []:
lines.append("chunked: %d chunks %s:%s" % (count, parent_kind, (parent_id or "")[:60]))
for tag, count in stats.get("top_tags") or []:
lines.append("tag: %d %s" % (count, tag))
records_total = stats.get("records_total", 0)
lines.append("tag coverage: %d of %d records tagged, %d with only the record tag, %.1f tags per record" % (
stats.get("tagged_records", 0), records_total, stats.get("only_record_tag", 0),
(stats.get("record_tag_rows", 0) / records_total) if records_total else 0.0))
for relation, count in stats.get("edges_by_relation") or []:
lines.append("edge [%s]: n=%d" % (relation, count))
for node, degree in stats.get("top_degree") or []:
lines.append("linked: degree %d %s" % (degree, (node or "")[:80]))
lines.append("isolated records: %d of %d (%s)" % (stats.get("isolated_records", 0), records_total,
RagStats._pct(stats.get("isolated_records", 0), records_total)))
for record_id in stats.get("isolated_sample") or []:
lines.append("isolated: %s" % record_id)
for kind, count in stats.get("fts_by_kind") or []:
lines.append("fts [%s]: n=%d" % (kind, count))
for role, kind, count in stats.get("events_by_kind") or []:
lines.append("events [%s/%s]: n=%d" % (role, kind, count))
lines.append("searches: total=%d window=%d zero-hit=%d (%s) avg-hits=%.2f avg-q=%.1f chars/%.1f terms 24h=%d 7d=%d" % (
stats.get("searches_total", 0), stats.get("window_searches", 0), stats.get("zero_hit_window", 0),
RagStats._pct(stats.get("zero_hit_window", 0), stats.get("window_searches", 0)), stats.get("avg_hits", 0.0),
stats.get("avg_query_chars", 0.0), stats.get("avg_query_terms", 0.0), stats.get("searches_24h", 0), stats.get("searches_7d", 0)))
dist = stats.get("hits_dist") or {}
lines.append("hits dist: 0=%d 1=%d 2-5=%d 6+=%d" % (dist.get("0", 0), dist.get("1", 0), dist.get("2-5", 0), dist.get("6+", 0)))
for kind, count in stats.get("hits_by_kind") or []:
lines.append("hits [%s]: n=%d" % (kind, count))
lines.append("queries: %d distinct in window, %d repeated" % (stats.get("queries_distinct", 0), stats.get("repeated_queries", 0)))
for query, count, avg in stats.get("top_queries") or []:
lines.append("query: %dx avg-hits=%.1f %s" % (count, avg, " ".join(query.split())[:80]))
for query in stats.get("zero_hit_sample") or []:
lines.append("zero-hit: %s" % " ".join(query.split())[:80])
lines.append("reads: total=%d window=%d distinct=%d 24h=%d 7d=%d" % (
stats.get("reads_total", 0), stats.get("window_reads", 0), stats.get("read_distinct", 0),
stats.get("reads_24h", 0), stats.get("reads_7d", 0)))
lines.append("read-follow: %d of %d window searches (%s)" % (stats.get("read_follow", 0), stats.get("window_searches", 0),
RagStats._pct(stats.get("read_follow", 0), stats.get("window_searches", 0))))
rank = stats.get("rank_dist") or {}
lines.append("read rank: 1=%d 2=%d 3=%d 4-5=%d 6+=%d unshown=%d" % (
rank.get("1", 0), rank.get("2", 0), rank.get("3", 0), rank.get("4-5", 0), rank.get("6+", 0), rank.get("unshown", 0)))
for record_id, count in stats.get("top_read_ids") or []:
lines.append("read-id: %dx %s" % (count, record_id))
for record_id, count in stats.get("top_shown") or []:
lines.append("shown: %dx %s" % (count, record_id))
for chunk_id, count in stats.get("top_shown_chunks") or []:
lines.append("shown-chunk: %dx %s" % (count, (chunk_id or "")[:80]))
lines.append("coverage: %d distinct shown, %d shown-never-read, %d direct-read-never-shown" % (
stats.get("shown_distinct", 0), stats.get("shown_never_read", 0), stats.get("direct_reads", 0)))
for record_id in stats.get("shown_never_sample") or []:
lines.append("shown-never-read: %s" % record_id)
for record_id in stats.get("direct_sample") or []:
lines.append("direct-read: %s" % record_id)
lines.append("context uses per record (shown + read events in window):")
meta = stats.get("effectiveness_meta") or {}
for record_id, shown, read, total in stats.get("effectiveness") or []:
info = meta.get(record_id) or {}
through = ("%s/%s" % (read, shown)) if shown else ("%d/0" % read)
lines.append("use: total=%d shown=%d read=%d through=%s col-reads=%d %s [%s] %s" % (
total, shown, read, through, info.get("reads_col", 0), record_id,
info.get("kind", "?"), " ".join(str(info.get("title", "")).split())[:60]))
eval_result = stats.get("eval")
if eval_result is not None:
overall = eval_result.get("overall") or {}
lines.append("eval: %d goldens at k=%d hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f" % (
eval_result.get("goldens", 0), eval_result.get("k", EVAL_K), overall.get("hit", 0.0),
overall.get("mrr", 0.0), overall.get("ndcg", 0.0), overall.get("cp", 0.0)))
for category in sorted((eval_result.get("by_category") or {}).keys()):
entry = eval_result["by_category"][category]
mean = entry.get("mean") or {}
lines.append("eval [%s]: n=%d hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f" % (
category, entry.get("goldens", 0), mean.get("hit", 0.0), mean.get("mrr", 0.0),
mean.get("ndcg", 0.0), mean.get("cp", 0.0)))
lines.append("eval implicit: %d searches, %d followed by a read, %d zero-hit" % (
eval_result.get("searches", 0), eval_result.get("read_follow", 0), eval_result.get("zero_hit_count", 0)))
elif stats.get("eval_error"):
lines.append("eval: skipped (%s)" % stats["eval_error"])
lines.append("note: hit lists truncate at 1500 chars in the log, shown counts are lower bounds")
return "\n".join(lines)
class Agent:
def __init__(self, config, store, persist=True, quiet=False, depth=0, auto=False, yolo=False, ctx=None):
self.config = config
@@ -21798,6 +22170,8 @@ def boot(args):
mode = "cap-speak"
elif getattr(args, "eval_retrieval", False):
mode = "eval"
elif getattr(args, "stats_rag", False):
mode = "stats-rag"
elif getattr(args, "update_models", False):
mode = "update-models"
elif getattr(args, "prompt", None):
@@ -39955,6 +40329,7 @@ def build_parser():
parser.add_argument("--no-opencode", action="store_true", help="skip the opencode CLI backends for this run (same as TAI_OPENCODE=0)")
parser.add_argument("--update-models", action="store_true", help="rediscover the free model lists of every provider, drop models that are gone, and exit")
parser.add_argument("--eval-retrieval", action="store_true", help="score vault retrieval (Hit@K, MRR, nDCG, Context Precision) on generated goldens plus implicit pairs, print the report, and exit")
parser.add_argument("--stats-rag", action="store_true", help="print RAG usage and effectiveness stats (vault totals, searches, reads, per-record context uses, retrieval eval) and exit")
parser.add_argument("--version", action="store_true")
parser.add_argument("--cap-manifest", action="store_true", help="print tai's CAP agent manifest (cap-agent.toml) and exit")
parser.add_argument("--cap-speak", action="store_true", help="speak CAP stream-json on stdio as an agent (bulk session config via CAP_SESSION_CONFIG)")
@@ -40166,6 +40541,12 @@ def main(argv=None):
finally:
agent.store.close()
return 0
if args.stats_rag:
try:
print(RagStats.format_report(RagStats.collect(agent.store, agent.profile)))
finally:
agent.store.close()
return 0
if args.scheduler:
print("scheduler online, press Ctrl-C to stop")
try:
taicpp/.clang-format
+3
View File
@@ -0,0 +1,3 @@
BasedOnStyle: LLVM
IndentWidth: 4
ColumnLimit: 100
taicpp/Makefile
+58
View File
@@ -0,0 +1,58 @@
CXX ?= g++
CXXSTD = -std=c++23
WARN = -Wall -Wextra
CXXFLAGS ?= $(CXXSTD) $(WARN) -O2
BUILD = build
VERSION = $(shell cat VERSION)
BIN = $(BUILD)/detz
GEN_INC = $(BUILD)/gen
VERSION_HPP = $(GEN_INC)/version.hpp
SRCS = main.cpp src/cli/cli.cpp
OBJS = $(BUILD)/main.o $(BUILD)/cli.o
all: $(BIN)
$(VERSION_HPP): VERSION
@mkdir -p $(GEN_INC)
@sed 's/@VERSION@/$(VERSION)/' src/cli/version.hpp.in > $@
$(BUILD)/main.o: main.cpp src/cli/cli.h $(VERSION_HPP)
@mkdir -p $(BUILD)
$(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c main.cpp -o $@
$(BUILD)/cli.o: src/cli/cli.cpp src/cli/cli.h $(VERSION_HPP)
@mkdir -p $(BUILD)
$(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c src/cli/cli.cpp -o $@
$(BIN): $(OBJS)
$(CXX) $(CXXFLAGS) $(OBJS) -o $@
test: $(BIN) $(BUILD)/detz_test
./$(BUILD)/detz_test
./$(BIN) --version
$(BUILD)/detz_test: tests/test_main.cpp src/cli/cli.cpp $(VERSION_HPP)
@mkdir -p $(BUILD)
$(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli tests/test_main.cpp src/cli/cli.cpp -o $@
lint:
$(CXX) $(CXXSTD) $(WARN) -Werror -fsyntax-only main.cpp -I$(GEN_INC) -Isrc/cli || true
version-bump:
tools/version-bump.sh $(PART)
changelog:
tools/changelog.sh "$(MSG)"
tag:
tools/tag.sh
release:
PART=$(PART) MSG="$(MSG)" tools/release.sh
clean:
rm -rf $(BUILD)
-include src/*/*.mk
taicpp/VERSION
+1
View File
@@ -0,0 +1 @@
0.1.0
taicpp/build/cli.o
BIN
View File
Binary file not shown.
taicpp/build/detz
Executable
BIN
View File
Binary file not shown.
taicpp/build/detz_test
Executable
BIN
View File
Binary file not shown.
taicpp/build/gen/version.hpp
+3
View File
@@ -0,0 +1,3 @@
#pragma once
#include <string_view>
namespace detz { constexpr std::string_view kVersion = "0.1.0"; }
taicpp/build/main.o
BIN
View File
Binary file not shown.
taicpp/main.cpp
+2
View File
@@ -0,0 +1,2 @@
#include "src/cli/cli.h"
int main(int argc, char** argv) { return detz::cli::DetzCli{}.run(argc, argv); }
taicpp/src/agent/.gitkeep
View File
taicpp/src/cap/.gitkeep
View File
taicpp/src/cli/.gitkeep
View File
taicpp/src/cli/cli.cpp
+17
View File
@@ -0,0 +1,17 @@
#include "cli.h"
#include "version.hpp"
#include <cstdio>
namespace detz::cli {
Command Parser::parse(int argc, char** argv) {
Command c;
if (argc > 1) { c.name = argv[1]; for (int i = 2; i < argc; ++i) c.args.emplace_back(argv[i]); }
return c;
}
int Command::run() {
if (name == "--version" || name == "version") { std::puts(kVersion.data()); return 0; }
std::puts("detz — C++23 port of tai.py. See --help. Version:");
std::puts(kVersion.data());
return 0;
}
int DetzCli::run(int argc, char** argv) { return Parser{}.parse(argc, argv).run(); }
} // namespace detz::cli
taicpp/src/cli/cli.h
+10
View File
@@ -0,0 +1,10 @@
#pragma once
#include <string>
#include <vector>
namespace detz::cli {
struct Command { std::string name{"--help"}; std::vector<std::string> args; int run(); };
class Parser { public: Command parse(int argc, char** argv); };
class DetzCli { public: int run(int argc, char** argv); };
// molo-subcommand adapter stub (M1 wires `molo detz` registration here).
class MoloSubcommand { public: const char* name() const { return "detz"; } };
} // namespace detz::cli
taicpp/src/cli/version.hpp.in
+3
View File
@@ -0,0 +1,3 @@
#pragma once
#include <string_view>
namespace detz { constexpr std::string_view kVersion = "@VERSION@"; }
taicpp/src/comms/.gitkeep
View File
taicpp/src/config/.gitkeep
View File
taicpp/src/llm/.gitkeep
View File
taicpp/src/mcp/.gitkeep
View File
taicpp/src/plexer/.gitkeep
View File
taicpp/src/seal/.gitkeep
View File
taicpp/src/shell/.gitkeep
View File
taicpp/src/store/.gitkeep
View File
taicpp/src/tui/.gitkeep
View File
taicpp/src/util/.gitkeep
View File
taicpp/tests/test_main.cpp
+12
View File
@@ -0,0 +1,12 @@
#include "../src/cli/cli.h"
#include <cassert>
#include <cstdio>
int main() {
detz::cli::Parser p;
const char* a0 = "detz";
auto c = p.parse(1, (char**)&a0);
assert(c.name == "--help");
assert(c.run() == 0);
std::puts("M0 OK");
return 0;
}
taicpp/tools/changelog.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
msg=${1:-update}
v=$(cat VERSION); d=$(date +%F)
{ echo "# Changelog"; echo; echo "## [$v] - $d"; echo "- $msg"; echo; tail -n +2 CHANGELOG.md; } > CHANGELOG.md.new
mv CHANGELOG.md.new CHANGELOG.md
echo "changelog updated"
taicpp/tools/release.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
: "${PART:=patch}"
tools/version-bump.sh "$PART"
tools/changelog.sh "$MSG"
git add VERSION CHANGELOG.md 2>/dev/null || true
git commit -m "detz: release v$(cat VERSION) $MSG" 2>/dev/null || true
tools/tag.sh
taicpp/tools/tag.sh
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -e
v=$(cat VERSION)
git tag -a "detz-v$v" -m "detz v$v" 2>/dev/null || echo "tag detz-v$v (no git or exists)"
taicpp/tools/version-bump.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
part=${1:-patch}
v=$(cat VERSION)
maj=${v%%.*}; rest=${v#*.}; min=${rest%%.*}; pat=${rest##*.}
case $part in major) maj=$((maj+1)); min=0; pat=0;; minor) min=$((min+1)); pat=0;; *) pat=$((pat+1));; esac
echo "$maj.$min.$pat" > VERSION
echo "VERSION -> $(cat VERSION)"
test_tai.py
+79
View File
@@ -2239,6 +2239,14 @@ class BundledSkillsTests(unittest.TestCase):
self.assertNotIn("`/plan`", text, label)
self.assertNotIn("/skill plan", text, label)
def test_deep_research_trigger_prefers_use(self):
text = tai.BUNDLED_SKILLS["deep-research"]["SKILL.md"]
skill = tai.parse_skill_front(text.split("\n---", 1)[0].splitlines()[1:])
self.assertNotIn("\n", skill["description"])
catalog = tai.skill_catalog({"deep-research": {"description": skill["description"]}})
self.assertIn("prefer it over a single lookup", catalog)
self.assertIn("resolving an issue", catalog)
def test_skills_doc_names_everything(self):
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "skills.md"), encoding="utf-8") as handle:
doc = handle.read()
@@ -4801,6 +4809,77 @@ class RetrievalEvalTests(unittest.TestCase):
self.assertEqual(pairs["zero_hit"], ["zzz-nope-nothing"])
class RagStatsTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
os.environ["TAI_HOME"] = self.tmp.name
class FakeArgs:
profile = "t"
yes = True
self.config = tai.Config(FakeArgs())
self.store = tai.Store(self.config, tai.Seal(self.config.home, "rag-stats-1"))
self.app = FakeSecretApp(self.store)
self.app.profile = "t"
self.app.config = self.config
self.tools = tai.Tools(self.app)
def tearDown(self):
self.store.close()
self.tmp.cleanup()
os.environ.pop("TAI_HOME", None)
def seed(self):
ids = {}
ids["title"] = self.store.add_record("note", "quasar beacon", "short body here", [], "t")
ids["stuffed"] = self.store.add_record("note", "unrelated title", "background words filler text. " * 10 + "quasar beacon " * 10, [], "t")
ids["harbor"] = self.store.add_record("note", "harbor launch checklist", "step one step two step three", [], "t")
return ids
def test_collect_empty(self):
stats = tai.RagStats.collect(self.store, "t", include_eval=False)
self.assertEqual(stats["records_total"], 0)
self.assertEqual(stats["searches_total"], 0)
self.assertEqual(stats["reads_total"], 0)
self.assertIn("rag stats: profile t", tai.RagStats.format_report(stats))
def test_collect_counts_search_and_read(self):
ids = self.seed()
self.tools.dispatch("search", json.dumps({"query": "quasar beacon", "kinds": ["record"]}))
self.tools.dispatch("record_read", json.dumps({"id": ids["title"]}))
self.tools.dispatch("search", json.dumps({"query": "zzz-nope-nothing"}))
stats = tai.RagStats.collect(self.store, "t", include_eval=False)
self.assertEqual(stats["records_total"], 3)
self.assertEqual(stats["searches_total"], 2)
self.assertEqual(stats["reads_total"], 1)
self.assertEqual(stats["window_searches"], 2)
self.assertEqual(stats["zero_hit_window"], 1)
self.assertEqual(stats["read_follow"], 1)
self.assertEqual(stats["rank_dist"]["1"], 1)
shown = dict(stats["top_shown"])
self.assertGreaterEqual(shown.get(ids["title"], 0), 1)
by_id = {row[0]: row for row in stats["effectiveness"]}
self.assertIn(ids["title"], by_id)
_rid, shown_n, read_n, total = by_id[ids["title"]]
self.assertGreaterEqual(shown_n, 1)
self.assertEqual(read_n, 1)
self.assertGreaterEqual(total, 2)
def test_format_report_covers_sections(self):
ids = self.seed()
self.tools.dispatch("search", json.dumps({"query": "quasar beacon", "kinds": ["record"]}))
self.tools.dispatch("record_read", json.dumps({"id": ids["title"]}))
report = tai.RagStats.format_report(tai.RagStats.collect(self.store, "t"))
for section in ("context uses per record", "read-follow", "hits dist", "tag coverage",
"read rank", "queries:", "eval:", ids["title"]):
self.assertIn(section, report)
def test_parser_flag(self):
self.assertTrue(tai.build_parser().parse_args(["--stats-rag"]).stats_rag)
self.assertFalse(tai.build_parser().parse_args([]).stats_rag)
class SupervisorCapTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()