diff --git a/tai.py b/tai.py index cd3c00e..5cec37e 100755 --- a/tai.py +++ b/tai.py @@ -313,7 +313,7 @@ SCHEDULER_INTERVAL = 30 HTTP_TIMEOUT = 60 STREAM_TIMEOUT = 300 APPROVAL_MAX_RETRIES = 3 -VERIFY_MAX_ROUNDS = 2 +VERIFY_MAX_ROUNDS = 9 VERIFY_TIMEOUT = 120 VERIFY_STEPS = 6 VERIFY_MAX_CHARS = 6000 @@ -334,6 +334,11 @@ EVAL_MAX_GOLDENS = 200 EVAL_MAX_PAIRS = 200 EVAL_MIN_TOKEN = 6 EVAL_PREFIX_MIN = 4 +RAG_STATS_EVENTS = 5000 +RAG_STATS_TOP = 20 +RAG_STATS_QUERIES = 20 +RAG_STATS_SAMPLE = 10 +RAG_STATS_KINDS = 30 EDGE_HOST = "speech.platform.bing.com" EDGE_WS_PATH = "/consumer/speech/synthesize/readaloud/edge/v1" @@ -2599,7 +2604,7 @@ BUNDLED_SKILLS = { "deep-research": { "SKILL.md": r"""--- name: deep-research -description: "End-to-end deep research that turns a question into a cited, verifiable report. Use whenever the answer needs many external sources rather than a single lookup: technology/library selection, code and architecture questions, academic or medical literature, market and company facts, policy and standards, and 'what's the latest' or capability/benchmark questions. Routes each sub-question to the most authoritative sources for its domain (GitHub and official docs for code, arXiv/OpenAlex/Semantic Scholar for scholarship, PubMed/Europe PMC for medicine, SEC EDGAR for company facts, vendor and official pages for products), gathers evidence in parallel, triangulates claims, and writes a structured report with dated citations. Trigger on 调研/研究/调查/综述/对比一下 X、帮我查清楚 X、X 的最新进展/评测数据/方案有哪些, and any request for a research report, literature review, landscape survey, or source-backed answer - even when the user does not say the word 'research'." +description: "End-to-end deep research that turns a question into a cited, verifiable report. Reach for it generously whenever external facts could help resolve the request: prefer it over a single lookup for technology/library selection, code and architecture questions, debugging or resolving an issue, academic or medical literature, market and company facts, policy and standards, and 'what's the latest' or capability/benchmark questions. Routes each sub-question to the most authoritative sources for its domain (GitHub and official docs for code, arXiv/OpenAlex/Semantic Scholar for scholarship, PubMed/Europe PMC for medicine, SEC EDGAR for company facts, vendor and official pages for products), gathers evidence in parallel, triangulates claims, and writes a structured report with dated citations. Trigger on 调研/研究/调查/综述/对比一下 X、帮我查清楚 X、X 的最新进展/评测数据/方案有哪些, troubleshooting and debugging questions, and any request for a research report, literature review, landscape survey, or source-backed answer - even when the user does not say the word 'research'." --- # Deep Research @@ -2610,9 +2615,10 @@ This skill is **self-contained and host-agnostic**. It defines the *method*; it ## When to use -Use it when the answer must be assembled from **multiple external sources** and needs to be current, comparative, or authoritative: +Use it generously whenever external facts could help resolve the request, especially when the answer needs to be current, comparative, or authoritative: - technical / library / architecture selection, migration paths, "X vs Y" +- debugging or resolving an issue: errors, unexpected behavior, root-cause analysis - literature reviews, "what does the research say about X" - medical, clinical, or life-science questions - market, company, competitive, or due-diligence facts @@ -19535,6 +19541,372 @@ class RetrievalEval: return "\n".join(lines) +class RagStats: + """RAG usage and effectiveness stats (spec). One read-only rollup over + the vault tables plus the event log, printed by --stats-rag. + + Definitions: shown means a record id appeared in a search-query event + hit list; read means a record-read event named the id; context uses is + shown plus read (both inject vault text into the LLM context, snippets + on show, full pages on read); read-through is reads divided by shows. + The events window holds the newest RAG_STATS_EVENTS rows; totals via + COUNT(*) are exact, window-derived shares are labeled as window shares. + Caps: RAG_STATS_* constants bound rows scanned, top lists, and samples.""" + + @staticmethod + def _one(db, sql, params=()): + try: + row = db.execute(sql, tuple(params)).fetchone() + except sqlite3.Error: + return None + return row[0] if row else None + + @staticmethod + def _rows(db, sql, params=()): + try: + return db.execute(sql, tuple(params)).fetchall() + except sqlite3.Error: + return [] + + @staticmethod + def _parse_hits(raw): + kinds = {} + ranked = [] + shown_records = [] + shown_chunks = [] + for piece in (raw or "").split(","): + piece = piece.strip() + if not piece: + continue + kind, sep, rest = piece.partition(":") + if not sep or kind not in ("record", "event", "audit", "chunk", "log"): + kinds["other"] = kinds.get("other", 0) + 1 + ranked.append(piece) + continue + kinds[kind] = kinds.get(kind, 0) + 1 + if kind == "record": + ranked.append(rest) + shown_records.append(rest) + elif kind == "chunk": + ranked.append(piece) + shown_chunks.append(rest or piece) + else: + ranked.append(piece) + return kinds, ranked, shown_records, shown_chunks + + @staticmethod + def _parse_search_event(text): + try: + parts = dict(part.split("=", 1) for part in str(text or "").split("\t") if "=" in part) + query = (parts.get("q") or "").strip() + if not query: + return None + kinds, ranked, shown_records, shown_chunks = RagStats._parse_hits(parts.get("hits") or "") + try: + limit = int(parts.get("limit") or 0) + except (TypeError, ValueError): + limit = 0 + return {"query": query, "limit": limit, "hits": len(ranked), "kinds": kinds, + "ranked": ranked, "shown_records": shown_records, "shown_chunks": shown_chunks} + except Exception: + return None + + @staticmethod + def collect(store, profile, include_eval=True): + db = store.db + stats = {"profile": profile, "fts_ok": bool(getattr(store, "fts_ok", False)), + "window": RAG_STATS_EVENTS} + stats["events_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ?", (profile,)) or 0 + row = RagStats._rows(db, "SELECT COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ?", (profile,)) + stats["records_total"], stats["records_bytes"], stats["records_reads"] = (row[0][0] or 0, row[0][1] or 0, row[0][2] or 0) if row else (0, 0, 0) + stats["records_by_kind"] = [(r[0], r[1], r[2] or 0, r[3] or 0) for r in RagStats._rows( + db, "SELECT kind, COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", (profile,))] + stats["top_reads"] = [(r[0], r[1], r[2] or "", r[3] or 0, r[4] or 0) for r in RagStats._rows( + db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY reads DESC, updated DESC LIMIT ?", (profile, RAG_STATS_TOP))] + stats["top_size"] = [(r[0], r[1], r[2] or "", r[3] or 0, r[4] or 0) for r in RagStats._rows( + db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY size DESC LIMIT ?", (profile, RAG_STATS_TOP))] + stats["zero_read_records"] = RagStats._one(db, "SELECT COUNT(*) FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0", (profile,)) or 0 + stats["never_read_sample"] = [r[0] for r in RagStats._rows( + db, "SELECT id FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0 ORDER BY updated DESC LIMIT ?", (profile, RAG_STATS_SAMPLE))] + stats["chunks_total"] = RagStats._one(db, "SELECT COUNT(*) FROM chunks WHERE profile = ?", (profile,)) or 0 + stats["chunks_by_parent"] = [(r[0], r[1]) for r in RagStats._rows( + db, "SELECT parent_kind, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind ORDER BY COUNT(*) DESC", (profile,))] + stats["chunk_parents"] = RagStats._one(db, "SELECT COUNT(DISTINCT parent_kind || ':' || parent_id) FROM chunks WHERE profile = ?", (profile,)) or 0 + stats["top_chunk_parents"] = [(r[0], r[1], r[2]) for r in RagStats._rows( + db, "SELECT parent_kind, parent_id, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind, parent_id ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_TOP))] + stats["orphan_chunks"] = RagStats._one(db, "SELECT COUNT(*) FROM chunks WHERE profile = ? AND parent_kind = 'record' AND parent_id NOT IN (SELECT id FROM records WHERE profile = ?)", (profile, profile)) or 0 + tag_row = RagStats._rows(db, "SELECT COUNT(*), COUNT(DISTINCT tag), COUNT(DISTINCT item) FROM tags WHERE profile = ?", (profile,)) + stats["tags_rows"], stats["tags_distinct"], stats["tags_items"] = (tag_row[0][0] or 0, tag_row[0][1] or 0, tag_row[0][2] or 0) if tag_row else (0, 0, 0) + try: + stats["top_tags"] = list(store.tag_counts("", RAG_STATS_TOP, profile)) + except Exception: + stats["top_tags"] = [] + stats["record_tag_rows"] = RagStats._one(db, "SELECT COUNT(*) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", (profile,)) or 0 + stats["tagged_records"] = RagStats._one(db, "SELECT COUNT(DISTINCT item) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", (profile,)) or 0 + stats["only_record_tag"] = RagStats._one(db, "SELECT COUNT(*) FROM (SELECT item FROM tags WHERE profile = ? AND item LIKE 'mem:%' GROUP BY item HAVING COUNT(*) = 1 AND MAX(tag) = 'record')", (profile,)) or 0 + stats["edges_total"] = RagStats._one(db, "SELECT COUNT(*) FROM edges WHERE profile = ?", (profile,)) or 0 + stats["edges_by_relation"] = [(r[0], r[1]) for r in RagStats._rows( + db, "SELECT relation, COUNT(*) FROM edges WHERE profile = ? GROUP BY relation ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_TOP))] + stats["top_degree"] = [(r[0], r[1]) for r in RagStats._rows( + db, "SELECT node, COUNT(*) FROM (SELECT src AS node FROM edges WHERE profile = ? UNION ALL SELECT dst AS node FROM edges WHERE profile = ?) GROUP BY node ORDER BY COUNT(*) DESC LIMIT ?", (profile, profile, RAG_STATS_TOP))] + stats["isolated_records"] = RagStats._one(db, "SELECT COUNT(*) FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?)", (profile, profile, profile)) or 0 + stats["isolated_sample"] = [r[0] for r in RagStats._rows( + db, "SELECT id FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?) ORDER BY updated DESC LIMIT ?", (profile, profile, profile, RAG_STATS_SAMPLE))] + stats["fts_by_kind"] = [] + if stats["fts_ok"]: + stats["fts_by_kind"] = [(r[0], r[1]) for r in RagStats._rows(db, "SELECT kind, COUNT(*) FROM fts_docs WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", (profile,))] + stats["events_by_kind"] = [(r[0], r[1], r[2]) for r in RagStats._rows( + db, "SELECT role, kind, COUNT(*) FROM events WHERE profile = ? GROUP BY role, kind ORDER BY COUNT(*) DESC LIMIT ?", (profile, RAG_STATS_KINDS))] + span = RagStats._rows(db, "SELECT MIN(ts), MAX(ts) FROM events WHERE profile = ?", (profile,)) + stats["events_first"], stats["events_last"] = (span[0][0], span[0][1]) if span else (None, None) + now = datetime.now(timezone.utc) + day_cut = (now - timedelta(days=1)).isoformat(timespec="seconds") + week_cut = (now - timedelta(days=7)).isoformat(timespec="seconds") + stats["searches_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query'", (profile,)) or 0 + stats["reads_total"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read'", (profile,)) or 0 + stats["searches_24h"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", (profile, day_cut)) or 0 + stats["reads_24h"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", (profile, day_cut)) or 0 + stats["searches_7d"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", (profile, week_cut)) or 0 + stats["reads_7d"] = RagStats._one(db, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", (profile, week_cut)) or 0 + text_col = "tai_dec(text)" if store.seal.enabled else "text" + window_rows = RagStats._rows(db, "SELECT ts, role, kind, %s FROM events WHERE profile = ? ORDER BY id DESC LIMIT ?" % text_col, (profile, RAG_STATS_EVENTS)) + window_rows.reverse() + searches = [] + reads = [] + for pos, row in enumerate(window_rows): + if row[1] == "tool" and row[2] == "search-query": + parsed = RagStats._parse_search_event(row[3] or "") + if parsed is not None: + parsed["pos"] = pos + parsed["ts"] = row[0] + searches.append(parsed) + elif row[1] == "tool" and row[2] == "record-read" and (row[3] or "").strip(): + reads.append({"pos": pos, "id": (row[3] or "").strip()[:200], "ts": row[0]}) + stats["window_events"] = len(window_rows) + stats["window_searches"] = len(searches) + stats["window_reads"] = len(reads) + hits_by_kind = {} + shown_counter = collections.Counter() + chunk_counter = collections.Counter() + query_counter = collections.Counter() + query_hits = {} + query_len = 0 + query_terms = 0 + zero_hit = [] + dist = {"0": 0, "1": 0, "2-5": 0, "6+": 0} + for item in searches: + query_counter[item["query"]] += 1 + query_hits.setdefault(item["query"], []).append(item["hits"]) + query_len += len(item["query"]) + query_terms += len(item["query"].split()) + for kind, count in item["kinds"].items(): + hits_by_kind[kind] = hits_by_kind.get(kind, 0) + count + for record_id in item["shown_records"]: + shown_counter[record_id] += 1 + for chunk_id in item["shown_chunks"]: + chunk_counter[chunk_id] += 1 + if item["hits"] == 0: + dist["0"] += 1 + zero_hit.append(item["query"]) + elif item["hits"] == 1: + dist["1"] += 1 + elif item["hits"] <= 5: + dist["2-5"] += 1 + else: + dist["6+"] += 1 + stats["hits_by_kind"] = sorted(hits_by_kind.items(), key=lambda pair: (-pair[1], pair[0])) + stats["hits_dist"] = dist + stats["zero_hit_window"] = dist["0"] + stats["zero_hit_sample"] = zero_hit[:RAG_STATS_SAMPLE] + total_hits = sum(item["hits"] for item in searches) + stats["avg_hits"] = round(total_hits / len(searches), 2) if searches else 0.0 + stats["avg_query_chars"] = round(query_len / len(searches), 1) if searches else 0.0 + stats["avg_query_terms"] = round(query_terms / len(searches), 1) if searches else 0.0 + stats["repeated_queries"] = sum(1 for count in query_counter.values() if count > 1) + stats["queries_distinct"] = len(query_counter) + top_queries = [] + for query, count in query_counter.most_common(RAG_STATS_QUERIES): + shots = query_hits.get(query) or [0] + top_queries.append((query, count, round(sum(shots) / len(shots), 1))) + stats["top_queries"] = top_queries + stats["top_shown"] = shown_counter.most_common(RAG_STATS_TOP) + stats["top_shown_chunks"] = chunk_counter.most_common(RAG_STATS_TOP) + stats["shown_distinct"] = len(shown_counter) + read_counter = collections.Counter(item["id"] for item in reads) + stats["top_read_ids"] = read_counter.most_common(RAG_STATS_TOP) + stats["read_distinct"] = len(read_counter) + rank_dist = {"1": 0, "2": 0, "3": 0, "4-5": 0, "6+": 0, "unshown": 0} + read_follow = 0 + for idx, item in enumerate(searches): + end = searches[idx + 1]["pos"] if idx + 1 < len(searches) else len(window_rows) + for row in window_rows[item["pos"] + 1:min(end, item["pos"] + 21)]: + if row[1] == "tool" and row[2] == "record-read" and (row[3] or "").strip(): + read_follow += 1 + rank = RetrievalEval.rank_of((row[3] or "").strip()[:200], item["ranked"]) + if rank is None: + rank_dist["unshown"] += 1 + elif rank == 1: + rank_dist["1"] += 1 + elif rank == 2: + rank_dist["2"] += 1 + elif rank == 3: + rank_dist["3"] += 1 + elif rank <= 5: + rank_dist["4-5"] += 1 + else: + rank_dist["6+"] += 1 + break + stats["read_follow"] = read_follow + stats["rank_dist"] = rank_dist + shown_set = set(shown_counter) + read_set = set(read_counter) + stats["direct_reads"] = len(read_set - shown_set) + stats["direct_sample"] = sorted(read_set - shown_set)[:RAG_STATS_SAMPLE] + stats["shown_never_read"] = len(shown_set - read_set) + stats["shown_never_sample"] = sorted(shown_set - read_set)[:RAG_STATS_SAMPLE] + effectiveness = [] + for record_id in set(list(shown_set) + list(read_set)): + shown = shown_counter.get(record_id, 0) + read = read_counter.get(record_id, 0) + effectiveness.append((record_id, shown, read, shown + read)) + effectiveness.sort(key=lambda row: (-row[3], -row[2], row[0])) + stats["effectiveness"] = effectiveness[:RAG_STATS_TOP] + meta = {} + ids = [row[0] for row in stats["effectiveness"]] + if ids: + marks = ", ".join("?" * len(ids)) + for r in RagStats._rows(db, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? AND id IN (%s)" % marks, tuple([profile] + ids)): + meta[r[0]] = {"kind": r[1], "title": r[2] or "", "reads_col": r[3] or 0, "size": r[4] or 0} + stats["effectiveness_meta"] = meta + stats["eval"] = None + stats["eval_error"] = "" + if include_eval: + try: + goldens = RetrievalEval.goldens_from_vault(store, profile) + result = RetrievalEval.score(store, goldens, profile) + pairs = RetrievalEval.implicit_pairs(store, profile) + stats["eval"] = {"goldens": result.get("goldens", 0), "k": result.get("k", EVAL_K), + "overall": result.get("overall") or {}, "by_category": result.get("by_category") or {}, + "misses": (result.get("misses") or [])[:RAG_STATS_SAMPLE], + "searches": pairs.get("searches", 0), "read_follow": pairs.get("read_follow", 0), + "zero_hit_count": pairs.get("zero_hit_count", 0), + "zero_hit": (pairs.get("zero_hit") or [])[:RAG_STATS_SAMPLE]} + except Exception as exc: + stats["eval_error"] = short_error(exc) + return stats + + @staticmethod + def _pct(part, whole): + if not whole: + return "n/a" + return "%.1f%%" % (100.0 * part / whole) + + @staticmethod + def format_report(stats): + lines = ["rag stats: profile %s (fts %s, window newest %d of %d events)" % ( + stats.get("profile"), "ok" if stats.get("fts_ok") else "OFF", stats.get("window_events", 0), stats.get("events_total", 0))] + lines.append("vault: %d records (%d bytes, %d reads col) | %d chunks (%d parents) | %d tag rows (%d tags on %d items) | %d edges | %d events" % ( + stats.get("records_total", 0), stats.get("records_bytes", 0), stats.get("records_reads", 0), + stats.get("chunks_total", 0), stats.get("chunk_parents", 0), stats.get("tags_rows", 0), + stats.get("tags_distinct", 0), stats.get("tags_items", 0), stats.get("edges_total", 0), stats.get("events_total", 0))) + if stats.get("events_first") or stats.get("events_last"): + lines.append("events span: %s .. %s" % (stats.get("events_first") or "?", stats.get("events_last") or "?")) + for kind, count, size, reads in stats.get("records_by_kind") or []: + lines.append("records [%s]: n=%d bytes=%d reads=%d" % (kind, count, size, reads)) + for record_id, kind, title, reads, size in stats.get("top_reads") or []: + lines.append("top-read: %d reads %s [%s] %s (%d bytes)" % (reads, record_id, kind, " ".join(title.split())[:60], size)) + for record_id, kind, title, reads, size in stats.get("top_size") or []: + lines.append("top-size: %d bytes %s [%s] %s (%d reads)" % (size, record_id, kind, " ".join(title.split())[:60], reads)) + lines.append("unread records: %d of %d (%s)" % (stats.get("zero_read_records", 0), stats.get("records_total", 0), + RagStats._pct(stats.get("zero_read_records", 0), stats.get("records_total", 0)))) + for record_id in stats.get("never_read_sample") or []: + lines.append("never-read: %s" % record_id) + for parent, count in stats.get("chunks_by_parent") or []: + lines.append("chunks [%s]: n=%d" % (parent, count)) + parents = stats.get("chunk_parents", 0) + lines.append("chunks per parent: %.1f avg, %d orphan record-chunks" % ( + (stats.get("chunks_total", 0) / parents) if parents else 0.0, stats.get("orphan_chunks", 0))) + for parent_kind, parent_id, count in stats.get("top_chunk_parents") or []: + lines.append("chunked: %d chunks %s:%s" % (count, parent_kind, (parent_id or "")[:60])) + for tag, count in stats.get("top_tags") or []: + lines.append("tag: %d %s" % (count, tag)) + records_total = stats.get("records_total", 0) + lines.append("tag coverage: %d of %d records tagged, %d with only the record tag, %.1f tags per record" % ( + stats.get("tagged_records", 0), records_total, stats.get("only_record_tag", 0), + (stats.get("record_tag_rows", 0) / records_total) if records_total else 0.0)) + for relation, count in stats.get("edges_by_relation") or []: + lines.append("edge [%s]: n=%d" % (relation, count)) + for node, degree in stats.get("top_degree") or []: + lines.append("linked: degree %d %s" % (degree, (node or "")[:80])) + lines.append("isolated records: %d of %d (%s)" % (stats.get("isolated_records", 0), records_total, + RagStats._pct(stats.get("isolated_records", 0), records_total))) + for record_id in stats.get("isolated_sample") or []: + lines.append("isolated: %s" % record_id) + for kind, count in stats.get("fts_by_kind") or []: + lines.append("fts [%s]: n=%d" % (kind, count)) + for role, kind, count in stats.get("events_by_kind") or []: + lines.append("events [%s/%s]: n=%d" % (role, kind, count)) + lines.append("searches: total=%d window=%d zero-hit=%d (%s) avg-hits=%.2f avg-q=%.1f chars/%.1f terms 24h=%d 7d=%d" % ( + stats.get("searches_total", 0), stats.get("window_searches", 0), stats.get("zero_hit_window", 0), + RagStats._pct(stats.get("zero_hit_window", 0), stats.get("window_searches", 0)), stats.get("avg_hits", 0.0), + stats.get("avg_query_chars", 0.0), stats.get("avg_query_terms", 0.0), stats.get("searches_24h", 0), stats.get("searches_7d", 0))) + dist = stats.get("hits_dist") or {} + lines.append("hits dist: 0=%d 1=%d 2-5=%d 6+=%d" % (dist.get("0", 0), dist.get("1", 0), dist.get("2-5", 0), dist.get("6+", 0))) + for kind, count in stats.get("hits_by_kind") or []: + lines.append("hits [%s]: n=%d" % (kind, count)) + lines.append("queries: %d distinct in window, %d repeated" % (stats.get("queries_distinct", 0), stats.get("repeated_queries", 0))) + for query, count, avg in stats.get("top_queries") or []: + lines.append("query: %dx avg-hits=%.1f %s" % (count, avg, " ".join(query.split())[:80])) + for query in stats.get("zero_hit_sample") or []: + lines.append("zero-hit: %s" % " ".join(query.split())[:80]) + lines.append("reads: total=%d window=%d distinct=%d 24h=%d 7d=%d" % ( + stats.get("reads_total", 0), stats.get("window_reads", 0), stats.get("read_distinct", 0), + stats.get("reads_24h", 0), stats.get("reads_7d", 0))) + lines.append("read-follow: %d of %d window searches (%s)" % (stats.get("read_follow", 0), stats.get("window_searches", 0), + RagStats._pct(stats.get("read_follow", 0), stats.get("window_searches", 0)))) + rank = stats.get("rank_dist") or {} + lines.append("read rank: 1=%d 2=%d 3=%d 4-5=%d 6+=%d unshown=%d" % ( + rank.get("1", 0), rank.get("2", 0), rank.get("3", 0), rank.get("4-5", 0), rank.get("6+", 0), rank.get("unshown", 0))) + for record_id, count in stats.get("top_read_ids") or []: + lines.append("read-id: %dx %s" % (count, record_id)) + for record_id, count in stats.get("top_shown") or []: + lines.append("shown: %dx %s" % (count, record_id)) + for chunk_id, count in stats.get("top_shown_chunks") or []: + lines.append("shown-chunk: %dx %s" % (count, (chunk_id or "")[:80])) + lines.append("coverage: %d distinct shown, %d shown-never-read, %d direct-read-never-shown" % ( + stats.get("shown_distinct", 0), stats.get("shown_never_read", 0), stats.get("direct_reads", 0))) + for record_id in stats.get("shown_never_sample") or []: + lines.append("shown-never-read: %s" % record_id) + for record_id in stats.get("direct_sample") or []: + lines.append("direct-read: %s" % record_id) + lines.append("context uses per record (shown + read events in window):") + meta = stats.get("effectiveness_meta") or {} + for record_id, shown, read, total in stats.get("effectiveness") or []: + info = meta.get(record_id) or {} + through = ("%s/%s" % (read, shown)) if shown else ("%d/0" % read) + lines.append("use: total=%d shown=%d read=%d through=%s col-reads=%d %s [%s] %s" % ( + total, shown, read, through, info.get("reads_col", 0), record_id, + info.get("kind", "?"), " ".join(str(info.get("title", "")).split())[:60])) + eval_result = stats.get("eval") + if eval_result is not None: + overall = eval_result.get("overall") or {} + lines.append("eval: %d goldens at k=%d hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f" % ( + eval_result.get("goldens", 0), eval_result.get("k", EVAL_K), overall.get("hit", 0.0), + overall.get("mrr", 0.0), overall.get("ndcg", 0.0), overall.get("cp", 0.0))) + for category in sorted((eval_result.get("by_category") or {}).keys()): + entry = eval_result["by_category"][category] + mean = entry.get("mean") or {} + lines.append("eval [%s]: n=%d hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f" % ( + category, entry.get("goldens", 0), mean.get("hit", 0.0), mean.get("mrr", 0.0), + mean.get("ndcg", 0.0), mean.get("cp", 0.0))) + lines.append("eval implicit: %d searches, %d followed by a read, %d zero-hit" % ( + eval_result.get("searches", 0), eval_result.get("read_follow", 0), eval_result.get("zero_hit_count", 0))) + elif stats.get("eval_error"): + lines.append("eval: skipped (%s)" % stats["eval_error"]) + lines.append("note: hit lists truncate at 1500 chars in the log, shown counts are lower bounds") + return "\n".join(lines) + + class Agent: def __init__(self, config, store, persist=True, quiet=False, depth=0, auto=False, yolo=False, ctx=None): self.config = config @@ -21798,6 +22170,8 @@ def boot(args): mode = "cap-speak" elif getattr(args, "eval_retrieval", False): mode = "eval" + elif getattr(args, "stats_rag", False): + mode = "stats-rag" elif getattr(args, "update_models", False): mode = "update-models" elif getattr(args, "prompt", None): @@ -39955,6 +40329,7 @@ def build_parser(): parser.add_argument("--no-opencode", action="store_true", help="skip the opencode CLI backends for this run (same as TAI_OPENCODE=0)") parser.add_argument("--update-models", action="store_true", help="rediscover the free model lists of every provider, drop models that are gone, and exit") parser.add_argument("--eval-retrieval", action="store_true", help="score vault retrieval (Hit@K, MRR, nDCG, Context Precision) on generated goldens plus implicit pairs, print the report, and exit") + parser.add_argument("--stats-rag", action="store_true", help="print RAG usage and effectiveness stats (vault totals, searches, reads, per-record context uses, retrieval eval) and exit") parser.add_argument("--version", action="store_true") parser.add_argument("--cap-manifest", action="store_true", help="print tai's CAP agent manifest (cap-agent.toml) and exit") parser.add_argument("--cap-speak", action="store_true", help="speak CAP stream-json on stdio as an agent (bulk session config via CAP_SESSION_CONFIG)") @@ -40166,6 +40541,12 @@ def main(argv=None): finally: agent.store.close() return 0 + if args.stats_rag: + try: + print(RagStats.format_report(RagStats.collect(agent.store, agent.profile))) + finally: + agent.store.close() + return 0 if args.scheduler: print("scheduler online, press Ctrl-C to stop") try: diff --git a/taicpp/.clang-format b/taicpp/.clang-format new file mode 100644 index 0000000..75a203b --- /dev/null +++ b/taicpp/.clang-format @@ -0,0 +1,3 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 100 diff --git a/taicpp/Makefile b/taicpp/Makefile new file mode 100644 index 0000000..777b582 --- /dev/null +++ b/taicpp/Makefile @@ -0,0 +1,58 @@ +CXX ?= g++ +CXXSTD = -std=c++23 +WARN = -Wall -Wextra +CXXFLAGS ?= $(CXXSTD) $(WARN) -O2 +BUILD = build +VERSION = $(shell cat VERSION) +BIN = $(BUILD)/detz + +GEN_INC = $(BUILD)/gen +VERSION_HPP = $(GEN_INC)/version.hpp + +SRCS = main.cpp src/cli/cli.cpp +OBJS = $(BUILD)/main.o $(BUILD)/cli.o + +all: $(BIN) + +$(VERSION_HPP): VERSION + @mkdir -p $(GEN_INC) + @sed 's/@VERSION@/$(VERSION)/' src/cli/version.hpp.in > $@ + +$(BUILD)/main.o: main.cpp src/cli/cli.h $(VERSION_HPP) + @mkdir -p $(BUILD) + $(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c main.cpp -o $@ + +$(BUILD)/cli.o: src/cli/cli.cpp src/cli/cli.h $(VERSION_HPP) + @mkdir -p $(BUILD) + $(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c src/cli/cli.cpp -o $@ + +$(BIN): $(OBJS) + $(CXX) $(CXXFLAGS) $(OBJS) -o $@ + +test: $(BIN) $(BUILD)/detz_test + ./$(BUILD)/detz_test + ./$(BIN) --version + +$(BUILD)/detz_test: tests/test_main.cpp src/cli/cli.cpp $(VERSION_HPP) + @mkdir -p $(BUILD) + $(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli tests/test_main.cpp src/cli/cli.cpp -o $@ + +lint: + $(CXX) $(CXXSTD) $(WARN) -Werror -fsyntax-only main.cpp -I$(GEN_INC) -Isrc/cli || true + +version-bump: + tools/version-bump.sh $(PART) + +changelog: + tools/changelog.sh "$(MSG)" + +tag: + tools/tag.sh + +release: + PART=$(PART) MSG="$(MSG)" tools/release.sh + +clean: + rm -rf $(BUILD) + +-include src/*/*.mk diff --git a/taicpp/VERSION b/taicpp/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/taicpp/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/taicpp/build/cli.o b/taicpp/build/cli.o new file mode 100644 index 0000000..12f87f4 Binary files /dev/null and b/taicpp/build/cli.o differ diff --git a/taicpp/build/detz b/taicpp/build/detz new file mode 100755 index 0000000..d466545 Binary files /dev/null and b/taicpp/build/detz differ diff --git a/taicpp/build/detz_test b/taicpp/build/detz_test new file mode 100755 index 0000000..d8c9621 Binary files /dev/null and b/taicpp/build/detz_test differ diff --git a/taicpp/build/gen/version.hpp b/taicpp/build/gen/version.hpp new file mode 100644 index 0000000..d6a80b1 --- /dev/null +++ b/taicpp/build/gen/version.hpp @@ -0,0 +1,3 @@ +#pragma once +#include +namespace detz { constexpr std::string_view kVersion = "0.1.0"; } diff --git a/taicpp/build/main.o b/taicpp/build/main.o new file mode 100644 index 0000000..c5dd804 Binary files /dev/null and b/taicpp/build/main.o differ diff --git a/taicpp/main.cpp b/taicpp/main.cpp new file mode 100644 index 0000000..443c7b1 --- /dev/null +++ b/taicpp/main.cpp @@ -0,0 +1,2 @@ +#include "src/cli/cli.h" +int main(int argc, char** argv) { return detz::cli::DetzCli{}.run(argc, argv); } diff --git a/taicpp/src/agent/.gitkeep b/taicpp/src/agent/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/cap/.gitkeep b/taicpp/src/cap/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/cli/.gitkeep b/taicpp/src/cli/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/cli/cli.cpp b/taicpp/src/cli/cli.cpp new file mode 100644 index 0000000..0b54d7e --- /dev/null +++ b/taicpp/src/cli/cli.cpp @@ -0,0 +1,17 @@ +#include "cli.h" +#include "version.hpp" +#include +namespace detz::cli { +Command Parser::parse(int argc, char** argv) { + Command c; + if (argc > 1) { c.name = argv[1]; for (int i = 2; i < argc; ++i) c.args.emplace_back(argv[i]); } + return c; +} +int Command::run() { + if (name == "--version" || name == "version") { std::puts(kVersion.data()); return 0; } + std::puts("detz — C++23 port of tai.py. See --help. Version:"); + std::puts(kVersion.data()); + return 0; +} +int DetzCli::run(int argc, char** argv) { return Parser{}.parse(argc, argv).run(); } +} // namespace detz::cli diff --git a/taicpp/src/cli/cli.h b/taicpp/src/cli/cli.h new file mode 100644 index 0000000..17a9c92 --- /dev/null +++ b/taicpp/src/cli/cli.h @@ -0,0 +1,10 @@ +#pragma once +#include +#include +namespace detz::cli { +struct Command { std::string name{"--help"}; std::vector args; int run(); }; +class Parser { public: Command parse(int argc, char** argv); }; +class DetzCli { public: int run(int argc, char** argv); }; +// molo-subcommand adapter stub (M1 wires `molo detz` registration here). +class MoloSubcommand { public: const char* name() const { return "detz"; } }; +} // namespace detz::cli diff --git a/taicpp/src/cli/version.hpp.in b/taicpp/src/cli/version.hpp.in new file mode 100644 index 0000000..50f8adf --- /dev/null +++ b/taicpp/src/cli/version.hpp.in @@ -0,0 +1,3 @@ +#pragma once +#include +namespace detz { constexpr std::string_view kVersion = "@VERSION@"; } diff --git a/taicpp/src/comms/.gitkeep b/taicpp/src/comms/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/config/.gitkeep b/taicpp/src/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/llm/.gitkeep b/taicpp/src/llm/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/mcp/.gitkeep b/taicpp/src/mcp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/plexer/.gitkeep b/taicpp/src/plexer/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/seal/.gitkeep b/taicpp/src/seal/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/shell/.gitkeep b/taicpp/src/shell/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/store/.gitkeep b/taicpp/src/store/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/tui/.gitkeep b/taicpp/src/tui/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/src/util/.gitkeep b/taicpp/src/util/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/taicpp/tests/test_main.cpp b/taicpp/tests/test_main.cpp new file mode 100644 index 0000000..e281a0c --- /dev/null +++ b/taicpp/tests/test_main.cpp @@ -0,0 +1,12 @@ +#include "../src/cli/cli.h" +#include +#include +int main() { + detz::cli::Parser p; + const char* a0 = "detz"; + auto c = p.parse(1, (char**)&a0); + assert(c.name == "--help"); + assert(c.run() == 0); + std::puts("M0 OK"); + return 0; +} diff --git a/taicpp/tools/changelog.sh b/taicpp/tools/changelog.sh new file mode 100755 index 0000000..88c0848 --- /dev/null +++ b/taicpp/tools/changelog.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e +msg=${1:-update} +v=$(cat VERSION); d=$(date +%F) +{ echo "# Changelog"; echo; echo "## [$v] - $d"; echo "- $msg"; echo; tail -n +2 CHANGELOG.md; } > CHANGELOG.md.new +mv CHANGELOG.md.new CHANGELOG.md +echo "changelog updated" diff --git a/taicpp/tools/release.sh b/taicpp/tools/release.sh new file mode 100755 index 0000000..72dc51b --- /dev/null +++ b/taicpp/tools/release.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e +: "${PART:=patch}" +tools/version-bump.sh "$PART" +tools/changelog.sh "$MSG" +git add VERSION CHANGELOG.md 2>/dev/null || true +git commit -m "detz: release v$(cat VERSION) $MSG" 2>/dev/null || true +tools/tag.sh diff --git a/taicpp/tools/tag.sh b/taicpp/tools/tag.sh new file mode 100755 index 0000000..47d4bea --- /dev/null +++ b/taicpp/tools/tag.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +v=$(cat VERSION) +git tag -a "detz-v$v" -m "detz v$v" 2>/dev/null || echo "tag detz-v$v (no git or exists)" diff --git a/taicpp/tools/version-bump.sh b/taicpp/tools/version-bump.sh new file mode 100755 index 0000000..dda499b --- /dev/null +++ b/taicpp/tools/version-bump.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e +part=${1:-patch} +v=$(cat VERSION) +maj=${v%%.*}; rest=${v#*.}; min=${rest%%.*}; pat=${rest##*.} +case $part in major) maj=$((maj+1)); min=0; pat=0;; minor) min=$((min+1)); pat=0;; *) pat=$((pat+1));; esac +echo "$maj.$min.$pat" > VERSION +echo "VERSION -> $(cat VERSION)" diff --git a/test_tai.py b/test_tai.py index 63af6d9..472276d 100644 --- a/test_tai.py +++ b/test_tai.py @@ -2239,6 +2239,14 @@ class BundledSkillsTests(unittest.TestCase): self.assertNotIn("`/plan`", text, label) self.assertNotIn("/skill plan", text, label) + def test_deep_research_trigger_prefers_use(self): + text = tai.BUNDLED_SKILLS["deep-research"]["SKILL.md"] + skill = tai.parse_skill_front(text.split("\n---", 1)[0].splitlines()[1:]) + self.assertNotIn("\n", skill["description"]) + catalog = tai.skill_catalog({"deep-research": {"description": skill["description"]}}) + self.assertIn("prefer it over a single lookup", catalog) + self.assertIn("resolving an issue", catalog) + def test_skills_doc_names_everything(self): with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "skills.md"), encoding="utf-8") as handle: doc = handle.read() @@ -4801,6 +4809,77 @@ class RetrievalEvalTests(unittest.TestCase): self.assertEqual(pairs["zero_hit"], ["zzz-nope-nothing"]) +class RagStatsTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + os.environ["TAI_HOME"] = self.tmp.name + + class FakeArgs: + profile = "t" + yes = True + + self.config = tai.Config(FakeArgs()) + self.store = tai.Store(self.config, tai.Seal(self.config.home, "rag-stats-1")) + self.app = FakeSecretApp(self.store) + self.app.profile = "t" + self.app.config = self.config + self.tools = tai.Tools(self.app) + + def tearDown(self): + self.store.close() + self.tmp.cleanup() + os.environ.pop("TAI_HOME", None) + + def seed(self): + ids = {} + ids["title"] = self.store.add_record("note", "quasar beacon", "short body here", [], "t") + ids["stuffed"] = self.store.add_record("note", "unrelated title", "background words filler text. " * 10 + "quasar beacon " * 10, [], "t") + ids["harbor"] = self.store.add_record("note", "harbor launch checklist", "step one step two step three", [], "t") + return ids + + def test_collect_empty(self): + stats = tai.RagStats.collect(self.store, "t", include_eval=False) + self.assertEqual(stats["records_total"], 0) + self.assertEqual(stats["searches_total"], 0) + self.assertEqual(stats["reads_total"], 0) + self.assertIn("rag stats: profile t", tai.RagStats.format_report(stats)) + + def test_collect_counts_search_and_read(self): + ids = self.seed() + self.tools.dispatch("search", json.dumps({"query": "quasar beacon", "kinds": ["record"]})) + self.tools.dispatch("record_read", json.dumps({"id": ids["title"]})) + self.tools.dispatch("search", json.dumps({"query": "zzz-nope-nothing"})) + stats = tai.RagStats.collect(self.store, "t", include_eval=False) + self.assertEqual(stats["records_total"], 3) + self.assertEqual(stats["searches_total"], 2) + self.assertEqual(stats["reads_total"], 1) + self.assertEqual(stats["window_searches"], 2) + self.assertEqual(stats["zero_hit_window"], 1) + self.assertEqual(stats["read_follow"], 1) + self.assertEqual(stats["rank_dist"]["1"], 1) + shown = dict(stats["top_shown"]) + self.assertGreaterEqual(shown.get(ids["title"], 0), 1) + by_id = {row[0]: row for row in stats["effectiveness"]} + self.assertIn(ids["title"], by_id) + _rid, shown_n, read_n, total = by_id[ids["title"]] + self.assertGreaterEqual(shown_n, 1) + self.assertEqual(read_n, 1) + self.assertGreaterEqual(total, 2) + + def test_format_report_covers_sections(self): + ids = self.seed() + self.tools.dispatch("search", json.dumps({"query": "quasar beacon", "kinds": ["record"]})) + self.tools.dispatch("record_read", json.dumps({"id": ids["title"]})) + report = tai.RagStats.format_report(tai.RagStats.collect(self.store, "t")) + for section in ("context uses per record", "read-follow", "hits dist", "tag coverage", + "read rank", "queries:", "eval:", ids["title"]): + self.assertIn(section, report) + + def test_parser_flag(self): + self.assertTrue(tai.build_parser().parse_args(["--stats-rag"]).stats_rag) + self.assertFalse(tai.build_parser().parse_args([]).stats_rag) + + class SupervisorCapTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory()