diff --git a/taicpp/src/install/audittool.cpp b/taicpp/src/install/audittool.cpp new file mode 100644 index 0000000..61b2742 --- /dev/null +++ b/taicpp/src/install/audittool.cpp @@ -0,0 +1,229 @@ +#include "audittool.h" +#include +#include "miscdatetime.h" +#include "miscpy.h" +#include "skillfile.h" +#include "skilltool.h" +namespace detz::install { +using misc::DbValue; +using misc::PyFs; +using misc::PyStr; + +static DbValue opt_text(const std::optional& s) { return s ? DbValue(*s) : DbValue(); } +static DbValue opt_int(const std::optional& v) { return v ? DbValue(*v) : DbValue(); } +static std::optional to_opt_int(const DbValue& v) { return v.is_null() ? std::nullopt : std::optional(v.as_int()); } +static std::optional to_opt_text(const DbValue& v) { return v.is_null() ? std::nullopt : std::optional(v.as_text()); } + +int64_t AuditVault::audit_event_full(const std::string& actor, const std::string& action, const std::string& path, const std::string& message, + const std::optional& old_in, const std::optional& new_in, const std::vector& tags, + const std::optional& old_size, const std::optional& new_size, const std::string& profile_in) { + std::string profile = scope(profile_in); + auto [old_text, computed_old] = SelfRelease::capped_text(old_in); + auto [new_text, computed_new] = SelfRelease::capped_text(new_in); + if (old_in && old_size && *old_size > static_cast(old_in->size())) { + old_text = PyStr::prefix(*old_in, SelfRelease::AUDIT_MAX_CHARS) + SelfRelease::trunc_marker(*old_size); + computed_old = old_size; + } + if (new_in && new_size && *new_size > static_cast(new_in->size())) { + new_text = PyStr::prefix(*new_in, SelfRelease::AUDIT_MAX_CHARS) + SelfRelease::trunc_marker(*new_size); + computed_new = new_size; + } + db_.execute("INSERT INTO audit(profile, ts, actor, action, path, message, old_size, new_size, old, new, tags) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + {profile, misc::DateTime::now_utc().isoformat(), actor, action, path, PyStr::prefix(message, 2000), opt_int(computed_old), opt_int(computed_new), + opt_text(old_text), opt_text(new_text), PyStr::join(misc::TagNames::normalize_tags(tags), " ")}); + return db_.last_insert_rowid(); +} + +std::vector AuditVault::audit_history(const std::string& path, const std::string& tag, int limit, const std::string& profile_in) { + std::string profile = scope(profile_in); + std::string sql = "SELECT id, ts, actor, action, path, message, old_size, new_size, tags FROM audit"; + std::vector clauses = {"profile = ?"}; + std::vector params = {profile}; + if (!path.empty()) { + clauses.push_back("path = ?"); + params.emplace_back(path); + } + if (!tag.empty()) { + clauses.push_back("tags LIKE ?"); + params.emplace_back("%" + tag + "%"); + } + sql += " WHERE " + PyStr::join(clauses, " AND "); + sql += " ORDER BY id DESC LIMIT ?"; + params.emplace_back(std::max(1, std::min(limit, 100))); + std::vector out; + for (const auto& r : db_.query(sql, params)) { + AuditRow row; + row.id = r[0].as_int(); + row.ts = r[1].as_text(); + row.actor = r[2].as_text(); + row.action = r[3].as_text(); + row.path = r[4].as_text(); + row.message = r[5].as_text(); + row.old_size = to_opt_int(r[6]); + row.new_size = to_opt_int(r[7]); + row.tags = r[8].as_text(); + out.push_back(row); + } + return out; +} + +std::optional AuditVault::audit_get(int64_t row_id, const std::string& profile_in) { + std::string profile = scope(profile_in); + auto rows = db_.query("SELECT id, ts, actor, action, path, message, old_size, new_size, old, new, tags FROM audit WHERE profile = ? AND id = ?", {profile, row_id}); + if (rows.empty()) return std::nullopt; + const auto& r = rows[0]; + AuditRow row; + row.id = r[0].as_int(); + row.ts = r[1].as_text(); + row.actor = r[2].as_text(); + row.action = r[3].as_text(); + row.path = r[4].as_text(); + row.message = r[5].as_text(); + row.old_size = to_opt_int(r[6]); + row.new_size = to_opt_int(r[7]); + row.old_text = to_opt_text(r[8]); + row.new_text = to_opt_text(r[9]); + row.tags = r[10].as_text(); + return row; +} + +std::vector AuditVault::audit_search(rag::FtsSearch& fts, const std::string& query, const std::string& profile_in, int limit) { + std::string profile = scope(profile_in); + std::vector found; + for (const auto& hit : fts.fts_search(query, {"audit"}, profile, limit)) { + std::string item = hit.item.as_text(); + std::size_t colon = item.find(':'); + if (colon == std::string::npos) continue; + std::string tail = item.substr(colon + 1); + if (!PyStr::isdigit_ascii(tail)) continue; + auto row = audit_get(std::stoll(tail), profile); + if (row) { + row->snippet = hit.snippet; + found.push_back(*row); + } + } + return found; +} + +std::vector> AuditVault::tag_counts(const std::string& prefix, int limit, const std::string& profile_in) { + std::string profile = scope(profile_in); + std::string clean = misc::TagNames::normalize_tag(prefix); + std::string sql = "SELECT tag, COUNT(*) FROM tags WHERE profile = ?"; + std::vector params = {profile}; + if (!clean.empty()) { + sql += " AND tag LIKE ? ESCAPE '\\'"; + std::string esc = PyStr::replace_all(PyStr::replace_all(PyStr::replace_all(clean, "\\", "\\\\"), "%", "\\%"), "_", "\\_"); + params.emplace_back(esc + "%"); + } + sql += " GROUP BY tag ORDER BY COUNT(*) DESC, tag LIMIT ?"; + params.emplace_back(std::max(1, std::min(limit, 200))); + std::vector> out; + for (const auto& r : db_.query(sql, params)) out.emplace_back(r[0].as_text(), r[1].as_int()); + return out; +} + +// int(args.get(key) or dflt); nullopt on ValueError/TypeError +static std::optional arg_int(const misc::Json& args, const std::string& key, int64_t dflt) { + const misc::Json* v = args.get(key); + std::string text = skills::SkillTool::arg_text(args, key); + if (!v || text.empty()) return dflt; + switch (v->kind()) { + case misc::Json::Kind::Bool: return v->as_bool() ? 1 : 0; + case misc::Json::Kind::Int: return v->as_int(); + case misc::Json::Kind::Float: return static_cast(v->as_double()); + case misc::Json::Kind::String: { + std::string s = PyStr::strip(text); + std::size_t i = 0; + bool neg = false; + if (i < s.size() && (s[i] == '+' || s[i] == '-')) { neg = s[i] == '-'; ++i; } + if (i >= s.size()) return std::nullopt; + int64_t val = 0; + for (; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') return std::nullopt; + val = val * 10 + (s[i] - '0'); + } + return neg ? -val : val; + } + default: return std::nullopt; + } +} + +std::string AuditTool::run_audit(const misc::Json& args) { + std::string path = PyStr::strip(skills::SkillTool::arg_text(args, "path")); + std::string tag = PyStr::strip(skills::SkillTool::arg_text(args, "tag")); + std::string query = PyStr::strip(skills::SkillTool::arg_text(args, "query")); + auto limit = arg_int(args, "limit", 20); + if (!limit) return "error: invalid limit"; + std::vector rows; + int lim = static_cast(*limit); + if (!query.empty()) rows = host_.vault().audit_search(host_.fts(), query, host_.active_profile(), lim); + else rows = host_.vault().audit_history(path, tag, lim, host_.active_profile()); + if (rows.empty()) return "no audit rows match"; + std::vector lines; + for (const auto& row : rows) { + std::string old_size = row.old_size ? std::to_string(*row.old_size) : "-"; + std::string new_size = row.new_size ? std::to_string(*row.new_size) : "-"; + std::string line = "#" + std::to_string(row.id) + " " + PyStr::prefix(row.ts, 19) + " " + row.actor + " " + row.action + " " + row.path + " (" + old_size + "->" + new_size + " bytes)"; + if (!row.message.empty()) line += " :: " + PyStr::prefix(row.message, 120); + if (!row.tags.empty()) line += " [" + row.tags + "]"; + lines.push_back(line); + } + return PyStr::join(lines, "\n"); +} + +std::string AuditTool::run_restore(const misc::Json& args) { + auto parsed = arg_int(args, "id", 0); + if (!parsed) return "error: invalid audit id"; + int64_t row_id = *parsed; + auto row = host_.vault().audit_get(row_id, host_.active_profile()); + if (!row) return "error: unknown audit #" + std::to_string(row_id); + if (row->action == "snapshot" || row->action == "release") { + return "error: audit #" + std::to_string(row_id) + " is a " + row->action + " row, restorable rows are write, edit, delete, shell-snapshot, restore"; + } + std::string target, which; + std::optional full_size; + if (row->new_text) { + target = *row->new_text; + which = "post"; + full_size = row->new_size; + } else if (row->old_text) { + target = *row->old_text; + which = "pre"; + full_size = row->old_size; + } else { + return "error: audit #" + std::to_string(row_id) + " holds no content"; + } + if (target.find("[truncated, full size") != std::string::npos) { + return "error: audit #" + std::to_string(row_id) + " " + which + "-image is truncated (full size " + (full_size ? std::to_string(*full_size) : std::string("None")) + + " bytes), cannot restore safely"; + } + if (!host_.ask_approval("restore " + row->path + " from audit #" + std::to_string(row_id) + " (" + which + "-image)")) return "denied by user"; + std::string message = "restored from audit #" + std::to_string(row_id) + " (" + which + "-image)"; + if (PyStr::starts_with(row->path, "sandbox:")) return host_.box_restore(row->path.substr(8), target, message); + CappedRead current; + if (PyFs::isfile(row->path)) current = SelfRelease::read_capped(row->path); + try { + std::string parent = PyFs::dirname(row->path); + if (!parent.empty()) PyFs::makedirs(parent); + PyFs::write_text(row->path, target); + host_.read_files_add(PyFs::abspath(row->path)); + } catch (const misc::OsError& e) { + return "error: " + PyStr::short_error(e.what()); + } + host_.audit_file("restore", row->path, std::nullopt, target, message); + host_.upsert_file_record(row->path, target, host_.active_profile()); + if (!current.size) host_.note_diff(row->path, current.text, target); + return "restored " + row->path + " from audit #" + std::to_string(row_id) + " (" + which + "-image)"; +} + +std::string AuditTool::run_tags(const misc::Json& args) { + std::string prefix = PyStr::strip(skills::SkillTool::arg_text(args, "prefix")); + auto limit = arg_int(args, "limit", 50); + if (!limit) return "error: invalid limit"; + auto counts = host_.vault().tag_counts(prefix, static_cast(*limit), host_.active_profile()); + if (counts.empty()) return "no tags yet"; + std::vector lines; + for (const auto& c : counts) lines.push_back(c.first + " (" + std::to_string(c.second) + ")"); + return std::to_string(counts.size()) + " tags:\n" + PyStr::join(lines, "\n"); +} +} // namespace detz::install diff --git a/taicpp/src/install/audittool.h b/taicpp/src/install/audittool.h new file mode 100644 index 0000000..fd25103 --- /dev/null +++ b/taicpp/src/install/audittool.h @@ -0,0 +1,73 @@ +#pragma once +// Port of Store.audit_event/audit_history/audit_get/audit_search/tag_counts and the Tools backends +// run_audit, run_restore, run_tags. +#include +#include +#include +#include +#include +#include "ftssearch.h" +#include "miscjson.h" +#include "release.h" +#include "sqlitedb.h" +namespace detz::install { + +struct AuditRow { + int64_t id{0}; + std::string ts, actor, action, path, message; + std::optional old_size, new_size; + std::optional old_text, new_text; // audit_get only + std::string tags; + std::string snippet; // audit_search only +}; + +class AuditVault : public AuditPort { + public: + AuditVault(misc::SqliteDb& db, std::string profile) : db_(db), profile_(std::move(profile)) {} + int64_t audit_event(const std::string& actor, const std::string& action, const std::string& path, const std::string& message, + const std::optional& old_text, const std::optional& new_text, + const std::vector& tags) override { + return audit_event_full(actor, action, path, message, old_text, new_text, tags, std::nullopt, std::nullopt, ""); + } + int64_t audit_event_full(const std::string& actor, const std::string& action, const std::string& path, const std::string& message, + const std::optional& old_text, const std::optional& new_text, const std::vector& tags, + const std::optional& old_size, const std::optional& new_size, const std::string& profile); + std::vector audit_history(const std::string& path, const std::string& tag, int limit, const std::string& profile = ""); + std::optional audit_get(int64_t row_id, const std::string& profile = ""); + std::vector audit_search(rag::FtsSearch& fts, const std::string& query, const std::string& profile, int limit); + std::vector> tag_counts(const std::string& prefix, int limit, const std::string& profile = ""); + + private: + misc::SqliteDb& db_; + std::string profile_; + std::string scope(const std::string& p) const { return p.empty() ? profile_ : p; } +}; + +// What the Tools object supplies around the vault +class AuditHost { + public: + virtual ~AuditHost() = default; + virtual bool ask_approval(const std::string& what) = 0; + virtual std::string active_profile() const = 0; + virtual AuditVault& vault() = 0; + virtual rag::FtsSearch& fts() = 0; + virtual void read_files_add(const std::string& abs_path) = 0; // self.read_files.add(("home", ...)) + virtual void audit_file(const std::string& action, const std::string& path, const std::optional& old_text, + const std::optional& new_text, const std::string& message) = 0; + virtual void upsert_file_record(const std::string& path, const std::string& text, const std::string& profile) = 0; + virtual void note_diff(const std::string& path, const std::optional& old_text, const std::optional& new_text) = 0; + virtual std::string box_restore(const std::string& path, const std::string& target, const std::string& message) = 0; +}; + +class AuditTool { + public: + explicit AuditTool(AuditHost& host) : host_(host) {} + std::string run_audit(const misc::Json& args); + std::string run_restore(const misc::Json& args); + std::string run_tags(const misc::Json& args); + + private: + AuditHost& host_; +}; + +} // namespace detz::install diff --git a/taicpp/src/install/install.cpp b/taicpp/src/install/install.cpp new file mode 100644 index 0000000..88ae769 --- /dev/null +++ b/taicpp/src/install/install.cpp @@ -0,0 +1,714 @@ +#include "install.h" +#include +#include +#include +#include +#include +#include +#include "installconsts.h" +#include "skilltool.h" +#include "telegram.h" +namespace detz::install { +using misc::Json; +using misc::OsError; +using misc::PyFs; +using misc::PyProc; +using misc::PyStr; + +static std::string short_err(const std::exception& e) { return PyStr::short_error(e.what()); } +static std::string home_dir() { return PyFs::home(); } + +// ---------------------------------------------------------------- RcBlocks +bool RcBlocks::present(const std::string& path, const std::string& block) { + try { + return PyFs::read_text(path).find(block) != std::string::npos; + } catch (const OsError&) { + return false; + } +} + +std::string RcBlocks::strip_blocks(const std::string& raw, const std::string& mark_begin, const std::string& mark_end) { + std::string out; + std::size_t pos = 0; + while (true) { + std::size_t b = raw.find(mark_begin, pos); + if (b == std::string::npos) break; + std::size_t e = raw.find(mark_end, b + mark_begin.size()); + if (e == std::string::npos) break; + out.append(raw, pos, b - pos); + pos = e + mark_end.size(); + if (pos < raw.size() && raw[pos] == '\n') ++pos; + } + out.append(raw, pos, std::string::npos); + return out; +} + +bool RcBlocks::remove(const std::string& path, const std::string& mark_begin, const std::string& mark_end) { + std::string raw; + try { + raw = PyFs::read_text(path); + } catch (const OsError&) { + return false; + } + std::string stripped = strip_blocks(raw, mark_begin, mark_end); + if (stripped == raw) return false; + std::string backup = path + ".bak-tai"; + if (!raw.empty() && !PyFs::exists(backup)) PyFs::write_text(backup, raw); + PyFs::write_text(path, stripped); + return true; +} + +bool RcBlocks::upsert(const std::string& path, const std::string& mark_begin, const std::string& mark_end, const std::string& block) { + std::string raw; + try { + raw = PyFs::read_text(path); + } catch (const OsError&) { + raw = ""; + } + if (raw.find(block) != std::string::npos) return false; + std::string backup = path + ".bak-tai"; + if (!raw.empty() && !PyFs::exists(backup)) PyFs::write_text(backup, raw); + std::string stripped = strip_blocks(raw, mark_begin, mark_end); + if (!stripped.empty() && stripped.back() != '\n') stripped += "\n"; + std::string parent = PyFs::dirname(path); + if (!parent.empty() && !PyFs::isdir(parent)) PyFs::makedirs(parent); + PyFs::write_text(path, stripped + block); + return true; +} + +// ---------------------------------------------------------------- Systemd +std::string Systemd::unit_dir(const std::string&) { return PyFs::join(PyFs::join(PyFs::join(home_dir(), ".config"), "systemd"), "user"); } + +std::pair Systemd::service_state(const std::string& name) { + if (!PyFs::which("systemctl")) return {"unknown", "unknown"}; + auto active = PyProc::run_host({"systemctl", "--user", "is-active", name}, 15); + auto enabled = PyProc::run_host({"systemctl", "--user", "is-enabled", name}, 15); + return {active.ok ? "active" : "inactive", enabled.ok ? "enabled" : "disabled"}; +} + +std::string Systemd::getuser() { + for (const char* key : {"LOGNAME", "USER", "LNAME", "USERNAME"}) { + const char* v = std::getenv(key); + if (v && *v) return v; + } + struct passwd* pw = getpwuid(getuid()); + return pw ? pw->pw_name : ""; +} + +std::string Systemd::activate_service(const std::string& name) { + if (!PyFs::which("systemctl")) return "systemctl missing, unit written only"; + PyProc::run_host({"systemctl", "--user", "daemon-reload"}); + auto state = service_state(name); + if (state.first == "active") { + auto r = PyProc::run_host({"systemctl", "--user", "restart", name}); + return r.ok ? "restarted, changes effective immediately" : "restart failed: " + PyStr::prefix(PyStr::strip(r.out), 150); + } + auto r = PyProc::run_host({"systemctl", "--user", "enable", "--now", name}); + if (r.ok) { + std::string user = PyFs::env("USER"); + if (user.empty()) user = getuser(); + PyProc::run_host({"loginctl", "enable-linger", user}); + return "enabled and started"; + } + return "enable failed: " + PyStr::prefix(PyStr::strip(r.out), 150); +} + +std::pair Systemd::remove_service_unit(const std::string& home, const std::string& name) { + std::string path = PyFs::join(unit_dir(home), name); + if (PyFs::which("systemctl")) { + PyProc::run_host({"systemctl", "--user", "disable", "--now", name}); + PyProc::run_host({"systemctl", "--user", "daemon-reload"}); + } + if (PyFs::exists(path)) { + try { + PyFs::remove(path); + return {true, "removed " + path}; + } catch (const OsError& e) { + return {false, short_err(e)}; + } + } + return {true, "not present"}; +} + +// ---------------------------------------------------------------- Install +bool Install::shell_present(const std::string& shell) { return PyFs::which(shell).has_value(); } + +bool Install::venv_available(const std::string& python) { + auto r = PyProc::run_host({python, "-c", + "import importlib.util as u;print(int(u.find_spec('venv') is not None), int(u.find_spec('ensurepip') is not None))"}, + 60); + if (!r.ok) return false; + std::string t = PyStr::strip(r.out); + if (t.size() < 3 || t[0] != '1') return false; + if (t[2] == '1') return true; + return PyFs::which("pip3").has_value() || PyFs::which("pip").has_value(); +} + +std::string Install::venv_dir(const std::string& home) { return PyFs::join(home, "venv"); } + +std::optional Install::venv_python(const std::string& home) { + std::string cand = PyFs::join(PyFs::join(venv_dir(home), "bin"), "python"); + if (PyFs::isfile(cand) && PyFs::is_executable(cand)) return cand; + return std::nullopt; +} + +std::pair Install::ensure_venv(const std::string& home, const std::string& python) { + if (auto found = venv_python(home)) return {true, "kept " + *found}; + if (!venv_available(python)) return {false, "python venv module unavailable"}; + misc::ProcResult done = PyProc::run({python, "-m", "venv", venv_dir(home)}, 300); + if (done.failed) return {false, done.error}; + if (done.code != 0 || !venv_python(home)) { + std::string err = PyStr::prefix(PyStr::strip(done.err), 200); + return {false, err.empty() ? "venv creation failed" : err}; + } + return {true, "created " + *venv_python(home)}; +} + +std::optional Install::script_version(const std::string& path) { + std::string text; + try { + text = PyFs::read_text(path); + } catch (const OsError&) { + return std::nullopt; + } + std::size_t start = 0; + while (start <= text.size()) { + std::size_t nl = text.find('\n', start); + std::string line = text.substr(start, nl == std::string::npos ? std::string::npos : nl - start); + static const std::string head = "VERSION = \""; + if (PyStr::starts_with(line, head) && line.size() > head.size() && line.back() == '"') { + std::string inner = line.substr(head.size(), line.size() - head.size() - 1); + int parts = 0; + bool ok = true; + std::size_t i = 0; + while (i <= inner.size() && ok) { + std::size_t dot = inner.find('.', i); + std::string piece = inner.substr(i, dot == std::string::npos ? std::string::npos : dot - i); + if (!PyStr::isdigit_ascii(piece)) ok = false; + ++parts; + if (dot == std::string::npos) break; + i = dot + 1; + } + if (ok && parts == 3) return inner; + } + if (nl == std::string::npos) break; + start = nl + 1; + } + return std::nullopt; +} + +std::string Install::shell_rc_path(const std::string& shell) { + std::string home = home_dir(); + if (shell == "fish") { + std::string config_home = PyFs::env("XDG_CONFIG_HOME"); + if (config_home.empty()) config_home = PyFs::join(home, ".config"); + return PyFs::join(PyFs::join(config_home, "fish"), "config.fish"); + } + if (shell == "zsh") return PyFs::join(home, ".zshrc"); + return PyFs::join(home, ".bashrc"); +} + +// ---------------------------------------------------------------- StdIo +void StdIo::print(const std::string& line) { std::cout << line << std::endl; } + +std::optional StdIo::getpass(const std::string& prompt) { + int fd = ::open("/dev/tty", O_RDWR | O_NOCTTY); + bool tty = fd >= 0; + if (!tty) { + fd = 0; + std::cerr << "Warning: Password input may be echoed." << std::endl; + } + int out_fd = tty ? fd : 2; + ssize_t w = ::write(out_fd, prompt.data(), prompt.size()); + (void)w; + struct termios old_term; + bool restore = false; + if (tty && tcgetattr(fd, &old_term) == 0) { + struct termios t = old_term; + t.c_lflag &= ~static_cast(ECHO); + tcsetattr(fd, TCSAFLUSH, &t); + restore = true; + } + std::string line; + bool got_any = false, eof = false; + char c; + while (true) { + ssize_t n = ::read(fd, &c, 1); + if (n <= 0) { eof = true; break; } + got_any = true; + if (c == '\n') break; + line += c; + } + if (restore) tcsetattr(fd, TCSAFLUSH, &old_term); + if (tty) { + w = ::write(out_fd, "\n", 1); + ::close(fd); + } + if (eof && !got_any) return std::nullopt; + if (!line.empty() && line.back() == '\r') line.pop_back(); + return line; +} + +// ---------------------------------------------------------------- Installer +Installer::Installer(InstallContext ctx, ContainerPort& box, InstallIo& io, telegram::TelegramTransport* transport) + : ctx_(std::move(ctx)), box_(box), io_(io), transport_(transport), skills_(ctx_.version) { + if (!transport_) { + own_transport_ = std::make_unique(); + transport_ = own_transport_.get(); + } +} + +std::string Installer::tai_home() { + std::string h = PyFs::env("TAI_HOME"); + return h.empty() ? PyFs::join(home_dir(), ".tai") : h; +} + +std::optional Installer::script_version(const std::string& path) const { + return ctx_.version_reader ? ctx_.version_reader(path) : Install::script_version(path); +} + +std::string Installer::service_exec(const std::string& home, const std::string& script, const std::vector& args) const { + std::vector parts; + if (ctx_.script_needs_interpreter) { + if (auto python = Install::venv_python(home)) parts.push_back(*python); + } + parts.push_back(script); + parts.insert(parts.end(), args.begin(), args.end()); + return PyStr::join(parts, " "); +} + +bool Installer::hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::BASHRC_BLOCK()); } +bool Installer::remove_bashrc_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END()); } +bool Installer::upsert_bashrc_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END(), InstallConsts::BASHRC_BLOCK()); } +bool Installer::zsh_hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::ZSH_NOTFOUND_BLOCK()); } +bool Installer::remove_zsh_hook_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END()); } +bool Installer::upsert_zsh_hook_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END(), InstallConsts::ZSH_NOTFOUND_BLOCK()); } +bool Installer::fish_hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::FISH_NOTFOUND_BLOCK()); } +bool Installer::remove_fish_hook_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END()); } +bool Installer::upsert_fish_hook_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::BASHRC_MARK_BEGIN(), InstallConsts::BASHRC_MARK_END(), InstallConsts::FISH_NOTFOUND_BLOCK()); } +bool Installer::plexer_hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::PLEXER_BASHRC_BLOCK()); } +bool Installer::remove_plexer_bashrc_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END()); } +bool Installer::upsert_plexer_bashrc_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END(), InstallConsts::PLEXER_BASHRC_BLOCK()); } +bool Installer::zsh_plexer_hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::PLEXER_BASHRC_BLOCK()); } +bool Installer::remove_zsh_plexer_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END()); } +bool Installer::upsert_zsh_plexer_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END(), InstallConsts::PLEXER_BASHRC_BLOCK()); } +bool Installer::fish_plexer_hook_present(const std::string& p) { return RcBlocks::present(p, InstallConsts::FISH_PLEXER_BLOCK()); } +bool Installer::remove_fish_plexer_block(const std::string& p) { return RcBlocks::remove(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END()); } +bool Installer::upsert_fish_plexer_block(const std::string& p) { return RcBlocks::upsert(p, InstallConsts::PLEXER_BASHRC_MARK_BEGIN(), InstallConsts::PLEXER_BASHRC_MARK_END(), InstallConsts::FISH_PLEXER_BLOCK()); } + +std::string Installer::shell_hook_status_line(const std::string& label, const std::string& shell, const std::function& present_fn) const { + std::string path = Install::shell_rc_path(shell); + std::string note = Install::shell_present(shell) ? "" : " (" + shell + " not found)"; + return label + ": " + (present_fn(path) ? "installed" : "not installed") + " in " + path + note; +} + +std::string Installer::vault_line(const std::string& home, misc::SqliteDb* store) { + std::string db_path = PyFs::join(home, "memory.db"); + if (!PyFs::isfile(db_path)) return "vault: empty, install ops never touch data"; + if (!store) return "vault: kept " + db_path + ", install ops never touch data"; + try { + auto secrets = store->query("SELECT COUNT(*) FROM secrets")[0][0].as_int(); + auto schedules = store->query("SELECT COUNT(*) FROM schedules")[0][0].as_int(); + auto records = store->query("SELECT COUNT(*) FROM records")[0][0].as_int(); + return "vault: kept (" + std::to_string(secrets) + " secrets, " + std::to_string(schedules) + " schedules, " + std::to_string(records) + + " records), install ops never touch data"; + } catch (const misc::SqliteError&) { + return "vault: kept " + db_path + ", install ops never touch data"; + } +} + +std::vector Installer::status_lines(const std::string& home, misc::SqliteDb* store) const { + std::vector lines; + std::string h = home_dir(); + std::string binary = PyFs::join(PyFs::join(PyFs::join(h, ".local"), "bin"), "tai.py"); + if (PyFs::isfile(binary)) { + auto v = script_version(binary); + lines.push_back("binary: installed " + binary + " (" + (v ? *v : "unknown") + ", running " + ctx_.version + ")"); + } else { + lines.push_back("binary: not installed"); + } + lines.push_back(std::string("bash-hook: ") + (hook_present(PyFs::join(h, ".bashrc")) ? "installed" : "not installed") + " in ~/.bashrc"); + lines.push_back(shell_hook_status_line("zsh-hook", "zsh", zsh_hook_present)); + lines.push_back(shell_hook_status_line("fish-hook", "fish", fish_hook_present)); + lines.push_back(std::string("plexer-hook: ") + (plexer_hook_present(PyFs::join(h, ".bashrc")) ? "installed" : "not installed") + " in ~/.bashrc"); + lines.push_back(shell_hook_status_line("zsh-plexer-hook", "zsh", zsh_plexer_hook_present)); + lines.push_back(shell_hook_status_line("fish-plexer-hook", "fish", fish_plexer_hook_present)); + if (auto python = Install::venv_python(home)) lines.push_back("venv: ready " + *python); + else if (Install::venv_available(ctx_.python)) lines.push_back("venv: not created"); + else lines.push_back("venv: unavailable (no venv module)"); + for (const auto& pr : std::vector>{{"tai-scheduler.service", "scheduler-service"}, {"tai-telegram.service", "telegram-service"}}) { + std::string unit = PyFs::join(Systemd::unit_dir(home), pr.first); + if (!PyFs::isfile(unit)) { + lines.push_back(pr.second + ": not installed"); + continue; + } + auto st = Systemd::service_state(pr.first); + lines.push_back(pr.second + ": unit present, " + st.first + ", " + st.second); + } + auto engine = box_.container_engine(); + if (!engine) { + lines.push_back("container: no engine (install podman)"); + } else { + std::string state = box_.box_state(*engine); + if (state == "running") lines.push_back("container: running, " + box_.box_venv_state(*engine)); + else lines.push_back("container: " + state); + } + lines.push_back(skills::BundledSkills::status_line(home)); + lines.push_back(vault_line(home, store)); + return lines; +} + +std::string Installer::op_binary(const std::string& mode, const std::string&) const { + std::string target = PyFs::join(PyFs::join(PyFs::join(home_dir(), ".local"), "bin"), "tai.py"); + std::string src = PyFs::abspath(ctx_.script_path); + auto current = script_version(target); + if (mode == "install" && current) return "binary: already installed " + *current + " (running " + ctx_.version + ")"; + if (mode == "uninstall") { + if (!PyFs::exists(target)) return "binary: not present"; + try { + PyFs::remove(target); + return "binary: removed " + target; + } catch (const OsError& e) { + return "binary: remove failed: " + short_err(e); + } + } + try { + PyFs::makedirs(PyFs::dirname(target)); + std::string blob = PyFs::read_bytes(src); + try { + PyFs::write_bytes(target, blob); + } catch (const OsError& e) { + if (e.err() != ETXTBSY) throw; + // a running native binary cannot be rewritten in place: write beside it and rename + PyFs::atomic_write(target, blob); + } + PyFs::chmod(target, 0755); + } catch (const OsError& e) { + return "binary: copy failed: " + short_err(e); + } + if (!current) return "binary: installed " + ctx_.version; + return "binary: refreshed " + *current + " -> " + ctx_.version; +} + +std::string Installer::op_hook(const std::string& mode, const std::string&) const { + std::string path = PyFs::join(home_dir(), ".bashrc"); + if (mode == "uninstall") return std::string("bash-hook: ") + (remove_bashrc_block(path) ? "removed" : "not present"); + if (mode == "reinstall") remove_bashrc_block(path); + bool changed; + try { + changed = upsert_bashrc_block(path); + } catch (const OsError& e) { + return "bash-hook: write failed: " + short_err(e); + } + if (changed) return "bash-hook: installed, run: source ~/.bashrc"; + return "bash-hook: already present"; +} + +std::string Installer::op_plexer_hook(const std::string& mode, const std::string&) const { + std::string path = PyFs::join(home_dir(), ".bashrc"); + if (mode == "uninstall") return std::string("plexer-hook: ") + (remove_plexer_bashrc_block(path) ? "removed" : "not present"); + if (mode == "reinstall") remove_plexer_bashrc_block(path); + bool changed; + try { + changed = upsert_plexer_bashrc_block(path); + } catch (const OsError& e) { + return "plexer-hook: write failed: " + short_err(e); + } + if (changed) return "plexer-hook: installed, run: source ~/.bashrc (new interactive shells auto-wrap; existing ones are unaffected until restarted)"; + return "plexer-hook: already present"; +} + +std::string Installer::op_shell_hook(const std::string& mode, const std::string& shell, const std::string& label, + const std::function& upsert_fn, const std::function& remove_fn, + const std::string& source_hint) const { + std::string path = Install::shell_rc_path(shell); + if ((mode == "install" || mode == "upgrade") && !Install::shell_present(shell)) return label + ": " + shell + " not found, skipped"; + if (mode == "uninstall") return label + ": " + (remove_fn(path) ? "removed" : "not present"); + if (mode == "reinstall") remove_fn(path); + bool changed; + try { + changed = upsert_fn(path); + } catch (const OsError& e) { + return label + ": write failed: " + short_err(e); + } + if (changed) return label + ": installed, run: " + source_hint; + return label + ": already present"; +} + +std::string Installer::op_zsh_hook(const std::string& mode, const std::string&) const { + return op_shell_hook(mode, "zsh", "zsh-hook", upsert_zsh_hook_block, remove_zsh_hook_block, "source ~/.zshrc"); +} +std::string Installer::op_fish_hook(const std::string& mode, const std::string&) const { + return op_shell_hook(mode, "fish", "fish-hook", upsert_fish_hook_block, remove_fish_hook_block, "open a new fish shell"); +} +std::string Installer::op_zsh_plexer_hook(const std::string& mode, const std::string&) const { + return op_shell_hook(mode, "zsh", "zsh-plexer-hook", upsert_zsh_plexer_block, remove_zsh_plexer_block, + "source ~/.zshrc (new interactive shells auto-wrap; existing ones are unaffected until restarted)"); +} +std::string Installer::op_fish_plexer_hook(const std::string& mode, const std::string&) const { + return op_shell_hook(mode, "fish", "fish-plexer-hook", upsert_fish_plexer_block, remove_fish_plexer_block, + "open a new fish shell (new interactive shells auto-wrap; existing ones are unaffected until restarted)"); +} + +std::string Installer::op_venv(const std::string& mode, const std::string& home) const { + if (mode == "uninstall") { + std::string folder = Install::venv_dir(home); + if (!PyFs::isdir(folder)) return "venv: not present"; + try { + PyFs::rmtree(folder); + return "venv: removed " + folder; + } catch (const OsError& e) { + return "venv: remove failed: " + short_err(e); + } + } + if (mode == "reinstall" && PyFs::isdir(Install::venv_dir(home))) { + try { + PyFs::rmtree(Install::venv_dir(home)); + } catch (const OsError& e) { + return "venv: rebuild failed: " + short_err(e); + } + } + auto res = Install::ensure_venv(home, ctx_.python); + return "venv: " + res.second; +} + +std::string Installer::op_service(const std::string& mode, const std::string& home, const std::string& name, const std::string& label, const std::string& body) const { + std::string unit = PyFs::join(Systemd::unit_dir(home), name); + if (mode == "uninstall") return label + ": " + Systemd::remove_service_unit(home, name).second; + bool existed = PyFs::isfile(unit); + if (mode == "install" && existed) { + auto st = Systemd::service_state(name); + return label + ": already installed (" + st.first + ", " + st.second + ")"; + } + if (mode == "reinstall" && PyFs::which("systemctl")) misc::PyProc::run_host({"systemctl", "--user", "disable", "--now", name}); + try { + PyFs::makedirs(PyFs::dirname(unit)); + PyFs::write_text(unit, body); + } catch (const OsError& e) { + return label + ": unit write failed: " + short_err(e); + } + return label + ": unit " + (existed ? "rewritten" : "written") + ", " + Systemd::activate_service(name); +} + +std::string Installer::op_scheduler_service(const std::string& mode, const std::string& home) const { + std::string body = PyStr::percent(InstallConsts::SCHEDULER_UNIT(), {service_exec(home, PyFs::abspath(ctx_.script_path))}); + return op_service(mode, home, "tai-scheduler.service", "scheduler-service", body); +} + +std::string Installer::op_telegram_service(const std::string& mode, const std::string& home) const { + if (mode == "uninstall") return "telegram-service: " + Systemd::remove_service_unit(home, "tai-telegram.service").second; + std::string unit = PyFs::join(Systemd::unit_dir(home), "tai-telegram.service"); + if (mode == "install" && PyFs::isfile(unit)) { + auto st = Systemd::service_state("tai-telegram.service"); + return "telegram-service: already installed (" + st.first + ", " + st.second + ")"; + } + std::string token = PyFs::env("TELEGRAM_BOT_TOKEN"); + if (token.empty()) token = telegram::TelegramBot::load_telegram_token(home); + if (token.empty()) return "telegram-service: needs TELEGRAM_BOT_TOKEN in the environment"; + Json me; + try { + me = telegram::Telegram(token, transport_, ctx_.telegram_api).call("getMe", Json::object()); + } catch (const telegram::TelegramNetError& e) { + return "telegram-service: token rejected: " + short_err(e); + } catch (const misc::JsonError& e) { + return "telegram-service: token rejected: " + short_err(e); + } + std::string env_path = telegram::TelegramBot::telegram_env_path(home); + try { + PyFs::write_text(env_path, "TELEGRAM_BOT_TOKEN=" + token + "\n"); + PyFs::chmod(env_path, 0600); + } catch (const OsError& e) { + return "telegram-service: env write failed: " + short_err(e); + } + std::string body = PyStr::percent(InstallConsts::TELEGRAM_UNIT(), {service_exec(home, PyFs::abspath(ctx_.script_path)), env_path}); + if (mode == "reinstall" && PyFs::which("systemctl")) misc::PyProc::run_host({"systemctl", "--user", "disable", "--now", "tai-telegram.service"}); + try { + PyFs::makedirs(PyFs::dirname(unit)); + PyFs::write_text(unit, body); + } catch (const OsError& e) { + return "telegram-service: unit write failed: " + short_err(e); + } + const Json* result = me.is_object() ? me.get("result") : nullptr; + const Json* uname = result && result->is_object() ? result->get("username") : nullptr; + std::string who = uname && uname->is_string() && !uname->as_string().empty() ? uname->as_string() : "?"; + return "telegram-service: @" + who + ", " + Systemd::activate_service("tai-telegram.service"); +} + +std::string Installer::op_container(const std::string& mode, const std::string&) const { + auto engine = box_.container_engine(); + if (!engine) return "container: no engine (install podman)"; + if (mode == "uninstall") { + PyProc::run_host({*engine, "rm", "-f", InstallConsts::BOX_NAME}); + auto r = PyProc::run_host({*engine, "rmi", "-f", InstallConsts::BOX_IMAGE}); + return std::string("container: removed") + (r.ok ? ", image purged" : ""); + } + if (mode == "reinstall") { + PyProc::run_host({*engine, "rm", "-f", InstallConsts::BOX_NAME}); + PyProc::run_host({*engine, "rmi", "-f", InstallConsts::BOX_IMAGE}); + } + std::string eng; + try { + eng = box_.ensure_box(); + } catch (const misc::BackendError& e) { + return std::string("container: failed: ") + e.what(); + } + std::string state = box_.box_state(eng); + std::string detail = state == "running" ? ", " + box_.box_venv_state(eng) : ""; + if (mode == "reinstall") return "container: rebuilt and " + state + detail; + if (mode == "install" && state == "running") return "container: already running" + detail; + return "container: " + state + detail; +} + +std::string Installer::op_skills(const std::string& mode, const std::string& home) const { return skills_.op_skills(mode, home); } + +std::string Installer::install_report(const std::string& action, const std::vector& targets, const std::string& home, misc::SqliteDb* store) const { + if (action == "status") return PyStr::join(status_lines(home, store), "\n"); + std::vector lines; + for (const auto& target : targets) { + if (target == "binary") lines.push_back(op_binary(action, home)); + else if (target == "bash-hook") lines.push_back(op_hook(action, home)); + else if (target == "zsh-hook") lines.push_back(op_zsh_hook(action, home)); + else if (target == "fish-hook") lines.push_back(op_fish_hook(action, home)); + else if (target == "plexer-hook") lines.push_back(op_plexer_hook(action, home)); + else if (target == "zsh-plexer-hook") lines.push_back(op_zsh_plexer_hook(action, home)); + else if (target == "fish-plexer-hook") lines.push_back(op_fish_plexer_hook(action, home)); + else if (target == "venv") lines.push_back(op_venv(action, home)); + else if (target == "scheduler-service") lines.push_back(op_scheduler_service(action, home)); + else if (target == "telegram-service") lines.push_back(op_telegram_service(action, home)); + else if (target == "container") lines.push_back(op_container(action, home)); + else if (target == "skills") lines.push_back(op_skills(action, home)); + } + lines.push_back(vault_line(home, store)); + return PyStr::join(lines, "\n"); +} + +int Installer::install_self() const { + std::string home = tai_home(); + io_.print(op_binary("upgrade", home)); + io_.print(op_hook("install", home)); + // bash-hook always goes in for back-compat; $SHELL additionally gets its matching hook. + std::string login_shell = PyFs::basename(PyFs::env("SHELL")); + std::string rc = "~/.bashrc"; + if (login_shell == "zsh") { + io_.print(op_zsh_hook("install", home)); + rc = "~/.zshrc"; + } else if (login_shell == "fish") { + io_.print(op_fish_hook("install", home)); + rc = "~/.config/fish/config.fish"; + } + io_.print(op_venv("install", home)); + io_.print(op_skills("install", home)); + io_.print("restart your shell or run: source " + rc); + return 0; +} + +int Installer::install_telegram() const { + std::string token = PyFs::env("TELEGRAM_BOT_TOKEN"); + if (!token.empty()) { + io_.print("using TELEGRAM_BOT_TOKEN from environment"); + } else { + auto got = io_.getpass("Telegram bot token: "); + if (!got) { + io_.print("\naborted"); + return 1; + } + token = PyStr::strip(*got); + } + if (token.empty()) { + io_.print("empty token, aborted"); + return 1; + } + Json me; + try { + me = telegram::Telegram(token, transport_, ctx_.telegram_api).call("getMe", Json::object()); + } catch (const telegram::TelegramNetError& e) { + io_.print("token rejected: " + short_err(e)); + return 1; + } catch (const misc::JsonError& e) { + io_.print("token rejected: " + short_err(e)); + return 1; + } + const Json* result = me.is_object() ? me.get("result") : nullptr; + const Json* uname = result && result->is_object() ? result->get("username") : nullptr; + io_.print("token ok for @" + (uname && uname->is_string() && !uname->as_string().empty() ? uname->as_string() : std::string("?"))); + std::string home = tai_home(); + PyFs::makedirs(home); + std::string env_path = telegram::TelegramBot::telegram_env_path(home); + PyFs::write_text(env_path, "TELEGRAM_BOT_TOKEN=" + token + "\n"); + try { + PyFs::chmod(env_path, 0600); + } catch (const OsError&) { + } + io_.print("wrote " + env_path); + try { + box_.ensure_box(); + } catch (const misc::BackendError& e) { + io_.print(std::string("container failed: ") + e.what()); + return 1; + } + std::string unit_folder = PyFs::join(PyFs::join(PyFs::join(home_dir(), ".config"), "systemd"), "user"); + PyFs::makedirs(unit_folder); + std::string unit_path = PyFs::join(unit_folder, "tai-telegram.service"); + PyFs::write_text(unit_path, PyStr::percent(InstallConsts::TELEGRAM_UNIT(), {service_exec(home, PyFs::abspath(ctx_.script_path)), env_path})); + io_.print("wrote " + unit_path); + io_.print("service: " + Systemd::activate_service("tai-telegram.service")); + return 0; +} + +int Installer::uninstall_telegram() const { + std::string unit_path = PyFs::join(PyFs::join(PyFs::join(PyFs::join(home_dir(), ".config"), "systemd"), "user"), "tai-telegram.service"); + if (PyFs::which("systemctl")) PyProc::run_host({"systemctl", "--user", "disable", "--now", "tai-telegram.service"}); + if (PyFs::exists(unit_path)) { + try { + PyFs::remove(unit_path); + io_.print("removed " + unit_path); + } catch (const OsError& e) { + io_.print("unit remove failed: " + short_err(e)); + } + } else { + io_.print("no service unit found"); + } + if (PyFs::which("systemctl")) PyProc::run_host({"systemctl", "--user", "daemon-reload"}); + auto engine = box_.container_engine(); + if (!engine) { + io_.print("no container engine found"); + return 0; + } + auto r = PyProc::run_host({*engine, "rm", "-f", InstallConsts::BOX_NAME}); + io_.print(std::string("container remove: ") + (r.ok ? "ok" : "not present")); + r = PyProc::run_host({*engine, "rmi", "-f", InstallConsts::BOX_IMAGE}); + io_.print(std::string("image purge: ") + (r.ok ? "ok" : "not present")); + return 0; +} + +std::string Installer::run_install(const Json& args, const std::string& home, misc::SqliteDb* store, + const std::function& ask_approval, const std::function& refresh_skills) const { + std::string action = PyStr::strip(skills::SkillTool::arg_text(args, "action")); + for (auto& c : action) { + if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + } + if (action != "status" && action != "install" && action != "upgrade" && action != "reinstall" && action != "uninstall") { + return "error: action must be status, install, upgrade, reinstall, or uninstall"; + } + std::vector targets; + const Json* t = args.get("targets"); + bool given = t && !t->is_null() && !((t->is_array() || t->is_object()) && t->size() == 0) && !(t->is_string() && t->as_string().empty()); + if (!given) { + targets = InstallConsts::INSTALL_TARGETS(); + } else { + bool bad = !t->is_array(); + if (!bad) { + for (const auto& item : t->items()) { + const auto& valid = InstallConsts::INSTALL_TARGETS(); + if (!item.is_string() || std::find(valid.begin(), valid.end(), item.as_string()) == valid.end()) bad = true; + else targets.push_back(item.as_string()); + } + } + if (bad) return "error: unknown target, valid: " + PyStr::join(InstallConsts::INSTALL_TARGETS(), ", "); + } + if (action != "status" && !ask_approval("install " + action + ": " + PyStr::join(targets, ", ") + " (vault always kept)")) return "denied by user"; + std::string report = install_report(action, targets, home, store); + if (action != "status" && std::find(targets.begin(), targets.end(), "skills") != targets.end() && refresh_skills) refresh_skills(); + return report; +} +} // namespace detz::install diff --git a/taicpp/src/install/install.h b/taicpp/src/install/install.h new file mode 100644 index 0000000..6770839 --- /dev/null +++ b/taicpp/src/install/install.h @@ -0,0 +1,155 @@ +#pragma once +// Port of the tai.py install flow: Install probes, venv/service/rc-block helpers, op_* targets, +// status_lines, vault_line, install_report, install_self, install_telegram, uninstall_telegram and +// the install tool backend. Spec: docs/install.md. +#include +#include +#include +#include +#include +#include +#include "bundled.h" +#include "miscjson.h" +#include "miscpy.h" +#include "sqlitedb.h" +#include "telegramtransport.h" +namespace detz::install { + +// ---- rc hook blocks (_rc_block_present/_remove/_upsert) +class RcBlocks { + public: + static bool present(const std::string& path, const std::string& block); + static bool remove(const std::string& path, const std::string& mark_begin, const std::string& mark_end); + // throws misc::OsError like the Python open() calls do + static bool upsert(const std::string& path, const std::string& mark_begin, const std::string& mark_end, const std::string& block); + // re.sub(escape(begin) + ".*?" + escape(end) + "\n?", "", raw, flags=DOTALL) + static std::string strip_blocks(const std::string& raw, const std::string& mark_begin, const std::string& mark_end); +}; + +// ---- systemd user units +class Systemd { + public: + static std::string unit_dir(const std::string& home); + static std::pair service_state(const std::string& name); // (active, enabled) + static std::string activate_service(const std::string& name); + static std::pair remove_service_unit(const std::string& home, const std::string& name); + static std::string getuser(); // getpass.getuser +}; + +// class Install (probes) plus the venv helpers +class Install { + public: + static bool shell_present(const std::string& shell); + static bool venv_available(const std::string& python = "python3"); + static std::pair ensure_venv(const std::string& home, const std::string& python = "python3"); + static std::string venv_dir(const std::string& home); + static std::optional venv_python(const std::string& home); + static std::optional script_version(const std::string& path); // VERSION_RE over the file + static std::string shell_rc_path(const std::string& shell); +}; + +// Container engine probes (Sandbox.container_engine, box_state, box_venv_state, ensure_box). Owned by shell/sandbox. +class ContainerPort { + public: + virtual ~ContainerPort() = default; + virtual std::optional container_engine() = 0; + virtual std::string box_state(const std::string& engine) = 0; + virtual std::string box_venv_state(const std::string& engine) = 0; + virtual std::string ensure_box() = 0; // throws misc::BackendError +}; + +// print()/getpass() of the install flows +class InstallIo { + public: + virtual ~InstallIo() = default; + virtual void print(const std::string& line) = 0; + // nullopt on EOFError/KeyboardInterrupt + virtual std::optional getpass(const std::string& prompt) = 0; +}; +class StdIo : public InstallIo { + public: + void print(const std::string& line) override; + std::optional getpass(const std::string& prompt) override; +}; + +struct InstallContext { + std::string version = "1.44.2"; // VERSION + std::string script_path; // os.path.abspath(__file__): the file --install copies and units execute + std::string python = "python3"; // sys.executable, used for venv creation + // service_exec prefixes the venv python before the script. True is the literal tai.py behavior; a native + // binary sets it false so the unit runs the binary itself. + bool script_needs_interpreter = true; + // script_version of the installed copy; default scans for the VERSION line like VERSION_RE + std::function(const std::string&)> version_reader; + // Bot API base for getMe in the install flows (tests point it at a local server) + std::string telegram_api = "https://api.telegram.org/bot"; +}; + +class Installer { + public: + Installer(InstallContext ctx, ContainerPort& box, InstallIo& io, telegram::TelegramTransport* transport = nullptr); + // helpers whose Python form takes the module-level state + std::string service_exec(const std::string& home, const std::string& script, const std::vector& args = {}) const; + std::optional script_version(const std::string& path) const; + + std::string op_binary(const std::string& mode, const std::string& home) const; + std::string op_hook(const std::string& mode, const std::string& home) const; + std::string op_plexer_hook(const std::string& mode, const std::string& home) const; + std::string op_shell_hook(const std::string& mode, const std::string& shell, const std::string& label, + const std::function& upsert_fn, const std::function& remove_fn, + const std::string& source_hint) const; + std::string op_zsh_hook(const std::string& mode, const std::string& home) const; + std::string op_fish_hook(const std::string& mode, const std::string& home) const; + std::string op_zsh_plexer_hook(const std::string& mode, const std::string& home) const; + std::string op_fish_plexer_hook(const std::string& mode, const std::string& home) const; + std::string op_venv(const std::string& mode, const std::string& home) const; + std::string op_service(const std::string& mode, const std::string& home, const std::string& name, const std::string& label, const std::string& body) const; + std::string op_scheduler_service(const std::string& mode, const std::string& home) const; + std::string op_telegram_service(const std::string& mode, const std::string& home) const; + std::string op_container(const std::string& mode, const std::string& home) const; + std::string op_skills(const std::string& mode, const std::string& home) const; + + // store may be null (vault_line then only reports the file) + std::vector status_lines(const std::string& home, misc::SqliteDb* store = nullptr) const; + static std::string vault_line(const std::string& home, misc::SqliteDb* store = nullptr); + std::string install_report(const std::string& action, const std::vector& targets, const std::string& home, misc::SqliteDb* store = nullptr) const; + int install_self() const; + int install_telegram() const; + int uninstall_telegram() const; + + // Tools.run_install; ask_approval/refresh_skills are the app's + std::string run_install(const misc::Json& args, const std::string& home, misc::SqliteDb* store, + const std::function& ask_approval, const std::function& refresh_skills) const; + + // hook_present & friends (module-level wrappers over RcBlocks with the tai constants) + static bool hook_present(const std::string& path); + static bool remove_bashrc_block(const std::string& path); + static bool upsert_bashrc_block(const std::string& path); + static bool zsh_hook_present(const std::string& path); + static bool remove_zsh_hook_block(const std::string& path); + static bool upsert_zsh_hook_block(const std::string& path); + static bool fish_hook_present(const std::string& path); + static bool remove_fish_hook_block(const std::string& path); + static bool upsert_fish_hook_block(const std::string& path); + static bool plexer_hook_present(const std::string& path); + static bool remove_plexer_bashrc_block(const std::string& path); + static bool upsert_plexer_bashrc_block(const std::string& path); + static bool zsh_plexer_hook_present(const std::string& path); + static bool remove_zsh_plexer_block(const std::string& path); + static bool upsert_zsh_plexer_block(const std::string& path); + static bool fish_plexer_hook_present(const std::string& path); + static bool remove_fish_plexer_block(const std::string& path); + static bool upsert_fish_plexer_block(const std::string& path); + + private: + InstallContext ctx_; + ContainerPort& box_; + InstallIo& io_; + telegram::TelegramTransport* transport_; + std::unique_ptr own_transport_; + skills::BundledSkills skills_; + std::string shell_hook_status_line(const std::string& label, const std::string& shell, const std::function& present_fn) const; + static std::string tai_home(); +}; + +} // namespace detz::install diff --git a/taicpp/src/install/installconsts.cpp b/taicpp/src/install/installconsts.cpp new file mode 100644 index 0000000..43e68f4 --- /dev/null +++ b/taicpp/src/install/installconsts.cpp @@ -0,0 +1,112 @@ +#include "installconsts.h" +namespace detz::install { +const std::string& InstallConsts::BASHRC_MARK_BEGIN() { + static const std::string v = "# tai command-not-found hook - start"; + return v; +} +const std::string& InstallConsts::BASHRC_MARK_END() { + static const std::string v = "# tai command-not-found hook - end"; + return v; +} +const std::string& InstallConsts::BASHRC_BLOCK() { + static const std::string v = BASHRC_MARK_BEGIN() + "\n" + "alias tai=\"$HOME/.local/bin/tai.py\"\n" + "command_not_found_handle() {\n" + " \"$HOME/.local/bin/tai.py\" \"$@\"\n" + " return $?\n" + "}\n" + BASHRC_MARK_END() + "\n"; + return v; +} +const std::string& InstallConsts::ZSH_NOTFOUND_BLOCK() { + static const std::string v = BASHRC_MARK_BEGIN() + "\n" + "alias tai=\"$HOME/.local/bin/tai.py\"\n" + "command_not_found_handler() {\n" + " \"$HOME/.local/bin/tai.py\" \"$@\"\n" + " return $?\n" + "}\n" + BASHRC_MARK_END() + "\n"; + return v; +} +const std::string& InstallConsts::FISH_NOTFOUND_BLOCK() { + static const std::string v = BASHRC_MARK_BEGIN() + "\n" + "function tai\n" + " \"$HOME/.local/bin/tai.py\" $argv\n" + "end\n" + "function fish_command_not_found\n" + " \"$HOME/.local/bin/tai.py\" $argv\n" + "end\n" + BASHRC_MARK_END() + "\n"; + return v; +} +const std::string& InstallConsts::PLEXER_BASHRC_MARK_BEGIN() { + static const std::string v = "# tai plexer hook - start"; + return v; +} +const std::string& InstallConsts::PLEXER_BASHRC_MARK_END() { + static const std::string v = "# tai plexer hook - end"; + return v; +} +const std::string& InstallConsts::PLEXER_BASHRC_BLOCK() { + static const std::string v = PLEXER_BASHRC_MARK_BEGIN() + "\n" + "if [ -z \"$TAI_PLEXER_ACTIVE\" ] && [ -z \"$TMUX\" ] && [ ! -e \"$HOME/.tai-no-plexer\" ] && case \"$-\" in *i*) true;; *) false;; esac; then\n" + " if [ -t 0 ] && [ -t 1 ]; then\n" + " exec \"$HOME/.local/bin/tai.py\" --plexer-wrap\n" + " fi\n" + "fi\n" + + PLEXER_BASHRC_MARK_END() + "\n"; + return v; +} +const std::string& InstallConsts::FISH_PLEXER_BLOCK() { + static const std::string v = PLEXER_BASHRC_MARK_BEGIN() + "\n" + "if status is-interactive\n" + "and test -z \"$TAI_PLEXER_ACTIVE\"\n" + "and test -z \"$TMUX\"\n" + "and not test -e \"$HOME/.tai-no-plexer\"\n" + "and isatty stdin\n" + "and isatty stdout\n" + " exec \"$HOME/.local/bin/tai.py\" --plexer-wrap\n" + "end\n" + + PLEXER_BASHRC_MARK_END() + "\n"; + return v; +} +const std::string& InstallConsts::TELEGRAM_UNIT() { + static const std::string v = + "[Unit]\n" + "Description=tai telegram bot\n" + "After=network-online.target\n" + "Wants=network-online.target\n" + "\n" + "[Service]\n" + "Type=simple\n" + "WorkingDirectory=%%h\n" + "ExecStart=%s --telegram\n" + "EnvironmentFile=%s\n" + "Restart=on-failure\n" + "RestartSec=10\n" + "\n" + "[Install]\n" + "WantedBy=default.target\n"; + return v; +} +const std::string& InstallConsts::SCHEDULER_UNIT() { + static const std::string v = + "[Unit]\n" + "Description=tai scheduler\n" + "After=network-online.target\n" + "Wants=network-online.target\n" + "\n" + "[Service]\n" + "Type=simple\n" + "WorkingDirectory=%%h\n" + "ExecStart=%s --scheduler\n" + "Restart=on-failure\n" + "RestartSec=10\n" + "\n" + "[Install]\n" + "WantedBy=default.target\n"; + return v; +} +const std::vector& InstallConsts::INSTALL_TARGETS() { + static const std::vector v = {"binary", "bash-hook", "zsh-hook", "fish-hook", "plexer-hook", "zsh-plexer-hook", + "fish-plexer-hook", "venv", "scheduler-service", "telegram-service", "container", "skills"}; + return v; +} +} // namespace detz::install diff --git a/taicpp/src/install/installconsts.h b/taicpp/src/install/installconsts.h new file mode 100644 index 0000000..1f2227e --- /dev/null +++ b/taicpp/src/install/installconsts.h @@ -0,0 +1,24 @@ +#pragma once +// Install constants of tai.py: rc hook blocks, systemd unit templates, INSTALL_TARGETS, box names. +#include +#include +namespace detz::install { +class InstallConsts { + public: + static const std::string& BASHRC_MARK_BEGIN(); + static const std::string& BASHRC_MARK_END(); + static const std::string& BASHRC_BLOCK(); + static const std::string& ZSH_NOTFOUND_BLOCK(); + static const std::string& FISH_NOTFOUND_BLOCK(); + static const std::string& PLEXER_BASHRC_MARK_BEGIN(); + static const std::string& PLEXER_BASHRC_MARK_END(); + static const std::string& PLEXER_BASHRC_BLOCK(); + static const std::string& FISH_PLEXER_BLOCK(); + // Python %-templates: "%s" slots and "%%h" exactly as written in tai.py + static const std::string& TELEGRAM_UNIT(); + static const std::string& SCHEDULER_UNIT(); + static const std::vector& INSTALL_TARGETS(); + static constexpr const char* BOX_IMAGE = "tai-box:latest"; + static constexpr const char* BOX_NAME = "tai-box"; +}; +} // namespace detz::install diff --git a/taicpp/src/install/release.cpp b/taicpp/src/install/release.cpp new file mode 100644 index 0000000..b124b6f --- /dev/null +++ b/taicpp/src/install/release.cpp @@ -0,0 +1,260 @@ +#include "release.h" +#include +#include +#include +#include +#include "miscdatetime.h" +#include "miscpy.h" +#include "miscsha256.h" +#include "skilltool.h" +#include "sqlitedb.h" +namespace detz::install { +using misc::OsError; +using misc::PyFs; +using misc::PyStr; + +std::string SelfRelease::next_version(const std::string& current, const std::string& part) { + int parts[3] = {0, 0, 0}; + std::size_t i = 0; + for (int k = 0; k < 3; ++k) { + std::size_t dot = current.find('.', i); + std::string piece = current.substr(i, dot == std::string::npos ? std::string::npos : dot - i); + if (!PyStr::isdigit_ascii(piece)) throw std::invalid_argument("invalid literal for int() with base 10: " + PyStr::repr(piece)); + parts[k] = std::stoi(piece); + if (dot == std::string::npos) { + if (k < 2) throw std::invalid_argument("not enough values to unpack (expected 3, got " + std::to_string(k + 1) + ")"); + break; + } + i = dot + 1; + if (k == 2) throw std::invalid_argument("too many values to unpack (expected 3)"); + } + if (part == "major") return std::to_string(parts[0] + 1) + ".0.0"; + if (part == "minor") return std::to_string(parts[0]) + "." + std::to_string(parts[1] + 1) + ".0"; + if (part == "patch") return std::to_string(parts[0]) + "." + std::to_string(parts[1]) + "." + std::to_string(parts[2] + 1); + throw std::invalid_argument("part must be major, minor, or patch"); +} + +std::optional SelfRelease::find_version(const std::string& text) { + static const std::string head = "VERSION = \""; + std::size_t start = 0; + while (start <= text.size()) { + std::size_t nl = text.find('\n', start); + std::size_t end = nl == std::string::npos ? text.size() : nl; + std::string line = text.substr(start, end - start); + if (PyStr::starts_with(line, head) && line.size() > head.size() && line.back() == '"') { + std::string inner = line.substr(head.size(), line.size() - head.size() - 1); + std::vector pieces; + std::size_t i = 0; + while (true) { + std::size_t dot = inner.find('.', i); + pieces.push_back(inner.substr(i, dot == std::string::npos ? std::string::npos : dot - i)); + if (dot == std::string::npos) break; + i = dot + 1; + } + if (pieces.size() == 3 && std::all_of(pieces.begin(), pieces.end(), [](const std::string& p) { return PyStr::isdigit_ascii(p); })) { + return VersionMatch{inner, start, end}; + } + } + if (nl == std::string::npos) break; + start = nl + 1; + } + return std::nullopt; +} + +std::pair SelfRelease::release_bump(const std::string& script_file, const std::string& part) { + std::string text = PyFs::read_text(script_file); + auto match = find_version(text); + if (!match) throw std::invalid_argument("no VERSION line in " + script_file); + std::string old_v = match->version; + std::string new_v = next_version(old_v, part); + atomic_write(script_file, text.substr(0, match->start) + "VERSION = \"" + new_v + "\"" + text.substr(match->end)); + return {old_v, new_v}; +} + +void SelfRelease::atomic_write(const std::string& path, const std::string& content) { PyFs::atomic_write(path, content); } + +void SelfRelease::sealed_write(const std::string& path, const std::string& content) { + atomic_write(path, content); + try { + PyFs::chmod(path, 0600); + } catch (const OsError&) { + } +} + +void SelfRelease::quarantine(const std::string& path) { + if (PyFs::exists(path)) { + std::string dest = path + ".corrupt-" + std::to_string(static_cast(std::time(nullptr))); + ::rename(path.c_str(), dest.c_str()); // errors swallowed like the OSError pass + } +} + +std::string SelfRelease::backups_dir(const std::string& home) { return PyFs::join(home, "backups"); } + +int SelfRelease::prune_backups(const std::string& folder) { + std::vector> names; + try { + for (const auto& entry : PyFs::listdir_sorted(folder)) { + if (PyStr::starts_with(entry, "tai-") && PyStr::ends_with(entry, ".py")) { + struct stat st; + if (::stat(PyFs::join(folder, entry).c_str(), &st) != 0) throw OsError(errno, entry); + names.emplace_back(static_cast(st.st_mtim.tv_sec) * 1000000000LL + st.st_mtim.tv_nsec, entry); + } + } + } catch (const OsError&) { + return 0; + } + std::stable_sort(names.begin(), names.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + int pruned = 0; + std::size_t first = 0; + while (names.size() - first > static_cast(BACKUP_KEEP)) { + try { + PyFs::remove(PyFs::join(folder, names[first].second)); + ++first; + ++pruned; + } catch (const OsError&) { + break; + } + } + return pruned; +} + +std::pair SelfRelease::snapshot_self(const std::string& script_file, const std::string& folder, const std::string& version) { + std::string data = PyFs::read_bytes(script_file); + std::string digest = misc::Sha256::hex_prefix(data, 8); + std::string iso = misc::DateTime::now_utc().isoformat(true); // YYYY-MM-DDTHH:MM:SS+00:00 + std::string stamp = iso.substr(0, 4) + iso.substr(5, 2) + iso.substr(8, 2) + "T" + iso.substr(11, 2) + iso.substr(14, 2) + iso.substr(17, 2) + "Z"; + std::string name = "tai-" + version + "-" + stamp + "-" + digest + ".py"; + PyFs::makedirs(folder); + std::string dest = PyFs::join(folder, name); + PyFs::write_bytes(dest, data); + prune_backups(folder); + return {dest, digest}; +} + +bool SelfRelease::backup_has_digest(const std::string& folder, const std::string& digest) { + try { + for (const auto& entry : PyFs::listdir_sorted(folder)) { + if (PyStr::ends_with(entry, "-" + digest + ".py")) return true; + } + return false; + } catch (const OsError&) { + return false; + } +} + +std::optional SelfRelease::ensure_self_backup(const std::string& home, AuditPort* audit, FileRecordPort* records, + const std::string& script_file, const std::string& version, const std::string& actor) { + std::string folder = backups_dir(home); + std::string digest = misc::Sha256::hex_prefix(PyFs::read_bytes(script_file), 8); + if (backup_has_digest(folder, digest)) return std::nullopt; + auto snap = snapshot_self(script_file, folder, version); + try { + if (audit) { + audit->audit_event(actor, "snapshot", script_file, "backed up as " + PyFs::basename(snap.first), std::nullopt, std::nullopt, {"snapshot", "v" + version}); + CappedRead cr = read_capped(script_file, FILE_RECORD_MAX); + if (cr.text && records) records->upsert_file_record(script_file, *cr.text, cr.size); + } + } catch (const misc::SqliteError&) { + } catch (const OsError&) { + } + return snap.first; +} + +SelfRelease::ReleaseResult SelfRelease::do_release(AuditPort& store, const std::string& script_file, const std::string& folder, const std::string& part, + const std::string& message, const std::string& actor) { + if (PyStr::strip(message).empty()) throw std::invalid_argument("release message is required"); + auto bumped = release_bump(script_file, part); + auto snap = snapshot_self(script_file, folder, bumped.second); + int64_t row_id = store.audit_event(actor, "release", script_file, PyStr::prefix(PyStr::strip(message), 500), bumped.first, bumped.second, + {"release", "v" + bumped.second}); + return ReleaseResult{bumped.first, bumped.second, snap.first, row_id}; +} + +std::string SelfRelease::trunc_marker(int64_t true_bytes) { return "\n...[truncated, full size " + std::to_string(true_bytes) + " bytes]..."; } + +std::pair, std::optional> SelfRelease::capped_text(const std::optional& text) { + if (!text) return {std::nullopt, std::nullopt}; + int64_t bytes = static_cast(text->size()); + if (PyStr::len(*text) <= static_cast(AUDIT_MAX_CHARS)) return {*text, bytes}; + return {PyStr::prefix(*text, AUDIT_MAX_CHARS) + trunc_marker(bytes), bytes}; +} + +// Decodes UTF-8 replacing each invalid byte with U+FFFD (open(..., errors="replace")) +static std::string decode_replace(const std::string& raw) { + std::string out; + for (std::size_t i = 0; i < raw.size();) { + unsigned char c = static_cast(raw[i]); + int n = c < 0x80 ? 0 : (c >= 0xC2 && c < 0xE0) ? 1 : (c >= 0xE0 && c < 0xF0) ? 2 : (c >= 0xF0 && c < 0xF5) ? 3 : -1; + bool ok = n >= 0 && i + static_cast(n) < raw.size() + 0; + if (n > 0 && ok) { + for (int k = 1; k <= n; ++k) { + if ((static_cast(raw[i + static_cast(k)]) & 0xC0) != 0x80) { ok = false; break; } + } + } + if (n < 0 || !ok) { + out += "\xEF\xBF\xBD"; + ++i; + continue; + } + out.append(raw, i, static_cast(n) + 1); + i += static_cast(n) + 1; + } + return out; +} + +CappedRead SelfRelease::read_capped(const std::string& path, std::size_t cap) { + struct stat st; + if (::stat(path.c_str(), &st) != 0) return {}; + std::string raw; + try { + raw = PyFs::read_bytes(path); + } catch (const OsError&) { + return {}; + } + // universal newlines then replace-decoding, first cap+1 characters + std::string norm; + for (std::size_t i = 0; i < raw.size(); ++i) { + if (raw[i] == '\r') { + norm += '\n'; + if (i + 1 < raw.size() && raw[i + 1] == '\n') ++i; + } else { + norm += raw[i]; + } + } + std::string text = PyStr::prefix(decode_replace(norm), cap + 1); + if (PyStr::len(text) > cap) return {PyStr::prefix(text, cap), static_cast(st.st_size)}; + return {text, std::nullopt}; +} + +std::string ReleaseTool::run_release(const misc::Json& args) { + std::string part = PyStr::strip(skills::SkillTool::arg_text(args, "part")); + for (auto& c : part) { + if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + } + std::string message = PyStr::strip(skills::SkillTool::arg_text(args, "message")); + if (part != "major" && part != "minor" && part != "patch") return "error: part must be major, minor, or patch"; + if (message.empty()) return "error: release message is required"; + std::string script_file = host_.script_file(); + std::string upcoming; + try { + auto match = SelfRelease::find_version(PyFs::read_text(script_file)); + if (!match) return "error: no VERSION line in running script"; + upcoming = SelfRelease::next_version(match->version, part); + } catch (const OsError& e) { + return "error: " + PyStr::short_error(e.what()); + } catch (const std::invalid_argument& e) { + return "error: " + PyStr::short_error(e.what()); + } + if (!host_.ask_approval("release " + upcoming + " (" + part + "): " + PyStr::prefix(message, 120))) return "denied by user"; + try { + auto r = SelfRelease::do_release(host_.store(), script_file, SelfRelease::backups_dir(host_.home()), part, message, host_.audit_actor()); + return "released " + r.old_version + " -> " + r.new_version + ", backup " + PyFs::basename(r.dest) + ", audit #" + std::to_string(r.row_id); + } catch (const OsError& e) { + return "error: " + PyStr::short_error(e.what()); + } catch (const std::invalid_argument& e) { + return "error: " + PyStr::short_error(e.what()); + } catch (const misc::SqliteError& e) { + return "error: " + PyStr::short_error(e.what()); + } +} +} // namespace detz::install diff --git a/taicpp/src/install/release.h b/taicpp/src/install/release.h new file mode 100644 index 0000000..c3a28bb --- /dev/null +++ b/taicpp/src/install/release.h @@ -0,0 +1,82 @@ +#pragma once +// Port of the self-release and backup helpers of tai.py: next_version, release_bump, atomic_write, +// sealed_write, quarantine, backups_dir, prune_backups, snapshot_self, backup_has_digest, +// ensure_self_backup, do_release, read_capped, capped_text, trunc_marker, plus the run_release tool. +#include +#include +#include +#include +#include +#include "miscjson.h" +namespace detz::install { + +// The slice of Store the release flow uses +class AuditPort { + public: + virtual ~AuditPort() = default; + virtual int64_t audit_event(const std::string& actor, const std::string& action, const std::string& path, const std::string& message, + const std::optional& old_text, const std::optional& new_text, + const std::vector& tags) = 0; +}; + +// Store.upsert_file_record(path, content, true_size): the records table belongs to the store module +class FileRecordPort { + public: + virtual ~FileRecordPort() = default; + virtual void upsert_file_record(const std::string& path, const std::string& content, const std::optional& true_size) = 0; +}; + +struct CappedRead { std::optional text; std::optional size; }; + +class SelfRelease { + public: + static constexpr int BACKUP_KEEP = 10; + static constexpr int AUDIT_MAX_CHARS = 20000; + static constexpr int FILE_RECORD_MAX = 50000; + + // Throws std::invalid_argument("part must be major, minor, or patch") + static std::string next_version(const std::string& current, const std::string& part); + // VERSION_RE.search: group string "a.b.c" and the byte span of the whole line, nullopt without a match + struct VersionMatch { std::string version; std::size_t start, end; }; + static std::optional find_version(const std::string& text); + // Throws std::invalid_argument("no VERSION line in ") or misc::OsError + static std::pair release_bump(const std::string& script_file, const std::string& part); + static void atomic_write(const std::string& path, const std::string& content); + static void sealed_write(const std::string& path, const std::string& content); + static void quarantine(const std::string& path); + static std::string backups_dir(const std::string& home); + static int prune_backups(const std::string& folder); + // (dest, digest) + static std::pair snapshot_self(const std::string& script_file, const std::string& folder, const std::string& version); + static bool backup_has_digest(const std::string& folder, const std::string& digest); + // nullopt when a backup with this digest already exists. A store error is swallowed like the Python try/except. + static std::optional ensure_self_backup(const std::string& home, AuditPort* audit, FileRecordPort* records, + const std::string& script_file, const std::string& version, const std::string& actor = "main"); + struct ReleaseResult { std::string old_version, new_version, dest; int64_t row_id; }; + static ReleaseResult do_release(AuditPort& store, const std::string& script_file, const std::string& folder, const std::string& part, + const std::string& message, const std::string& actor = "main"); + + static std::pair, std::optional> capped_text(const std::optional& text); + static std::string trunc_marker(int64_t true_bytes); + static CappedRead read_capped(const std::string& path, std::size_t cap = AUDIT_MAX_CHARS); +}; + +// Tools.run_release +class ReleaseHost { + public: + virtual ~ReleaseHost() = default; + virtual bool ask_approval(const std::string& what) = 0; + virtual std::string home() const = 0; + virtual std::string audit_actor() const = 0; + virtual AuditPort& store() = 0; + virtual std::string script_file() const = 0; // os.path.abspath(__file__) +}; +class ReleaseTool { + public: + explicit ReleaseTool(ReleaseHost& host) : host_(host) {} + std::string run_release(const misc::Json& args); + private: + ReleaseHost& host_; +}; + +} // namespace detz::install diff --git a/taicpp/src/rag/chunker.cpp b/taicpp/src/rag/chunker.cpp new file mode 100644 index 0000000..a07e21f --- /dev/null +++ b/taicpp/src/rag/chunker.cpp @@ -0,0 +1,81 @@ +#include "chunker.h" +#include +#include "miscdatetime.h" +#include "miscpy.h" +namespace detz::rag { +using misc::DbValue; +using misc::PyStr; + +std::vector Chunker::chunk_lines(const std::string& text, int max_lines, int overlap_lines, int max_chars) { + std::vector lines = PyStr::splitlines(text); + std::size_t n = lines.size(); + std::vector chunks; + if (n == 0) return chunks; + std::size_t start = 0; + while (start < n) { + std::size_t end = start; + long size = 0; + int count = 0; + while (end < n && count < max_lines && size < max_chars) { + size += static_cast(PyStr::len(lines[end])) + 1; + ++count; + ++end; + } + std::vector window(lines.begin() + static_cast(start), lines.begin() + static_cast(end)); + std::string piece = PyStr::strip(PyStr::join(window, "\n")); + if (!piece.empty()) chunks.push_back(piece); + if (end >= n) break; + long next = static_cast(end) - overlap_lines; + start = std::max(start + 1, static_cast(std::max(0L, next))); + } + return chunks; +} + +std::string Chunker::collapse_repeated_lines(const std::string& text, int threshold) { + std::vector lines = PyStr::splitlines(text); + std::vector out; + std::size_t i = 0, n = lines.size(); + while (i < n) { + std::size_t j = i; + while (j < n && lines[j] == lines[i]) ++j; + std::size_t run = j - i; + if (run >= static_cast(threshold)) { + out.push_back(lines[i]); + out.push_back("... (repeated " + std::to_string(run - 1) + " more times) ..."); + } else { + out.insert(out.end(), lines.begin() + static_cast(i), lines.begin() + static_cast(j)); + } + i = j; + } + return PyStr::join(out, "\n"); +} + +void ChunkStore::add_chunks(const std::string& parent_kind, const std::string& parent_id, const std::vector& texts, const std::string& profile_in) { + std::string profile = scope(profile_in); + db_.execute("DELETE FROM chunks WHERE parent_kind = ? AND parent_id = ? AND profile = ?", {parent_kind, parent_id, profile}); + std::string stamp = misc::DateTime::now_utc().isoformat(); + int idx = 0; + for (const auto& text : texts) { + db_.execute("INSERT INTO chunks(parent_kind, parent_id, idx, text, profile, created) VALUES (?, ?, ?, ?, ?, ?)", + {parent_kind, parent_id, idx, text, profile, stamp}); + ++idx; + } +} + +std::map ChunkStore::get_chunks_by_ids(const std::vector& ids, const std::string& profile_in) { + std::map out; + if (ids.empty()) return out; + std::string profile = scope(profile_in); + std::string marks; + std::vector params = {profile}; + for (std::size_t i = 0; i < ids.size(); ++i) { + marks += i ? ", ?" : "?"; + params.emplace_back(ids[i]); + } + auto rows = db_.query("SELECT id, parent_kind, parent_id, idx, text FROM chunks WHERE profile = ? AND id IN (" + marks + ")", params); + for (const auto& r : rows) { + out[r[0].as_int()] = ChunkRow{r[1].as_text(), r[2].as_text(), r[3].as_int(), r[4].as_text()}; + } + return out; +} +} // namespace detz::rag diff --git a/taicpp/src/rag/chunker.h b/taicpp/src/rag/chunker.h new file mode 100644 index 0000000..492f5f4 --- /dev/null +++ b/taicpp/src/rag/chunker.h @@ -0,0 +1,38 @@ +#pragma once +// Port of tai.py chunk_lines and collapse_repeated_lines (the one RAG chunking rule), plus the +// chunks table writes of Store.add_chunks / get_chunks_by_ids. Spec: docs/rag.md. +#include +#include +#include +#include +#include "sqlitedb.h" +namespace detz::rag { + +class Chunker { + public: + static constexpr int CHUNK_MAX_LINES = 60; + static constexpr int CHUNK_OVERLAP_LINES = 3; + static constexpr int CHUNK_MAX_CHARS = 4000; + // Bounded windows of whole lines with small overlap, never splitting mid-line (lengths in code points). + static std::vector chunk_lines(const std::string& text, int max_lines = CHUNK_MAX_LINES, + int overlap_lines = CHUNK_OVERLAP_LINES, int max_chars = CHUNK_MAX_CHARS); + // Runs of >= threshold identical lines collapse to one line plus a count note. + static std::string collapse_repeated_lines(const std::string& text, int threshold = 3); +}; + +struct ChunkRow { std::string parent_kind, parent_id; int64_t idx{0}; std::string text; }; + +class ChunkStore { + public: + ChunkStore(misc::SqliteDb& db, std::string profile) : db_(db), profile_(std::move(profile)) {} + // Replaces any prior chunks for this parent, so re-indexing is idempotent. + void add_chunks(const std::string& parent_kind, const std::string& parent_id, const std::vector& texts, const std::string& profile = ""); + std::map get_chunks_by_ids(const std::vector& ids, const std::string& profile = ""); + + private: + misc::SqliteDb& db_; + std::string profile_; + std::string scope(const std::string& p) const { return p.empty() ? profile_ : p; } +}; + +} // namespace detz::rag diff --git a/taicpp/src/rag/ftssearch.cpp b/taicpp/src/rag/ftssearch.cpp new file mode 100644 index 0000000..d123997 --- /dev/null +++ b/taicpp/src/rag/ftssearch.cpp @@ -0,0 +1,97 @@ +#include "ftssearch.h" +#include +#include "miscpy.h" +namespace detz::rag { +using misc::DbValue; +using misc::PyStr; + +std::optional> FtsQuery::fts_escape(const std::string& query) { + std::vector terms; + for (const auto& run : PyStr::alnum_runs(query)) terms.push_back("\"" + run + "\""); + if (terms.empty()) return std::nullopt; + return terms; +} +std::string FtsQuery::fts_match(const std::vector& terms) { + std::vector parts; + for (const auto& t : terms) parts.push_back("{title body} : " + t); + return PyStr::join(parts, " AND "); +} +std::string FtsQuery::fts_match_or(const std::vector& terms) { + std::vector parts; + for (const auto& t : terms) parts.push_back("{title body} : " + t); + return PyStr::join(parts, " OR "); +} +std::string FtsQuery::fts_match_prefix(const std::vector& terms) { + std::vector parts; + for (const auto& t : terms) { + std::string stripped = t; + std::size_t b = stripped.find_first_not_of('"'); + std::size_t e = stripped.find_last_not_of('"'); + stripped = b == std::string::npos ? "" : stripped.substr(b, e - b + 1); + parts.push_back("{title body} : " + stripped + "*"); + } + return PyStr::join(parts, " AND "); +} +std::vector FtsQuery::fts_match_variants(const std::vector& terms) { + std::vector variants = {fts_match(terms), fts_match_prefix(terms)}; + if (terms.size() > 1) variants.push_back(fts_match_or(terms)); + return variants; +} +std::string FtsQuery::fts_rank(const std::string& table) { + return "bm25(" + table + ", 1.0, 1.0, 1.0, " + FTS_TITLE_WEIGHT + ", " + FTS_BODY_WEIGHT + ")"; +} + +static FtsHit hit_of(const misc::DbRow& row) { + FtsHit h; + h.item = row[0]; + h.kind = row[1].as_text(); + h.sub = row[2].as_text(); + h.title = row[3].as_text(); + h.snippet = row[4].as_text(); + h.rank = row[5].as_real(); + return h; +} + +std::vector FtsSearch::fts_search(const std::string& query, const std::vector& kinds, const std::string& profile_in, int limit) { + std::string profile = profile_in.empty() ? profile_ : profile_in; + if (!fts_ok_) return {}; + auto terms = FtsQuery::fts_escape(query); + if (!terms) return {}; + int capped = std::max(1, std::min(limit, 100)); + std::string rank = FtsQuery::fts_rank("fts_docs"); + std::string sql = "SELECT item, kind, sub, title, snippet(fts_docs, 4, '[', ']', '...', 12), " + rank + + " FROM fts_docs WHERE fts_docs MATCH ? AND profile = ?"; + std::vector tail; + if (!kinds.empty()) { + std::string marks; + for (std::size_t i = 0; i < kinds.size(); ++i) marks += i ? ", ?" : "?"; + sql += " AND kind IN (" + marks + ")"; + for (const auto& k : kinds) tail.emplace_back(k); + } + if (sealed_) sql += " AND kind != 'event' AND kind != 'log'"; + sql += " ORDER BY " + rank + " LIMIT ?"; + for (const auto& match : FtsQuery::fts_match_variants(*terms)) { + std::vector rows; + try { + std::vector params = {match, profile}; + params.insert(params.end(), tail.begin(), tail.end()); + params.emplace_back(capped); + rows = db_.query(sql, params); + } catch (const misc::SqliteError&) { + rows.clear(); + } + std::vector hits; + for (const auto& r : rows) hits.push_back(hit_of(r)); + bool wants_events = kinds.empty() || std::find(kinds.begin(), kinds.end(), "event") != kinds.end(); + if (sealed_ && mem_ && wants_events) { + mem_->sync(); + auto extra = mem_->search(match, profile, capped); + hits.insert(hits.end(), extra.begin(), extra.end()); + std::stable_sort(hits.begin(), hits.end(), [](const FtsHit& a, const FtsHit& b) { return a.rank < b.rank; }); + if (hits.size() > static_cast(capped)) hits.resize(static_cast(capped)); + } + if (!hits.empty()) return hits; + } + return {}; +} +} // namespace detz::rag diff --git a/taicpp/src/rag/ftssearch.h b/taicpp/src/rag/ftssearch.h new file mode 100644 index 0000000..99567ef --- /dev/null +++ b/taicpp/src/rag/ftssearch.h @@ -0,0 +1,51 @@ +#pragma once +// Port of fts_escape, fts_match, fts_match_or, fts_match_prefix, fts_match_variants, fts_rank and +// Store.fts_search (lexical FTS5 + bm25, no vectors). Spec: docs/rag.md. +#include +#include +#include +#include "sqlitedb.h" +namespace detz::rag { + +class FtsQuery { + public: + static constexpr const char* FTS_TITLE_WEIGHT = "3.0"; + static constexpr const char* FTS_BODY_WEIGHT = "1.0"; + // nullopt where Python returns None (no alphanumeric terms) + static std::optional> fts_escape(const std::string& query); + static std::string fts_match(const std::vector& terms); + static std::string fts_match_or(const std::vector& terms); + static std::string fts_match_prefix(const std::vector& terms); + static std::vector fts_match_variants(const std::vector& terms); + static std::string fts_rank(const std::string& table); +}; + +struct FtsHit { + misc::DbValue item; // typed like sqlite returns it (records: text id, events: integer id, others: "kind:id") + std::string kind, sub, title, snippet; + double rank{0}; +}; + +// Store.memdb for sealed vaults: decrypted events live in an in-memory FTS table. Optional. +class MemEventsPort { + public: + virtual ~MemEventsPort() = default; + virtual void sync() = 0; // Store._sync_mem_events + virtual std::vector search(const std::string& match, const std::string& profile, int capped) = 0; +}; + +class FtsSearch { + public: + // fts_ok mirrors Store.fts_ok; sealed mirrors Store.seal.enabled + FtsSearch(misc::SqliteDb& db, std::string profile, bool fts_ok = true, bool sealed = false, MemEventsPort* mem = nullptr) + : db_(db), profile_(std::move(profile)), fts_ok_(fts_ok), sealed_(sealed), mem_(mem) {} + std::vector fts_search(const std::string& query, const std::vector& kinds = {}, const std::string& profile = "", int limit = 10); + + private: + misc::SqliteDb& db_; + std::string profile_; + bool fts_ok_, sealed_; + MemEventsPort* mem_; +}; + +} // namespace detz::rag diff --git a/taicpp/src/rag/ragtool.cpp b/taicpp/src/rag/ragtool.cpp new file mode 100644 index 0000000..28dfc28 --- /dev/null +++ b/taicpp/src/rag/ragtool.cpp @@ -0,0 +1,139 @@ +#include "ragtool.h" +#include +#include "miscpy.h" +#include "skillfile.h" +namespace detz::rag { +using misc::Json; +using misc::PyStr; + +static bool truthy(const Json* v) { + if (!v) return false; + switch (v->kind()) { + case Json::Kind::Null: return false; + case Json::Kind::Bool: return v->as_bool(); + case Json::Kind::Int: return v->as_int() != 0; + case Json::Kind::Float: return v->as_double() != 0; + case Json::Kind::String: return !v->as_string().empty(); + default: return v->size() > 0; + } +} + +// str.split(":", 1)[1]; false on IndexError +static bool after_colon(const std::string& s, std::string& out) { + std::size_t c = s.find(':'); + if (c == std::string::npos) return false; + out = s.substr(c + 1); + return true; +} +static bool parse_int(const std::string& s, int64_t& out) { + std::string t = PyStr::strip(s); + if (t.empty()) return false; + std::size_t i = 0; + bool neg = false; + if (t[i] == '+' || t[i] == '-') { neg = t[i] == '-'; ++i; } + if (i >= t.size()) return false; + int64_t v = 0; + for (; i < t.size(); ++i) { + if (t[i] < '0' || t[i] > '9') return false; + v = v * 10 + (t[i] - '0'); + } + out = neg ? -v : v; + return true; +} + +std::string RagTool::run_search(const Json& args) { + const Json* q = args.get("query"); + std::string query = PyStr::strip(truthy(q) ? skills::SkillFiles::py_str(*q) : ""); + if (query.empty()) return "error: empty query"; + std::vector kinds = {"record", "event", "audit", "chunk"}; + const Json* raw_kinds = args.get("kinds"); + if (truthy(raw_kinds)) { + bool bad = !raw_kinds->is_array(); + std::vector given; + if (!bad) { + for (const auto& item : raw_kinds->items()) { + if (!item.is_string() || (item.as_string() != "record" && item.as_string() != "event" && item.as_string() != "audit" && item.as_string() != "chunk")) bad = true; + else given.push_back(item.as_string()); + } + } + if (bad) return "error: kinds must be a list of record, event, audit, chunk"; + kinds = given; + } + int64_t limit = 10; + const Json* raw_limit = args.get("limit"); + if (truthy(raw_limit)) { + if (raw_limit->is_int()) limit = raw_limit->as_int(); + else if (raw_limit->is_float()) limit = static_cast(raw_limit->as_double()); + else if (raw_limit->is_bool()) limit = raw_limit->as_bool(); + else if (raw_limit->is_string()) { + if (!parse_int(raw_limit->as_string(), limit)) return "error: invalid limit"; + } else return "error: invalid limit"; + } + bool expand = truthy(args.get("expand")); + std::string profile = host_.active_profile(); + std::vector hits = host_.fts_search(query, kinds, profile, static_cast(limit)); + bool has_event = std::any_of(hits.begin(), hits.end(), [](const FtsHit& h) { return h.kind == "event"; }); + if (host_.sealed() && std::find(kinds.begin(), kinds.end(), "event") != kinds.end() && !has_event) { + for (const auto& ev : host_.events_like(profile, query, static_cast(limit))) { + FtsHit h; + h.kind = "event"; + h.item = misc::DbValue("event " + ev.ts); + h.sub = ev.kind; + h.title = PyStr::prefix(ev.text, 60); + h.snippet = PyStr::prefix(ev.text, 160); + h.rank = 99.0; + hits.push_back(h); + } + std::stable_sort(hits.begin(), hits.end(), [](const FtsHit& a, const FtsHit& b) { return a.rank < b.rank; }); + } + std::vector ids; + std::size_t shown = limit < 0 ? (hits.size() > static_cast(-limit) ? hits.size() - static_cast(-limit) : 0) + : std::min(hits.size(), static_cast(limit)); + for (std::size_t i = 0; i < shown; ++i) ids.push_back(hits[i].kind + ":" + hits[i].item.as_text()); + host_.log_event(profile, "tool", "search-query", + "q=" + PyStr::prefix(query, 200) + "\tlimit=" + std::to_string(limit) + "\thits=" + PyStr::prefix(PyStr::join(ids, ","), 1500)); + if (hits.empty()) return "no matches for '" + PyStr::prefix(query, 80) + "'"; + std::vector chunk_ids; + for (const auto& h : hits) { + std::string tail; + int64_t id; + if (h.kind == "chunk" && after_colon(h.item.as_text(), tail) && parse_int(tail, id)) chunk_ids.push_back(id); + } + std::map chunk_rows; + if (!chunk_ids.empty()) chunk_rows = host_.get_chunks_by_ids(chunk_ids, profile); + std::vector lines; + for (const auto& h : hits) { + std::string item = h.item.as_text(); + if (h.kind == "record") { + lines.push_back("[record/" + h.sub + "] " + item + " :: " + PyStr::prefix(h.title, 80) + " -- " + h.snippet); + if (expand) { + std::vector neighbors; + for (const auto& edge : host_.edges_for(item, profile)) { + if (neighbors.size() < 3) neighbors.push_back(PyStr::prefix(host_.node_title(edge.other, profile), 60) + " (" + edge.relation + ")"); + } + if (!neighbors.empty()) lines.push_back(" linked: " + PyStr::join(neighbors, "; ")); + } + } else if (h.kind == "event") { + lines.push_back("[event/" + h.sub + "] #" + item + " " + PyStr::prefix(h.title, 60) + " -- " + h.snippet); + } else if (h.kind == "chunk") { + std::string tail; + int64_t chunk_id = 0; + bool have = after_colon(item, tail) && parse_int(tail, chunk_id); + auto it = have ? chunk_rows.find(chunk_id) : chunk_rows.end(); + if (it == chunk_rows.end()) { + lines.push_back("[chunk/" + h.sub + "] " + PyStr::prefix(h.title, 80) + " -- " + h.snippet); + } else { + const ChunkRow& c = it->second; + std::string preview = PyStr::prefix(c.text, 500) + (PyStr::len(c.text) > 500 ? "..." : ""); + lines.push_back("[chunk/" + c.parent_kind + "] " + PyStr::prefix(c.parent_id, 60) + "#" + std::to_string(c.idx) + " (parent " + c.parent_id + + ", read full with record_read) -- " + preview); + } + } else { + std::string tail; + after_colon(item, tail); + lines.push_back("[audit/" + h.sub + "] #" + tail + " " + PyStr::prefix(h.title, 80) + " -- " + h.snippet); + } + } + return PyStr::join(lines, "\n"); +} +} // namespace detz::rag diff --git a/taicpp/src/rag/ragtool.h b/taicpp/src/rag/ragtool.h new file mode 100644 index 0000000..1bacf9c --- /dev/null +++ b/taicpp/src/rag/ragtool.h @@ -0,0 +1,36 @@ +#pragma once +// Port of Tools.run_search: one ranked full-text call over records, events, audit and chunks. +#include +#include +#include "chunker.h" +#include "ftssearch.h" +#include "miscjson.h" +namespace detz::rag { + +struct EventRow { std::string ts, role, kind, text; }; +struct GraphEdge { std::string other, relation; }; + +// What the Python Tools reaches through self.app.store +class RagHost { + public: + virtual ~RagHost() = default; + virtual std::string active_profile() const = 0; + virtual bool sealed() const = 0; // store.seal.enabled + virtual std::vector fts_search(const std::string& query, const std::vector& kinds, const std::string& profile, int limit) = 0; + virtual std::vector events_like(const std::string& profile, const std::string& query, int limit) = 0; + virtual void log_event(const std::string& profile, const std::string& role, const std::string& kind, const std::string& text) = 0; + virtual std::map get_chunks_by_ids(const std::vector& ids, const std::string& profile) = 0; + virtual std::vector edges_for(const std::string& item, const std::string& profile) = 0; // outgoing and incoming + virtual std::string node_title(const std::string& node, const std::string& profile) = 0; +}; + +class RagTool { + public: + explicit RagTool(RagHost& host) : host_(host) {} + std::string run_search(const misc::Json& args); + + private: + RagHost& host_; +}; + +} // namespace detz::rag diff --git a/taicpp/src/rag/retrievaleval.cpp b/taicpp/src/rag/retrievaleval.cpp new file mode 100644 index 0000000..159224f --- /dev/null +++ b/taicpp/src/rag/retrievaleval.cpp @@ -0,0 +1,283 @@ +#include "retrievaleval.h" +#include +#include +#include +#include +#include +#include +#include "miscpy.h" +namespace detz::rag { +using misc::DbValue; +using misc::PyStr; + +static double round4(double x) { + char buf[64]; + std::snprintf(buf, sizeof buf, "%.4f", x); + return std::strtod(buf, nullptr); +} + +std::optional RetrievalEval::rank_of(const DbValue& expected_id, const std::vector& ranked_ids) { + for (std::size_t i = 0; i < ranked_ids.size(); ++i) { + if (ranked_ids[i] == expected_id) return static_cast(i) + 1; + } + return std::nullopt; +} +std::optional RetrievalEval::best_rank(const std::vector& expected_ids, const std::vector& ranked_ids) { + std::optional best; + for (const auto& item : expected_ids) { + auto r = rank_of(item, ranked_ids); + if (r && (!best || *r < *best)) best = r; + } + return best; +} +double RetrievalEval::hit_at_k(std::optional rank, int k) { return rank && *rank <= k ? 1.0 : 0.0; } +double RetrievalEval::reciprocal_rank(std::optional rank) { return rank && *rank != 0 ? 1.0 / *rank : 0.0; } +double RetrievalEval::ndcg_at_k(std::optional rank, int k) { + if (!rank || *rank > k) return 0.0; + return 1.0 / std::log2(static_cast(*rank) + 1); +} +double RetrievalEval::context_precision(const std::vector& expected_ids, const std::vector& ranked_ids, int k) { + if (expected_ids.empty()) return 0.0; + int hits = 0; + double total = 0.0; + for (std::size_t pos = 1; pos <= ranked_ids.size() && pos <= static_cast(std::max(k, 0)); ++pos) { + const DbValue& item = ranked_ids[pos - 1]; + if (std::find(expected_ids.begin(), expected_ids.end(), item) != expected_ids.end()) { + ++hits; + total += static_cast(hits) / static_cast(pos); + } + } + return hits ? total / hits : 0.0; +} + +std::vector RetrievalEval::tokens(const std::string& text, int min_len) { + std::vector seen; + for (const auto& token : PyStr::alnum_runs(text)) { + if (static_cast(token.size()) >= min_len && std::find(seen.begin(), seen.end(), token) == seen.end()) seen.push_back(token); + } + return seen; +} + +int RetrievalEval::doc_freq(misc::SqliteDb& db, const std::string& profile, const std::string& token) { + try { + auto rows = db.query("SELECT COUNT(DISTINCT item) FROM fts_docs WHERE fts_docs MATCH ? AND profile = ? AND kind = 'record'", + {"{title body} : \"" + token + "\"", profile}); + return rows.empty() ? 0 : static_cast(rows[0][0].as_int()); + } catch (const misc::SqliteError&) { + return 0; + } +} + +std::vector RetrievalEval::goldens_from_vault(misc::SqliteDb& db, const std::string& profile, int max_records) { + auto rows = db.query("SELECT id, title, content FROM records WHERE profile = ? ORDER BY id LIMIT ?", {profile, max_records}); + std::vector goldens; + for (std::size_t pos = 0; pos < rows.size(); ++pos) { + const auto& row = rows[pos]; + DbValue record_id = row[0]; + std::string title = row[1].as_text(); + std::string content = row[2].as_text(); + std::vector words = tokens(title + "\n" + content); + if (words.empty()) continue; + std::vector> ranked; + for (std::size_t idx = 0; idx < words.size() && idx < 24; ++idx) { + ranked.emplace_back(doc_freq(db, profile, words[idx]), -static_cast(PyStr::len(words[idx])), static_cast(idx), words[idx]); + } + std::sort(ranked.begin(), ranked.end()); + std::vector rare; + for (const auto& r : ranked) rare.push_back(std::get<3>(r)); + goldens.push_back({rare[0], {record_id}, "exact", record_id}); + if (rare.size() >= 2) { + std::vector first(rare.begin(), rare.begin() + static_cast(std::min(3, rare.size()))); + goldens.push_back({PyStr::join(first, " "), {record_id}, "phrase", record_id}); + } + std::string longest = words[0]; + for (const auto& w : words) { + if (w.size() > longest.size()) longest = w; + } + std::size_t cut = std::max(EVAL_PREFIX_MIN, longest.size() / 2); + if (cut < longest.size()) goldens.push_back({longest.substr(0, cut), {record_id}, "prefix", record_id}); + if (rare.size() >= 2 && rows.size() > 1) { + const auto& other = rows[(pos + 1) % rows.size()]; + std::vector other_words = tokens(other[1].as_text() + "\n" + other[2].as_text()); + std::set own(words.begin(), words.end()); + std::vector foreign; + for (const auto& t : other_words) { + if (!own.count(t)) foreign.push_back(t); + } + if (!foreign.empty()) { + goldens.push_back({rare[0] + " " + rare[1] + " " + foreign[0] + "zz", {record_id}, "noisy", record_id}); + } + } + if (goldens.size() >= static_cast(EVAL_MAX_GOLDENS)) break; + } + if (goldens.size() > static_cast(EVAL_MAX_GOLDENS)) goldens.resize(static_cast(EVAL_MAX_GOLDENS)); + return goldens; +} + +EvalMean RetrievalEval::mean(const std::vector& rows) { + if (rows.empty()) return {}; + EvalMean m; + double h = 0, r = 0, n = 0, c = 0; + for (const auto& row : rows) { + h += row.hit; r += row.mrr; n += row.ndcg; c += row.cp; + } + double size = static_cast(rows.size()); + m.hit = round4(h / size); + m.mrr = round4(r / size); + m.ndcg = round4(n / size); + m.cp = round4(c / size); + return m; +} + +EvalResult RetrievalEval::score(FtsSearch& search, const std::vector& goldens, const std::string& profile, + const std::vector& kinds, int k, int fetch) { + struct Row { std::string query, category; std::optional rank; EvalMean m; }; + std::vector rows; + for (const auto& golden : goldens) { + std::vector hits; + try { + hits = search.fts_search(golden.query, kinds, profile, fetch); + } catch (const std::exception&) { + hits.clear(); + } + std::vector ranked; + for (const auto& h : hits) ranked.push_back(h.item); + auto rank = best_rank(golden.expected, ranked); + Row row; + row.query = golden.query; + row.category = golden.category; + row.rank = rank; + row.m.hit = hit_at_k(rank, k); + row.m.mrr = reciprocal_rank(rank); + row.m.ndcg = ndcg_at_k(rank, k); + row.m.cp = context_precision(golden.expected, ranked, k); + rows.push_back(row); + } + EvalResult result; + result.k = k; + result.goldens = static_cast(rows.size()); + std::vector all; + for (const auto& r : rows) all.push_back(r.m); + result.overall = mean(all); + std::vector>> grouped; + for (const auto& r : rows) { + auto it = std::find_if(grouped.begin(), grouped.end(), [&](const auto& g) { return g.first == r.category; }); + if (it == grouped.end()) { + grouped.push_back({r.category, {}}); + it = grouped.end() - 1; + } + it->second.push_back(r.m); + } + for (const auto& g : grouped) result.by_category.push_back({g.first, EvalCategory{static_cast(g.second.size()), mean(g.second)}}); + for (const auto& r : rows) { + if (!r.rank && result.misses.size() < 10) result.misses.push_back({r.query, r.category}); + } + return result; +} + +std::optional>> RetrievalEval::parse_search_event(const std::string& text) { + std::map parts; + std::size_t start = 0; + while (true) { + std::size_t tab = text.find('\t', start); + std::string part = text.substr(start, tab == std::string::npos ? std::string::npos : tab - start); + std::size_t eq = part.find('='); + if (eq != std::string::npos) parts[part.substr(0, eq)] = part.substr(eq + 1); + if (tab == std::string::npos) break; + start = tab + 1; + } + std::string query = PyStr::strip(parts.count("q") ? parts["q"] : ""); + std::string raw = PyStr::strip(parts.count("hits") ? parts["hits"] : ""); + std::vector hits; + std::size_t s = 0; + while (s <= raw.size()) { + std::size_t comma = raw.find(',', s); + std::string piece = raw.substr(s, comma == std::string::npos ? std::string::npos : comma - s); + if (!piece.empty()) { + if (PyStr::starts_with(piece, "record:")) hits.push_back(piece.substr(piece.find(':') + 1)); + else hits.push_back(piece); + } + if (comma == std::string::npos) break; + s = comma + 1; + } + if (query.empty()) return std::nullopt; + return std::make_pair(query, hits); +} + +ImplicitPairs RetrievalEval::implicit_pairs(misc::SqliteDb& db, const std::string& profile, bool sealed, int limit) { + std::string text_col = sealed ? "tai_dec(text)" : "text"; + std::vector rows; + try { + rows = db.query("SELECT role, kind, " + text_col + " FROM events WHERE profile = ? ORDER BY id DESC LIMIT ?", {profile, limit}); + } catch (const misc::SqliteError&) { + rows.clear(); + } + std::reverse(rows.begin(), rows.end()); + struct Search { std::size_t pos; std::string query; std::vector hits; }; + std::vector searches; + for (std::size_t pos = 0; pos < rows.size(); ++pos) { + if (rows[pos][0].as_text() == "tool" && rows[pos][1].as_text() == "search-query") { + auto parsed = parse_search_event(rows[pos][2].as_text()); + if (parsed) searches.push_back({pos, parsed->first, parsed->second}); + } + } + ImplicitPairs out; + std::vector zero_hit; + int read_follow = 0; + for (std::size_t idx = 0; idx < searches.size(); ++idx) { + const auto& s = searches[idx]; + if (s.hits.empty()) zero_hit.push_back(s.query); + std::size_t end = idx + 1 < searches.size() ? searches[idx + 1].pos : rows.size(); + std::size_t stop = std::min(end, s.pos + 21); + for (std::size_t r = s.pos + 1; r < stop; ++r) { + std::string read_id = PyStr::strip(rows[r][2].as_text()); + if (rows[r][0].as_text() == "tool" && rows[r][1].as_text() == "record-read" && !read_id.empty()) { + ++read_follow; + std::optional rank; + for (std::size_t h = 0; h < s.hits.size(); ++h) { + if (s.hits[h] == read_id) { rank = static_cast(h) + 1; break; } + } + out.pairs.push_back({s.query, read_id, rank, static_cast(s.hits.size())}); + break; + } + } + if (out.pairs.size() >= static_cast(EVAL_MAX_PAIRS)) break; + } + out.searches = static_cast(searches.size()); + out.read_follow = read_follow; + if (out.pairs.size() > static_cast(EVAL_MAX_PAIRS)) out.pairs.resize(static_cast(EVAL_MAX_PAIRS)); + out.zero_hit_count = static_cast(zero_hit.size()); + out.zero_hit.assign(zero_hit.begin(), zero_hit.begin() + static_cast(std::min(10, zero_hit.size()))); + return out; +} + +static std::string fmt(const char* f, double a, double b, double c, double d) { + char buf[128]; + std::snprintf(buf, sizeof buf, f, a, b, c, d); + return buf; +} + +std::string RetrievalEval::format_report(const EvalResult& result, const ImplicitPairs& pairs) { + std::vector lines; + lines.push_back("retrieval eval: " + std::to_string(result.goldens) + " goldens at k=" + std::to_string(result.k)); + lines.push_back("overall: " + fmt("hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f", result.overall.hit, result.overall.mrr, result.overall.ndcg, result.overall.cp)); + auto cats = result.by_category; + std::sort(cats.begin(), cats.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& c : cats) { + lines.push_back(c.first + ": n=" + std::to_string(c.second.goldens) + " " + + fmt("hit=%.3f mrr=%.3f ndcg=%.3f cp=%.3f", c.second.mean.hit, c.second.mean.mrr, c.second.mean.ndcg, c.second.mean.cp)); + } + for (const auto& m : result.misses) lines.push_back("miss [" + m.category + "]: " + PyStr::prefix(m.query, 80)); + lines.push_back("implicit: " + std::to_string(pairs.searches) + " searches, " + std::to_string(pairs.read_follow) + " followed by a read, " + + std::to_string(pairs.zero_hit_count) + " zero-hit"); + for (const auto& q : pairs.zero_hit) lines.push_back("zero-hit: " + PyStr::prefix(q, 80)); + return PyStr::join(lines, "\n"); +} + +std::string RetrievalEval::run_eval(misc::SqliteDb& db, FtsSearch& search, const std::string& profile, bool sealed) { + auto goldens = goldens_from_vault(db, profile); + auto result = score(search, goldens, profile); + auto pairs = implicit_pairs(db, profile, sealed); + return format_report(result, pairs); +} +} // namespace detz::rag diff --git a/taicpp/src/rag/retrievaleval.h b/taicpp/src/rag/retrievaleval.h new file mode 100644 index 0000000..3c3cc41 --- /dev/null +++ b/taicpp/src/rag/retrievaleval.h @@ -0,0 +1,67 @@ +#pragma once +// Port of tai.py RetrievalEval (standing retrieval-quality eval: Hit@K, MRR, binary nDCG@K, +// Context Precision, implicit pairs). Spec: docs/rag.md. +#include +#include +#include +#include +#include "ftssearch.h" +#include "sqlitedb.h" +namespace detz::rag { + +struct Golden { + std::string query; + std::vector expected; + std::string category; + misc::DbValue source; +}; + +struct EvalMean { double hit{0}, mrr{0}, ndcg{0}, cp{0}; }; +struct EvalCategory { int goldens{0}; EvalMean mean; }; +struct EvalMiss { std::string query, category; }; +struct EvalResult { + int k{0}; + int goldens{0}; + EvalMean overall; + std::vector> by_category; // insertion ordered + std::vector misses; +}; +struct ImplicitPair { std::string query, read; std::optional rank; int shown{0}; }; +struct ImplicitPairs { + int searches{0}, read_follow{0}, zero_hit_count{0}; + std::vector pairs; + std::vector zero_hit; +}; + +class RetrievalEval { + public: + static constexpr int EVAL_K = 5; + static constexpr int EVAL_FETCH = 10; + static constexpr int EVAL_MAX_RECORDS = 50; + static constexpr int EVAL_MAX_GOLDENS = 200; + static constexpr int EVAL_MAX_PAIRS = 200; + static constexpr int EVAL_MIN_TOKEN = 6; + static constexpr int EVAL_PREFIX_MIN = 4; + + static std::optional rank_of(const misc::DbValue& expected_id, const std::vector& ranked_ids); + static std::optional best_rank(const std::vector& expected_ids, const std::vector& ranked_ids); + static double hit_at_k(std::optional rank, int k = EVAL_K); + static double reciprocal_rank(std::optional rank); + static double ndcg_at_k(std::optional rank, int k = EVAL_K); + static double context_precision(const std::vector& expected_ids, const std::vector& ranked_ids, int k = EVAL_K); + static std::vector tokens(const std::string& text, int min_len = EVAL_MIN_TOKEN); + static int doc_freq(misc::SqliteDb& db, const std::string& profile, const std::string& token); // _doc_freq + static std::vector goldens_from_vault(misc::SqliteDb& db, const std::string& profile, int max_records = EVAL_MAX_RECORDS); + static EvalResult score(FtsSearch& search, const std::vector& goldens, const std::string& profile, + const std::vector& kinds = {"record"}, int k = EVAL_K, int fetch = EVAL_FETCH); + static EvalMean mean(const std::vector& rows); // _mean + // sealed: events text is read through the tai_dec SQL function the seal registers + static ImplicitPairs implicit_pairs(misc::SqliteDb& db, const std::string& profile, bool sealed, int limit = 500); + // _parse_search_event: nullopt where Python returns None + static std::optional>> parse_search_event(const std::string& text); + static std::string format_report(const EvalResult& result, const ImplicitPairs& pairs); + // The --eval-retrieval CLI body (goldens_from_vault + score + implicit_pairs + format_report) + static std::string run_eval(misc::SqliteDb& db, FtsSearch& search, const std::string& profile, bool sealed); +}; + +} // namespace detz::rag diff --git a/taicpp/src/sched/schedtime.cpp b/taicpp/src/sched/schedtime.cpp new file mode 100644 index 0000000..13c897e --- /dev/null +++ b/taicpp/src/sched/schedtime.cpp @@ -0,0 +1,81 @@ +#include "schedtime.h" +#include +#include +#include "miscdatetime.h" +#include "miscpy.h" +#include "pyunicode.h" +namespace detz::sched { +using misc::DateTime; +using misc::PyStr; + +std::string ScheduleTime::parse_schedule_at(const std::string& raw) { + std::string text = PyStr::strip(raw); + if (text.empty()) throw std::invalid_argument("empty datetime"); + if (text.back() == 'Z' || text.back() == 'z') text = text.substr(0, text.size() - 1) + "+00:00"; + DateTime moment; + try { + moment = DateTime::fromisoformat(text); + } catch (const std::invalid_argument&) { + throw std::invalid_argument("invalid datetime, use ISO like 2026-10-08T09:00"); + } + return moment.to_utc().isoformat(); +} + +std::optional ScheduleTime::parse_duration(const std::string& raw) { + // re.fullmatch(r"(\d+)\s*([smhdw])?", str(raw or "").strip().lower()): \d is any Unicode Nd digit, \s any Unicode space + std::string text = PyStr::strip(raw); + std::size_t i = 0; + std::size_t n = 0; + int64_t value = 0; + bool any_digit = false; + while (i < text.size()) { + unsigned cp = misc::PyUnicode::decode(text, i, n); + int d = misc::PyUnicode::decimal_value(cp); + if (d < 0) break; + value = value * 10 + d; + any_digit = true; + i += n; + } + if (!any_digit) return std::nullopt; + while (i < text.size()) { + unsigned cp = misc::PyUnicode::decode(text, i, n); + if (!misc::PyUnicode::is_space(cp)) break; + i += n; + } + int64_t mult = 1; + if (i < text.size()) { + char unit = text[i]; + if (unit >= 'A' && unit <= 'Z') unit = static_cast(unit + 32); + switch (unit) { + case 's': mult = 1; break; + case 'm': mult = 60; break; + case 'h': mult = 3600; break; + case 'd': mult = 86400; break; + case 'w': mult = 604800; break; + default: return std::nullopt; + } + ++i; + if (i != text.size()) return std::nullopt; + } + return value * mult; +} + +std::string ScheduleTime::format_delay(double seconds) { + int64_t s = static_cast(std::trunc(seconds)); + if (s < 0) return "overdue"; + if (s < 60) return std::to_string(s) + "s"; + if (s < 3600) return std::to_string(s / 60) + "m"; + if (s < 86400) return std::to_string(s / 3600) + "h"; + return std::to_string(s / 86400) + "d"; +} + +std::string ScheduleTime::local_display(const std::string& utc_iso) { + try { + return DateTime::fromisoformat(utc_iso).to_local().strftime_minute(); + } catch (const std::invalid_argument&) { + return utc_iso; + } +} + +std::string ScheduleTime::now_iso() { return DateTime::now_utc().isoformat(true); } +} // namespace detz::sched diff --git a/taicpp/src/sched/schedtime.h b/taicpp/src/sched/schedtime.h new file mode 100644 index 0000000..823d1c0 --- /dev/null +++ b/taicpp/src/sched/schedtime.h @@ -0,0 +1,16 @@ +#pragma once +// Port of tai.py parse_schedule_at, parse_duration, format_delay, local_display, now_iso. +#include +#include +#include +namespace detz::sched { +class ScheduleTime { + public: + // Throws std::invalid_argument with the Python ValueError text. + static std::string parse_schedule_at(const std::string& raw); + static std::optional parse_duration(const std::string& raw); + static std::string format_delay(double seconds); + static std::string local_display(const std::string& utc_iso); + static std::string now_iso(); // datetime.now(timezone.utc).isoformat(timespec="seconds") +}; +} // namespace detz::sched diff --git a/taicpp/src/sched/schedtool.cpp b/taicpp/src/sched/schedtool.cpp new file mode 100644 index 0000000..aaed971 --- /dev/null +++ b/taicpp/src/sched/schedtool.cpp @@ -0,0 +1,134 @@ +#include "schedtool.h" +#include +#include +#include "miscdatetime.h" +#include "schedtime.h" +#include "scheduler.h" +#include "skillfile.h" +namespace detz::sched { +using misc::DateTime; +using misc::Json; +using misc::PyStr; + +bool ScheduleTool::profile_valid(const std::string& name) { + // ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ ($ tolerates one trailing newline) + std::string n = name; + if (!n.empty() && n.back() == '\n') n.pop_back(); + if (n.empty() || n.size() > 32) return false; + auto alnum = [](char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); }; + if (!alnum(n[0])) return false; + for (char c : n) { + if (!alnum(c) && c != '_' && c != '-') return false; + } + return true; +} + +std::string ScheduleTool::normalize_tag(const std::string& raw) { return misc::TagNames::normalize_tag(raw); } + +std::vector ScheduleTool::normalize_tags(const std::vector& items) { return misc::TagNames::normalize_tags(items); } + +static bool truthy(const Json* v) { + if (!v) return false; + switch (v->kind()) { + case Json::Kind::Null: return false; + case Json::Kind::Bool: return v->as_bool(); + case Json::Kind::Int: return v->as_int() != 0; + case Json::Kind::Float: return v->as_double() != 0; + case Json::Kind::String: return !v->as_string().empty(); + default: return v->size() > 0; + } +} +static std::string text_of(const Json* v) { + return truthy(v) ? skills::SkillFiles::py_str(*v) : ""; +} +// int(value) the way Python does for JSON scalars; false on TypeError/ValueError +static bool to_int(const Json* v, int64_t& out) { + if (!v) return false; + switch (v->kind()) { + case Json::Kind::Bool: out = v->as_bool(); return true; + case Json::Kind::Int: out = v->as_int(); return true; + case Json::Kind::Float: out = static_cast(v->as_double()); return true; + case Json::Kind::String: { + std::string s = PyStr::strip(v->as_string()); + std::size_t i = 0; + bool neg = false; + if (i < s.size() && (s[i] == '+' || s[i] == '-')) { neg = s[i] == '-'; ++i; } + if (i >= s.size()) return false; + int64_t val = 0; + for (; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') return false; + val = val * 10 + (s[i] - '0'); + if (val > 4000000000000000000LL) break; + } + out = neg ? -val : val; + return true; + } + default: return false; + } +} + +std::string ScheduleTool::run_schedule(const Json& args) { + std::string prompt = PyStr::strip(text_of(args.get("prompt"))); + if (prompt.empty()) return "error: empty prompt"; + const Json* at_raw = args.get("at"); + const Json* every_raw = args.get("every"); + bool has_at = at_raw && !at_raw->is_null(); + bool has_every = every_raw && !every_raw->is_null(); + if (has_at == has_every) return "error: pass exactly one of at or every"; + std::string next_run; + int64_t every_sec = 0; + if (has_at) { + try { + next_run = ScheduleTime::parse_schedule_at(skills::SkillFiles::py_str(*at_raw)); + } catch (const std::exception& exc) { + return "error: " + PyStr::short_error(exc.what()); + } + } else { + if (!to_int(every_raw, every_sec)) return "error: every must be seconds"; + if (every_sec < 60) return "error: every must be at least 60 seconds"; + next_run = DateTime::now_utc().plus_seconds(static_cast(every_sec)).isoformat(); + } + std::string name = PyStr::prefix(PyStr::strip(text_of(args.get("name"))), 80); + int64_t timeout = 600; + int64_t parsed = 0; + if (truthy(args.get("timeout")) && to_int(args.get("timeout"), parsed)) timeout = std::max(60, std::min(3600, parsed)); + std::string asked = PyStr::strip(text_of(args.get("profile"))); + if (!asked.empty() && !profile_valid(asked)) return "error: invalid profile name"; + if (!asked.empty() && asked != host_.active_profile()) return "error: cannot schedule for another profile"; + std::string profile = host_.active_profile(); + const Json* raw_tags = args.get("tags"); + std::vector tag_names; + if (truthy(raw_tags)) { + if (!raw_tags->is_array()) return "error: tags must be a list of names"; + for (const auto& t : raw_tags->items()) { + if (!t.is_string()) return "error: tags must be a list of names"; + tag_names.push_back(t.as_string()); + } + } + if (!host_.ask_approval("schedule '" + (name.empty() ? PyStr::prefix(prompt, 60) : name) + "'")) return "denied by user"; + std::vector tags = normalize_tags(tag_names); + int64_t row_id = host_.vault().add_schedule(name, prompt, profile, every_sec, next_run, timeout, &tags); + std::string when = every_sec ? "every " + ScheduleTime::format_delay(static_cast(every_sec)) : "at " + ScheduleTime::local_display(next_run); + return "scheduled #" + std::to_string(row_id) + " (" + when + ")"; +} + +std::string ScheduleTool::run_unschedule(const Json& args) { + int64_t row_id = 0; + const Json* v = args.get("id"); + if (truthy(v)) { + if (!to_int(v, row_id)) return "error: invalid schedule id"; + } + bool found = false; + for (const auto& item : host_.vault().list_schedules(host_.active_profile())) { + if (item.id == row_id) found = true; + } + if (!found) return "error: no schedule #" + std::to_string(row_id); + if (!host_.ask_approval("delete schedule #" + std::to_string(row_id))) return "denied by user"; + host_.vault().remove_schedule(row_id, host_.active_profile()); + return "deleted schedule #" + std::to_string(row_id); +} + +std::string ScheduleTool::run_schedules(const Json&) { + return Scheduler::format_schedule_lines(host_.vault().list_schedules(host_.active_profile()), DateTime::now_utc()); +} +} // namespace detz::sched diff --git a/taicpp/src/sched/schedtool.h b/taicpp/src/sched/schedtool.h new file mode 100644 index 0000000..5c514dc --- /dev/null +++ b/taicpp/src/sched/schedtool.h @@ -0,0 +1,31 @@ +#pragma once +// Port of the Tools schedule backends: run_schedule, run_unschedule, run_schedules (+ normalize_tags). +#include +#include +#include "miscjson.h" +#include "schedvault.h" +namespace detz::sched { + +class ScheduleHost { + public: + virtual ~ScheduleHost() = default; + virtual bool ask_approval(const std::string& what) = 0; + virtual std::string active_profile() const = 0; + virtual ScheduleVault& vault() = 0; +}; + +class ScheduleTool { + public: + explicit ScheduleTool(ScheduleHost& host) : host_(host) {} + std::string run_schedule(const misc::Json& args); + std::string run_unschedule(const misc::Json& args); + std::string run_schedules(const misc::Json& args); + static std::string normalize_tag(const std::string& raw); + static std::vector normalize_tags(const std::vector& items); + static bool profile_valid(const std::string& name); // PROFILE_RE + + private: + ScheduleHost& host_; +}; + +} // namespace detz::sched diff --git a/taicpp/src/sched/scheduler.cpp b/taicpp/src/sched/scheduler.cpp new file mode 100644 index 0000000..a35ed97 --- /dev/null +++ b/taicpp/src/sched/scheduler.cpp @@ -0,0 +1,206 @@ +#include "scheduler.h" +#include +#include +#include +#include "miscdatetime.h" +#include "schedtime.h" +namespace detz::sched { +using misc::DateTime; +using misc::DbValue; +using misc::Json; +using misc::PyStr; +using misc::SqliteDb; + +void StopFlag::set() { + std::lock_guard lock(m_); + set_.store(true); + cv_.notify_all(); +} +bool StopFlag::wait(double seconds) { + std::unique_lock lock(m_); + cv_.wait_for(lock, std::chrono::duration(seconds), [this] { return set_.load(); }); + return set_.load(); +} + +Scheduler::Scheduler(std::string db_path, const SealPort& seal, AgentPort& agents, misc::LogSink& log, std::ostream* out) + : db_path_(std::move(db_path)), seal_(seal), agents_(agents), log_(log), out_(out ? out : &std::cout) {} + +Scheduler::~Scheduler() { + for (auto& f : flags_) f->set(); + for (auto& t : loops_) { + if (t.joinable()) t.join(); + } + // Watchers are daemon threads in tai.py: never joined at exit. + for (auto& t : watchers_) { + if (t.joinable()) t.detach(); + } +} + +std::string Scheduler::format_schedule_lines(const std::vector& items, const DateTime& now) { + if (items.empty()) return "(no schedules)"; + std::vector lines; + for (const auto& item : items) { + double due_in; + try { + due_in = DateTime::fromisoformat(item.next_run).seconds_since(now); + } catch (const std::invalid_argument&) { + due_in = -1; + } + std::string when = item.every ? "every " + ScheduleTime::format_delay(static_cast(item.every)) : "at " + ScheduleTime::local_display(item.next_run); + std::string marked = item.tags.empty() ? "" : " [" + PyStr::join(item.tags, " ") + "]"; + lines.push_back("#" + std::to_string(item.id) + " [" + item.status + "] " + when + " (" + ScheduleTime::format_delay(due_in) + ") " + + (item.name.empty() ? "(no name)" : item.name) + marked + " :: " + PyStr::prefix(item.prompt, 80)); + if (!item.last.empty()) { + std::string extra = item.result.empty() ? "" : " :: " + PyStr::prefix(item.result, 120); + lines.push_back(" last: " + item.last + extra); + } + } + return PyStr::join(lines, "\n"); +} + +void Scheduler::note_schedule_result(const std::string& db_path, int64_t row_id, const std::string& status, const std::string& result) { + try { + SqliteDb db(db_path, 30, true); + db.execute("UPDATE schedules SET last_status = ?, last_result = ?, updated = ? WHERE id = ?", + {status, PyStr::prefix(result, 2000), DateTime::now_utc().isoformat(), row_id}); + } catch (const misc::SqliteError&) { + } +} + +void Scheduler::watch_schedule(AgentPort& agents, misc::LogSink& log, const std::string& db_path, int64_t row_id, int agent_id, int64_t timeout, const std::string& profile) { + auto polled = agents.poll_agent(agent_id, static_cast(timeout + SCHEDULER_LEASE_MARGIN)); + note_schedule_result(db_path, row_id, polled.first, polled.second); + Json meta = Json::object(); + meta.set("id", row_id); + meta.set("status", polled.first); + meta.set("result_chars", static_cast(PyStr::len(polled.second))); + log.event("INFO", "sched", "finished", "schedule #" + std::to_string(row_id) + " finished: " + polled.first, profile, "", "", meta); +} + +bool Scheduler::schedule_running(const std::string& last_status, const std::string& updated, int64_t timeout) { + if (last_status != "running") return false; + double age; + try { + age = DateTime::now_utc().seconds_since(DateTime::fromisoformat(updated)); + } catch (const std::invalid_argument&) { + return false; + } + return age < static_cast((timeout ? timeout : 600) + SCHEDULER_LEASE_MARGIN); +} + +int Scheduler::scheduler_tick() { + int fired = 0, skipped_cap = 0, skipped_overlap = 0; + DateTime now = DateTime::now_utc(); + { + SqliteDb db(db_path_, 30, true); + auto rows = db.query( + "SELECT id, prompt, profile, every_sec, next_run, timeout, last_status, updated FROM schedules " + "WHERE status = 'pending' AND next_run <= ? ORDER BY next_run", + {now.isoformat()}); + for (const auto& row : rows) { + int64_t row_id = row[0].as_int(); + std::string prompt = row[1].as_text(); + std::string profile = row[2].as_text(); + int64_t every_sec = row[3].as_int(); + std::string next_run = row[4].as_text(); + int64_t timeout = row[5].as_int(); + std::string last_status = row[6].as_text(); + std::string updated = row[7].as_text(); + if (fired >= SCHEDULER_MAX_FIRES) break; + if (every_sec && schedule_running(last_status, updated, timeout)) { + ++skipped_overlap; + continue; + } + if (agents_.running_agents() >= agents_.max_parallel()) { + ++skipped_cap; + continue; + } + std::string stamp = DateTime::now_utc().isoformat(); + int64_t changed; + if (every_sec) { + DateTime claimed_next; + try { + claimed_next = DateTime::fromisoformat(next_run); + } catch (const std::invalid_argument&) { + continue; + } + while (claimed_next <= DateTime::now_utc()) claimed_next = claimed_next.plus_seconds(static_cast(every_sec)); + changed = db.execute("UPDATE schedules SET next_run = ?, updated = ?, last_status = 'running' WHERE id = ? AND status = 'pending' AND next_run = ?", + {claimed_next.isoformat(), stamp, row_id, next_run}); + } else { + changed = db.execute("UPDATE schedules SET status = 'done', updated = ? WHERE id = ? AND status = 'pending'", {stamp, row_id}); + } + if (changed != 1) continue; + if (SealPort::is_sealed(prompt)) { + try { + prompt = seal_.unlock(prompt); + } catch (const SealError& exc) { + note_schedule_result(db_path_, row_id, "error: " + PyStr::short_error(exc.what())); + continue; + } + } + int agent_id; + try { + agent_id = agents_.spawn_agent(prompt, profile, timeout); + } catch (const AgentsFull&) { + if (every_sec) { + db.execute("UPDATE schedules SET next_run = ?, updated = ?, last_status = ? WHERE id = ?", {next_run, stamp, last_status, row_id}); + } else { + db.execute("UPDATE schedules SET status = 'pending', updated = ? WHERE id = ?", {stamp, row_id}); + } + ++skipped_cap; + continue; + } + { + std::lock_guard lock(watch_mutex_); + // The watcher captures values, never `this`: tai.py watchers are daemon threads that may outlive the scheduler. + AgentPort* agents = &agents_; + misc::LogSink* log = &log_; + std::string db_path = db_path_; + watchers_.emplace_back([agents, log, db_path, row_id, agent_id, timeout, profile] { + watch_schedule(*agents, *log, db_path, row_id, agent_id, timeout, profile); + }); + } + ++fired; + Json meta = Json::object(); + meta.set("id", row_id); + meta.set("profile", profile); + meta.set("every_sec", every_sec); + log_.event("INFO", "sched", "fired", "schedule #" + std::to_string(row_id) + " fired for " + profile, profile, "", "", meta); + } + } + if (skipped_cap || skipped_overlap) { + *out_ << "scheduler: skipped " << skipped_cap << " at parallel cap, " << skipped_overlap << " overlapping" << std::endl; + } + return fired; +} + +void Scheduler::run_scheduler_loop(StopFlag& stop) { + while (!stop.is_set()) { + try { + scheduler_tick(); + } catch (const std::exception& exc) { + *out_ << "scheduler error: " << PyStr::short_error(exc.what()) << std::endl; + } + stop.wait(SCHEDULER_INTERVAL); + } +} + +std::shared_ptr Scheduler::start_scheduler() { + auto flag = std::make_shared(); + flags_.push_back(flag); + loops_.emplace_back([this, flag] { run_scheduler_loop(*flag); }); + return flag; +} + +void Scheduler::join_watchers() { + std::vector taken; + { + std::lock_guard lock(watch_mutex_); + taken.swap(watchers_); + } + for (auto& t : taken) { + if (t.joinable()) t.join(); + } +} +} // namespace detz::sched diff --git a/taicpp/src/sched/scheduler.h b/taicpp/src/sched/scheduler.h new file mode 100644 index 0000000..ef75de6 --- /dev/null +++ b/taicpp/src/sched/scheduler.h @@ -0,0 +1,79 @@ +#pragma once +// Port of scheduler_tick, run_scheduler_loop, start_scheduler, watch_schedule, note_schedule_result, +// schedule_running and format_schedule_lines. The agent registry (spawn_agent/poll_agent/running_agents) +// belongs to the agent core and is reached through AgentPort. Spec: docs/sched.md. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "miscdatetime.h" +#include "miscpy.h" +#include "schedvault.h" +namespace detz::sched { + +class AgentsFull : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +class AgentPort { + public: + virtual ~AgentPort() = default; + virtual int running_agents() = 0; + virtual int max_parallel() const { return 8; } // AGENTS_MAX_PARALLEL + // spawn_agent(task, profile, timeout, config, seal, depth=0, runner): throws AgentsFull + virtual int spawn_agent(const std::string& task, const std::string& profile, int64_t timeout) = 0; + // poll_agent(id, wait=...) -> (status, text) + virtual std::pair poll_agent(int agent_id, double wait) = 0; +}; + +// threading.Event +class StopFlag { + public: + void set(); + bool is_set() const { return set_.load(); } + // Event.wait(timeout): returns true when the flag is set + bool wait(double seconds); + private: + std::atomic set_{false}; + std::mutex m_; + std::condition_variable cv_; +}; + +class Scheduler { + public: + static constexpr int SCHEDULER_MAX_FIRES = 4; + static constexpr int SCHEDULER_LEASE_MARGIN = 30; + static constexpr int SCHEDULER_INTERVAL = 30; + + Scheduler(std::string db_path, const SealPort& seal, AgentPort& agents, misc::LogSink& log = misc::LogSink::null_sink(), std::ostream* out = nullptr); + ~Scheduler(); + + static std::string format_schedule_lines(const std::vector& items, const misc::DateTime& now); + static bool schedule_running(const std::string& last_status, const std::string& updated, int64_t timeout); + static void note_schedule_result(const std::string& db_path, int64_t row_id, const std::string& status, const std::string& result = ""); + static void watch_schedule(AgentPort& agents, misc::LogSink& log, const std::string& db_path, int64_t row_id, int agent_id, int64_t timeout, const std::string& profile); + int scheduler_tick(); + void run_scheduler_loop(StopFlag& stop); + // start_scheduler: runs the loop on a daemon-style thread; the caller sets the returned flag to stop it + std::shared_ptr start_scheduler(); + // Test hook: waits for the watcher threads started by scheduler_tick. + void join_watchers(); + + private: + std::string db_path_; + const SealPort& seal_; + AgentPort& agents_; + misc::LogSink& log_; + std::ostream* out_; + std::mutex watch_mutex_; + std::vector watchers_; + std::vector loops_; + std::vector> flags_; +}; + +} // namespace detz::sched diff --git a/taicpp/src/sched/schedvault.cpp b/taicpp/src/sched/schedvault.cpp new file mode 100644 index 0000000..e1fbda5 --- /dev/null +++ b/taicpp/src/sched/schedvault.cpp @@ -0,0 +1,77 @@ +#include "schedvault.h" +#include "schedtime.h" +namespace detz::sched { +using misc::DbValue; +using misc::Json; + +const char* ScheduleVault::ddl() { + return "CREATE TABLE IF NOT EXISTS schedules (id INTEGER PRIMARY KEY, name TEXT, prompt TEXT, profile TEXT, every_sec INTEGER, " + "next_run TEXT, timeout INTEGER, status TEXT, last_status TEXT, last_result TEXT, created TEXT, updated TEXT)"; +} + +int64_t ScheduleVault::add_schedule(const std::string& name, const std::string& prompt, const std::string& profile, int64_t every_sec, + const std::string& next_run, int64_t timeout, const std::vector* tags) { + std::string stored = seal_.enabled() ? seal_.lock(prompt) : prompt; + std::string now = ScheduleTime::now_iso(); + db_.execute( + "INSERT INTO schedules (name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created, updated) " + "VALUES (?, ?, ?, ?, ?, ?, 'pending', '', '', ?, ?)", + {name, stored, profile, every_sec, next_run, timeout, now, ScheduleTime::now_iso()}); + int64_t id = db_.last_insert_rowid(); + std::string item = "sched:" + std::to_string(id); + if (!tags) { + tags_.tag_item(item, {"schedule"}, profile); + } else { + std::vector all = {"schedule"}; + all.insert(all.end(), tags->begin(), tags->end()); + tags_.set_item_tags(item, all, profile); + } + Json meta = Json::object(); + meta.set("id", id); + meta.set("name", name); + log_.event("DEBUG", "vault", "schedule_saved", "schedule #" + std::to_string(id) + " saved (" + name + ")", profile, "", "", meta); + return id; +} + +bool ScheduleVault::remove_schedule(int64_t row_id, const std::string& profile_in) { + std::string profile = scope(profile_in); + int64_t n = db_.execute("DELETE FROM schedules WHERE id = ? AND profile = ?", {row_id, profile}); + if (n > 0) { + std::string item = "sched:" + std::to_string(row_id); + db_.execute("DELETE FROM tags WHERE item = ? AND profile = ?", {item, profile}); + db_.execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {profile, item, item}); + Json meta = Json::object(); + meta.set("id", row_id); + log_.event("DEBUG", "vault", "schedule_removed", "schedule #" + std::to_string(row_id) + " removed", profile, "", "", meta); + return true; + } + return false; +} + +std::vector ScheduleVault::list_schedules(const std::string& profile_in) { + std::string profile = scope(profile_in); + auto rows = db_.query( + "SELECT id, name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created " + "FROM schedules WHERE profile = ? ORDER BY next_run", + {profile}); + std::vector items; + for (const auto& r : rows) { + ScheduleItem it; + it.id = r[0].as_int(); + it.name = r[1].as_text(); + std::string prompt = r[2].as_text(); + it.prompt = SealPort::is_sealed(prompt) ? seal_.safe_unlock(prompt) : prompt; + it.profile = r[3].as_text(); + it.every = r[4].as_int(); + it.next_run = r[5].as_text(); + it.timeout = r[6].as_int(); + it.status = r[7].as_text(); + it.last = r[8].as_text(); + it.result = r[9].as_text(); + it.created = r[10].as_text(); + it.tags = tags_.item_tags("sched:" + std::to_string(it.id), it.profile); + items.push_back(std::move(it)); + } + return items; +} +} // namespace detz::sched diff --git a/taicpp/src/sched/schedvault.h b/taicpp/src/sched/schedvault.h new file mode 100644 index 0000000..fa75fc9 --- /dev/null +++ b/taicpp/src/sched/schedvault.h @@ -0,0 +1,65 @@ +#pragma once +// Port of Store.add_schedule / remove_schedule / list_schedules over the vault `schedules` table. +// The seal and the tag vocabulary (canonical_tags) belong to the store module and are reached by ports. +#include +#include +#include +#include "miscpy.h" +#include "sqlitedb.h" +namespace detz::sched { + +struct ScheduleItem { + int64_t id{0}; + std::string name, prompt, profile; + int64_t every{0}; + std::string next_run; + int64_t timeout{0}; + std::string status, last, result, created; + std::vector tags; +}; + +// Seal as the scheduler sees it: PREFIX test, lock/unlock/safe_unlock, enabled flag. +class SealPort { + public: + static constexpr const char* PREFIX = "tai1$"; + virtual ~SealPort() = default; + virtual bool enabled() const = 0; + virtual std::string lock(const std::string& plain) const = 0; + // Throws SealError (below) + virtual std::string unlock(const std::string& sealed) const = 0; + virtual std::string safe_unlock(const std::string& sealed) const = 0; + static bool is_sealed(const std::string& text) { return text.compare(0, 5, PREFIX) == 0; } +}; +class SealError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +// Store.tag_item / set_item_tags / item_tags +class TagPort { + public: + virtual ~TagPort() = default; + virtual void tag_item(const std::string& item, const std::vector& tags, const std::string& profile) = 0; + virtual void set_item_tags(const std::string& item, const std::vector& tags, const std::string& profile) = 0; + virtual std::vector item_tags(const std::string& item, const std::string& profile) = 0; +}; + +class ScheduleVault { + public: + ScheduleVault(misc::SqliteDb& db, const SealPort& seal, TagPort& tags, std::string profile, misc::LogSink& log = misc::LogSink::null_sink()) + : db_(db), seal_(seal), tags_(tags), profile_(std::move(profile)), log_(log) {} + // tags == nullptr mirrors tags=None (only the default "schedule" tag) + int64_t add_schedule(const std::string& name, const std::string& prompt, const std::string& profile, int64_t every_sec, + const std::string& next_run, int64_t timeout, const std::vector* tags = nullptr); + bool remove_schedule(int64_t row_id, const std::string& profile = ""); + std::vector list_schedules(const std::string& profile = ""); + // "CREATE TABLE schedules" DDL, column order of Store.TABLES + static const char* ddl(); + + private: + misc::SqliteDb& db_; + const SealPort& seal_; + TagPort& tags_; + std::string profile_; + misc::LogSink& log_; + std::string scope(const std::string& p) const { return p.empty() ? profile_ : p; } +}; + +} // namespace detz::sched diff --git a/taicpp/src/sched/sqlitedb.cpp b/taicpp/src/sched/sqlitedb.cpp new file mode 100644 index 0000000..3f4027f --- /dev/null +++ b/taicpp/src/sched/sqlitedb.cpp @@ -0,0 +1,86 @@ +#include "sqlitedb.h" +#include +#include +namespace detz::misc { +int64_t DbValue::as_int() const { + if (is_int()) return std::get(v); + if (std::holds_alternative(v)) return static_cast(std::get(v)); + return 0; +} +double DbValue::as_real() const { + if (std::holds_alternative(v)) return std::get(v); + return static_cast(as_int()); +} +std::string DbValue::as_text() const { + if (is_text()) return std::get(v); + if (is_int()) return std::to_string(std::get(v)); + if (std::holds_alternative(v)) return std::to_string(std::get(v)); + return ""; +} + +SqliteDb::SqliteDb(const std::string& path, int timeout_s, bool wal) { + if (sqlite3_open(path.c_str(), &db_) != SQLITE_OK) { + std::string msg = db_ ? sqlite3_errmsg(db_) : "out of memory"; + if (db_) sqlite3_close(db_); + db_ = nullptr; + throw SqliteError(msg); + } + sqlite3_busy_timeout(db_, timeout_s * 1000); + if (wal) { + execute("PRAGMA journal_mode=WAL"); + execute("PRAGMA busy_timeout=" + std::to_string(timeout_s * 1000)); + } +} +SqliteDb::~SqliteDb() { + if (db_ && owned_) sqlite3_close(db_); +} + +std::vector SqliteDb::query(const std::string& sql, const std::vector& params) { + sqlite3_stmt* st = nullptr; + if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &st, nullptr) != SQLITE_OK) throw SqliteError(sqlite3_errmsg(db_)); + for (std::size_t i = 0; i < params.size(); ++i) { + int idx = static_cast(i) + 1; + const DbValue& p = params[i]; + if (p.is_null()) sqlite3_bind_null(st, idx); + else if (p.is_int()) sqlite3_bind_int64(st, idx, std::get(p.v)); + else if (std::holds_alternative(p.v)) sqlite3_bind_double(st, idx, std::get(p.v)); + else { + const std::string& s = std::get(p.v); + sqlite3_bind_text(st, idx, s.data(), static_cast(s.size()), SQLITE_TRANSIENT); + } + } + std::vector rows; + while (true) { + int rc = sqlite3_step(st); + if (rc == SQLITE_DONE) break; + if (rc != SQLITE_ROW) { + std::string msg = sqlite3_errmsg(db_); + sqlite3_finalize(st); + throw SqliteError(msg); + } + DbRow row; + int n = sqlite3_column_count(st); + for (int c = 0; c < n; ++c) { + switch (sqlite3_column_type(st, c)) { + case SQLITE_INTEGER: row.emplace_back(static_cast(sqlite3_column_int64(st, c))); break; + case SQLITE_FLOAT: row.emplace_back(sqlite3_column_double(st, c)); break; + case SQLITE_NULL: row.emplace_back(); break; + default: { + const unsigned char* t = sqlite3_column_text(st, c); + int len = sqlite3_column_bytes(st, c); + row.emplace_back(std::string(reinterpret_cast(t), static_cast(len))); + } + } + } + rows.push_back(std::move(row)); + } + sqlite3_finalize(st); + return rows; +} + +int64_t SqliteDb::execute(const std::string& sql, const std::vector& params) { + query(sql, params); + return sqlite3_changes(db_); +} +int64_t SqliteDb::last_insert_rowid() const { return sqlite3_last_insert_rowid(db_); } +} // namespace detz::misc diff --git a/taicpp/src/sched/sqlitedb.h b/taicpp/src/sched/sqlitedb.h new file mode 100644 index 0000000..a4067ef --- /dev/null +++ b/taicpp/src/sched/sqlitedb.h @@ -0,0 +1,49 @@ +#pragma once +// Minimal sqlite3 wrapper with Python sqlite3 semantics: positional parameters, rows as value vectors, +// rowcount of the last statement. Shared by sched and rag. +#include +#include +#include +#include +#include +struct sqlite3; +namespace detz::misc { +class SqliteError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +struct DbValue { + std::variant v; + DbValue() = default; + DbValue(int64_t x) : v(x) {} + DbValue(int x) : v(static_cast(x)) {} + DbValue(long long x) : v(static_cast(x)) {} + DbValue(double x) : v(x) {} + DbValue(const char* x) : v(std::string(x)) {} + DbValue(std::string x) : v(std::move(x)) {} + bool operator==(const DbValue& o) const { return v == o.v; } // Python equality across types: 5 != "5" + bool is_null() const { return std::holds_alternative(v); } + bool is_int() const { return std::holds_alternative(v); } + bool is_text() const { return std::holds_alternative(v); } + int64_t as_int() const; + double as_real() const; + std::string as_text() const; // str(value); None renders as "" +}; +using DbRow = std::vector; + +class SqliteDb { + public: + // timeout_s mirrors sqlite3.connect(timeout=...); wal switches journal_mode=WAL like the scheduler does. + explicit SqliteDb(const std::string& path, int timeout_s = 30, bool wal = true); + explicit SqliteDb(sqlite3* borrowed) : db_(borrowed), owned_(false) {} + ~SqliteDb(); + SqliteDb(const SqliteDb&) = delete; + SqliteDb& operator=(const SqliteDb&) = delete; + std::vector query(const std::string& sql, const std::vector& params = {}); + // execute: returns rowcount of the statement (sqlite3_changes) + int64_t execute(const std::string& sql, const std::vector& params = {}); + int64_t last_insert_rowid() const; + sqlite3* handle() const { return db_; } + private: + sqlite3* db_{nullptr}; + bool owned_{true}; +}; +} // namespace detz::misc diff --git a/taicpp/src/skills/bundled.cpp b/taicpp/src/skills/bundled.cpp new file mode 100644 index 0000000..785b93c --- /dev/null +++ b/taicpp/src/skills/bundled.cpp @@ -0,0 +1,235 @@ +#include "bundled.h" +#include +#include +#include "bundled_data.h" +#include "miscdatetime.h" +#include "miscjson.h" +#include "miscpy.h" +#include "miscsha256.h" +#include "skillfile.h" +namespace detz::skills { +using misc::Json; +using misc::PyFs; +using misc::PyStr; + +static std::string short_err(const std::exception& e) { return PyStr::short_error(e.what()); } + +std::vector BundledSkills::names() { + std::vector out; + for (const auto& kv : BundledData::files()) out.push_back(kv.first); + return out; +} + +std::string BundledSkills::bundled_skill_digest(const std::string& text) { return misc::Sha256::hex_prefix(text, 8); } + +bool BundledSkills::bundled_skill_pristine(const std::string& home, const std::string& name) { + std::string folder = PyFs::join(PyFs::join(home, "skills"), name); + for (const auto& kv : BundledData::files().at(name)) { + std::string path = PyFs::join(folder, kv.first); + try { + if (PyFs::read_text(path) != kv.second) return false; + } catch (const misc::OsError&) { + return false; + } + } + return true; +} + +WriteResult BundledSkills::write_bundled_skill(const std::string& home, const std::string& name, bool force) const { + std::string base = PyFs::join(home, "skills"); + std::string folder = PyFs::join(base, name); + const auto& files = BundledData::files().at(name); + WriteResult res; + try { + PyFs::makedirs(folder); + } catch (const misc::OsError& e) { + res.errors.push_back(name + ": " + short_err(e)); + return res; + } + for (const auto& kv : files) { // std::map iterates in sorted(files) order + const std::string& rel = kv.first; + std::string path = PyFs::join(folder, rel); + bool differ = false; + try { + if (PyFs::isfile(path)) { + if (PyFs::read_text(path) == kv.second) continue; + if (!force) { + res.kept.push_back(rel); + continue; + } + differ = true; + } else if (!PyFs::dirname(path).empty()) { + PyFs::makedirs(PyFs::dirname(path)); + } + } catch (const misc::OsError& e) { + res.errors.push_back(name + "/" + rel + ": " + short_err(e)); + continue; + } + try { + PyFs::write_text(path, kv.second); + } catch (const misc::OsError& e) { + res.errors.push_back(name + "/" + rel + ": " + short_err(e)); + continue; + } + res.written.push_back(rel); + if (differ) res.overwritten.push_back(rel); + } + if (res.errors.empty() && res.kept.empty()) { + Json digests = Json::object(); + for (const auto& kv : files) digests.set(kv.first, bundled_skill_digest(kv.second)); + Json meta = Json::object(); + meta.set("source", "tai-bundled"); + meta.set("mode", "bundled"); + meta.set("scope", "home"); + meta.set("upstream", BundledData::upstream().at(name)); + meta.set("bundled_version", version_); + meta.set("files", digests); + meta.set("installed_at", misc::DateTime::now_utc().isoformat()); + try { + SkillFiles::save_skill_install_meta(base, name, meta); + } catch (const misc::OsError& e) { + res.errors.push_back(name + " sidecar: " + short_err(e)); + } + } + return res; +} + +RemoveResult BundledSkills::remove_bundled_skill(const std::string& home, const std::string& name) { + std::string base = PyFs::join(home, "skills"); + std::string folder = PyFs::join(base, name); + const auto& files = BundledData::files().at(name); + std::optional meta = SkillFiles::load_skill_install_meta(base, name); + const Json* tracked = nullptr; + if (meta && meta->is_object()) { + const Json* t = meta->get("files"); + if (t && t->is_object() && t->size() > 0) tracked = t; + } + RemoveResult res; + if (!PyFs::isdir(folder)) { + std::string sidecar = SkillFiles::skill_install_meta_path(base, name); + if (PyFs::isfile(sidecar)) { + try { + PyFs::remove(sidecar); + } catch (const misc::OsError& e) { + res.errors.push_back(name + " sidecar: " + short_err(e)); + } + } + return res; + } + for (const auto& kv : files) { + const std::string& rel = kv.first; + std::string path = PyFs::join(folder, rel); + if (!PyFs::isfile(path)) continue; + std::string current; + try { + current = PyFs::read_text(path); + } catch (const misc::OsError& e) { + res.errors.push_back(name + "/" + rel + ": " + short_err(e)); + continue; + } + bool pristine = current == kv.second; + if (!pristine && tracked) { + const Json* want = tracked->get(rel); + pristine = want && want->is_string() && want->as_string() == bundled_skill_digest(current); + } + if (pristine) { + try { + PyFs::remove(path); + res.removed.push_back(rel); + } catch (const misc::OsError& e) { + res.errors.push_back(name + "/" + rel + ": " + short_err(e)); + } + } else { + res.leftovers.push_back(rel); + } + } + for (const auto& full : PyFs::walk_files(folder)) { + std::string rel = PyFs::relpath(full, folder); + if (!files.count(rel)) res.leftovers.push_back(rel); + } + if (res.leftovers.empty() && res.errors.empty()) { + try { + PyFs::rmtree(folder); + } catch (const misc::OsError& e) { + res.errors.push_back(name + ": " + short_err(e)); + return res; + } + std::string sidecar = SkillFiles::skill_install_meta_path(base, name); + if (PyFs::isfile(sidecar)) { + try { + PyFs::remove(sidecar); + } catch (const misc::OsError& e) { + res.errors.push_back(name + " sidecar: " + short_err(e)); + } + } + } + return res; +} + +static std::string plural_list(const std::string& label, const std::vector& items, const std::string& sep) { + return label + " " + std::to_string(items.size()) + " (" + PyStr::join(items, sep) + ")"; +} + +std::string BundledSkills::op_skills(const std::string& mode, const std::string& home) const { + std::vector all = names(); + if (mode == "uninstall") { + std::vector gone, kept, errors; + for (const auto& name : all) { + std::string folder = PyFs::join(PyFs::join(home, "skills"), name); + std::string sidecar = SkillFiles::skill_install_meta_path(PyFs::join(home, "skills"), name); + if (!PyFs::isdir(folder) && !PyFs::isfile(sidecar)) continue; + RemoveResult r = remove_bundled_skill(home, name); + errors.insert(errors.end(), r.errors.begin(), r.errors.end()); + if (!r.leftovers.empty()) { + std::vector left = r.leftovers; + std::sort(left.begin(), left.end()); + kept.push_back(name + " (" + PyStr::join(left, ", ") + ")"); + } else if (r.errors.empty()) { + gone.push_back(name); + } + } + if (gone.empty() && kept.empty() && errors.empty()) return "skills: not present"; + std::vector parts; + if (!gone.empty()) parts.push_back("removed " + std::to_string(gone.size()) + " (" + PyStr::join(gone, ", ") + ")"); + if (!kept.empty()) parts.push_back("kept modified " + std::to_string(kept.size()) + " (" + PyStr::join(kept, "; ") + ")"); + if (!errors.empty()) parts.push_back("errors: " + PyStr::join(errors, "; ")); + return "skills: " + PyStr::join(parts, ", "); + } + std::vector installed, refreshed, reinstalled, current, kept, errors; + for (const auto& name : all) { + std::string folder = PyFs::join(PyFs::join(home, "skills"), name); + bool existed = PyFs::isfile(PyFs::join(folder, "SKILL.md")); + WriteResult w = write_bundled_skill(home, name, mode == "reinstall"); + errors.insert(errors.end(), w.errors.begin(), w.errors.end()); + if (!w.kept.empty()) kept.push_back(name + " (" + PyStr::join(w.kept, ", ") + ")"); + else if (mode == "reinstall" && (!w.written.empty() || !w.overwritten.empty())) reinstalled.push_back(name); + else if (!w.written.empty() && !existed) installed.push_back(name); + else if (!w.written.empty()) refreshed.push_back(name); + else if (w.errors.empty()) current.push_back(name); + } + std::vector parts; + if (!installed.empty()) parts.push_back(plural_list("installed", installed, ", ")); + if (!refreshed.empty()) parts.push_back(plural_list("refreshed", refreshed, ", ")); + if (!reinstalled.empty()) parts.push_back(plural_list("reinstalled", reinstalled, ", ")); + if (!current.empty()) parts.push_back(plural_list("already current", current, ", ")); + if (!kept.empty()) parts.push_back(plural_list("kept modified", kept, "; ")); + if (!errors.empty()) parts.push_back("errors: " + PyStr::join(errors, "; ")); + if (parts.size() == 1 && current.size() == all.size()) return "skills: already present (" + std::to_string(all.size()) + ")"; + return "skills: " + PyStr::join(parts, ", "); +} + +std::string BundledSkills::status_line(const std::string& home) { + std::vector present, missing, modified; + for (const auto& name : names()) { + if (PyFs::isfile(PyFs::join(PyFs::join(PyFs::join(home, "skills"), name), "SKILL.md"))) present.push_back(name); + else missing.push_back(name); + } + for (const auto& name : present) { + if (!bundled_skill_pristine(home, name)) modified.push_back(name); + } + std::string detail = "skills: " + std::to_string(present.size()) + "/" + std::to_string(names().size()) + " bundled present"; + if (!missing.empty()) detail += ", missing: " + PyStr::join(missing, ", "); + if (!modified.empty()) detail += ", modified: " + PyStr::join(modified, ", "); + return detail; +} +} // namespace detz::skills diff --git a/taicpp/src/skills/bundled.h b/taicpp/src/skills/bundled.h new file mode 100644 index 0000000..c5165ec --- /dev/null +++ b/taicpp/src/skills/bundled.h @@ -0,0 +1,32 @@ +#pragma once +// Port of the bundled-skill install machinery: bundled_skill_digest, bundled_skill_pristine, +// write_bundled_skill, remove_bundled_skill, op_skills and the skills line of status_lines. +// Embedded text lives in the generated bundled_data.cpp. Spec: docs/skills.md. +#include +#include +namespace detz::skills { + +struct WriteResult { std::vector written, kept, errors, overwritten; }; +struct RemoveResult { std::vector removed, leftovers, errors; }; + +class BundledSkills { + public: + // tai.py VERSION this port emulates; recorded as bundled_version in the sidecar. + static constexpr const char* TAI_VERSION = "1.44.2"; + explicit BundledSkills(std::string version = TAI_VERSION) : version_(std::move(version)) {} + + static std::vector names(); // sorted(BUNDLED_SKILLS) + static std::string bundled_skill_digest(const std::string& text); + static bool bundled_skill_pristine(const std::string& home, const std::string& name); + WriteResult write_bundled_skill(const std::string& home, const std::string& name, bool force) const; + static RemoveResult remove_bundled_skill(const std::string& home, const std::string& name); + // op_skills(mode, home): mode is install, upgrade, reinstall or uninstall + std::string op_skills(const std::string& mode, const std::string& home) const; + // The "skills: P/N bundled present..." line of status_lines + static std::string status_line(const std::string& home); + + private: + std::string version_; +}; + +} // namespace detz::skills diff --git a/taicpp/src/skills/bundled_data.cpp b/taicpp/src/skills/bundled_data.cpp new file mode 100644 index 0000000..f1fb19b --- /dev/null +++ b/taicpp/src/skills/bundled_data.cpp @@ -0,0 +1,1953 @@ +// GENERATED by src/skills/gen_bundled.py from tai.py. Do not edit. +// Source: tai.py BUNDLED_SKILLS, BUNDLED_SKILL_UPSTREAM, SKILL_BLUEPRINTS (byte-identical). +#include "bundled_data.h" +namespace detz::skills { + +const std::map>& BundledData::files() { + static const std::map> data = { + {R"detzraw(deep-research)detzraw", { + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +name: deep-research +description: "End-to-end deep research that turns a question into a cited, verifiable report. Use whenever the answer needs many external sources rather than a single lookup: technology/library selection, code and architecture questions, academic or medical literature, market and company facts, policy and standards, and 'what's the latest' or capability/benchmark questions. Routes each sub-question to the most authoritative sources for its domain (GitHub and official docs for code, arXiv/OpenAlex/Semantic Scholar for scholarship, PubMed/Europe PMC for medicine, SEC EDGAR for company facts, vendor and official pages for products), gathers evidence in parallel, triangulates claims, and writes a structured report with dated citations. Trigger on 调研/研究/调查/综述/对比一下 X、帮我查清楚 X、X 的最新进展/评测数据/方案有哪些, and any request for a research report, literature review, landscape survey, or source-backed answer - even when the user does not say the word 'research'." +--- + +# Deep Research + +Turn a question into a report a reader can trust: every claim traceable to a source, sources chosen for authority rather than convenience, and uncertainty stated instead of hidden. + +This skill is **self-contained and host-agnostic**. It defines the *method*; it requires no custom agent, plugin, or MCP server. Where it says "spawn workers", use whatever parallel-subagent facility the host provides (see *Running workers*); if none exists, run the same steps yourself, sequentially. + +## When to use + +Use it when the answer must be assembled from **multiple external sources** and needs to be current, comparative, or authoritative: + +- technical / library / architecture selection, migration paths, "X vs Y" +- literature reviews, "what does the research say about X" +- medical, clinical, or life-science questions +- market, company, competitive, or due-diligence facts +- policy, regulation, standards +- "what's the latest / what changed recently" about a fast-moving area +- any request for a research report, survey, landscape, or source-backed answer + +Do **not** use it for a single fact ("capital of France"), a quick definition, or when the user explicitly wants a short direct answer - just answer. + +## Principles + +1. **Primary sources first.** Prefer the thing itself - the paper, the official docs, the repository, the filing, the standard - over articles that describe it. Secondary sources orient and corroborate. +2. **Provenance on every non-obvious claim.** Each such claim carries a source URL and the date it was retrieved; quote the exact supporting text when wording matters. +3. **Triangulate.** A load-bearing claim needs ≥2 independent sources. If only one exists, say so and lower confidence. +4. **Freshness.** For "latest/current/recent" questions, constrain retrieval to a recent window and state the cutoff date. Version- or date-sensitive facts must name the version or date. +5. **Surface disagreement.** When credible sources conflict, present both and explain the difference; never silently pick one or average them. +6. **Separate fact from inference.** Label anything you derived; never present an inference as a fact. +7. **Uncertainty is data.** State what could not be verified and how that bounds the conclusion. +8. **Write in the user's language**, even though this skill is written in English. + +## Workflow + +Before Phase 2 (Route), read `references/source-routing.md`. Before Phase 5, read `references/report-template.md`. Before Phase 6, run `references/quality-checklist.md`. When spawning workers, take their prompt from `references/subagent-briefs.md`. + +Emit a one-line progress note per phase so a long run stays visible, e.g. `[deep-research] gather (phase 3) - 4 workers, 21 sources`. + +### 0 · Preflight +Verify retrieval works before planning: in tai this means the `web_search` and `web_fetch` tools, plus `shell` for git and code search. Tai forbids curl and wget via shell: every JSON-API recipe in `references/source-routing.md` is a plain GET, so issue it through `web_fetch` with the same URL and headers. If `web_search` is unavailable or has no provider configured, use the domain APIs in `references/source-routing.md` as the search path rather than relying on scraping search-engine result pages, and say so in the report's method section. For questions about a local codebase, also use code-search and file-read tools. + +### 1 · Frame +Restate the question in one or two sentences, then: +- Decompose it into 3–7 concrete sub-questions that together answer it. +- State the deliverable (default: a Markdown report file) and pick a depth (below). +- Define what "answered" means - which sub-questions must be covered. +- Ask the user **only** if a missing detail would change the whole approach; otherwise state your assumptions and proceed. They asked for a finished report, not a questionnaire. + +### 2 · Route +Classify the domain(s) and list, per sub-question, the authoritative source families from `references/source-routing.md`. Prefer primary/official sources. Note paywalled sources up front so they do not block later phases. + +### 3 · Gather (parallel) +Spawn one worker per sub-question or source family (see *Running workers*). Give each worker: the sub-question, the assigned source families, the recency window, and the evidence format from `references/subagent-briefs.md`. Each worker returns an **evidence pack**: + +``` +- Claim: + Source: - <URL> (retrieved YYYY-MM-DD) + Quote: "<verbatim supporting text>" + Tier: T1 primary | T2 authoritative secondary | T3 preprint/reputable community | T4 unvetted + Confidence: high | medium | low +``` + +Workers return evidence, not prose. They must not pad or speculate. + +### 4 · Consolidate +Merge the packs into one evidence table; deduplicate. Mark each load-bearing claim **verified** (≥2 independent T1/T2), **single-source**, or **contradicted**. List gaps and send targeted follow-ups until every "answered" item is covered or explicitly marked unanswerable. + +### 5 · Synthesize +Write the report using `references/report-template.md`: +- inline citation next to every non-obvious claim; +- facts, analysis, and recommendations visibly separate; +- a comparison table when comparing options; +- a "Confidence, gaps & disagreements" section. + +### 6 · Verify (before delivery) +Run `references/quality-checklist.md`. For contested or high-stakes topics, re-fetch a sample of load-bearing citations to confirm they still support the claim, and look for counter-evidence. Fix or annotate anything that fails. Never ship a claim whose cited page does not support it. + +### 7 · Deliver +- Write the report to a file (default `docs/research/<slug>-<YYYY-MM-DD>.md` unless the user names a location). +- Follow it with a 3–6 line summary: strongest conclusion, biggest uncertainty, most trusted sources. +- List sources with tier and retrieval date. + +## Running workers + +The method is host-agnostic; the spawning mechanics below are tai's (`fork`/`poll`). + +- **Spawning:** use the `fork` tool (one call per worker, task plus timeout up to one hour) and collect results with `poll` (each call waits up to two minutes, so poll in a loop until every worker finishes). Pass the worker prompt from `references/subagent-briefs.md` with the placeholders filled in. Tai workers share the machine and tools but get 12 steps, cannot ask the user, and stop nesting after 2 levels: keep briefs tight and self-contained. +- **Bound concurrency to 2–3 workers per turn.** Some providers reject a turn containing several parallel tool calls (e.g. "an assistant message with 'tool_calls' must be followed by tool messages responding to each 'tool_call_id'"). When there are more sub-questions than that, run them in batches. If a batch errors, relaunch only the failed workers in smaller batches, and if the error persists, do the work yourself sequentially. Tai caps live subagents at 8: never hold more than 8 unfinished `fork`s at once. +- **Worker profile:** `fork` accepts a profile; use the current one unless the task needs another identity. +- **If `fork` is unavailable**, run the same briefs yourself, sequentially, one sub-question at a time. The evidence format and routing do not change. +- Keep workers independent: one worker's conclusions must not frame another's search. + +## Depth levels + +| Depth | Use for | Shape | +|---|---|---| +| quick | simple, current facts | one gather pass, no independent verification, short report | +| standard (default) | most requests | parallel gather + consolidate + inline verification + full report | +| deep | contested, high-stakes, or "comprehensive" | multi-round, a dedicated verification worker, explicit disagreement analysis | + +## Bounded runs + +- Announce phase, worker count, and recency window before a long gather. +- If a source is rate-limited or paywalled, record it and move on - do not retry indefinitely. +- If a sub-question is still unanswered after two rounds, mark it and proceed. + +## Why this shape + +A single search returns undifferentiated snippets; a report needs provenance. The hard part is not fetching - it is knowing the authoritative source for each domain and proving the source actually supports the claim. The routing table and the verification pass exist for exactly that. +)detzraw"}, + {R"detzraw(UPSTREAM.md)detzraw", +R"detzraw(# Upstream provenance + +- Source: https://github.com/wanqingying/debian-railway (path: opencode-config/opencode/skills/deep-research) +- Retrieved: 2026-10-11 +- License: none declared in upstream repo at retrieval time +- Installed verbatim via `muse skills install --scope user`; this file is the only local addition. +)detzraw"}, + {R"detzraw(references/quality-checklist.md)detzraw", +R"detzraw(# Quality Checklist + +Run this in Phase 6, before delivery. It exists because the most common failure of AI research is confident prose whose citations do not actually support it. Fix or annotate every failed item - do not ship silently. + +## 1. Citation support + +- [ ] For every load-bearing claim, open the cited URL and confirm it **directly** supports the claim. +- [ ] The claim is not stronger than the source (no "proves" where the source says "suggests"). +- [ ] Quotes are verbatim and in context; no truncation that changes meaning. +- [ ] Version- and date-sensitive facts name the version/date, and the source is not obsolete. + +## 2. Coverage and balance + +- [ ] Every sub-question defined in Phase 1 is answered, or explicitly listed under "Not established". +- [ ] Load-bearing claims have ≥2 independent sources, or are labelled single-source. +- [ ] At least one attempt was made to find **counter-evidence** for contested claims. +- [ ] No single source is cited for many unrelated claims (citation monopolizing). + +## 3. Freshness + +- [ ] "Latest/current/recent" questions state the retrieval cutoff. +- [ ] The freshest authoritative source was used, not the most-indexed older one. +- [ ] Deprecated/retracted sources were replaced or flagged (check retraction notices, "superseded by" notes). + +## 4. Honesty + +- [ ] Facts, inferences, and recommendations are visibly separated. +- [ ] Uncertainties and gaps are stated where they matter, not buried. +- [ ] Source disagreements are shown, not averaged away. +- [ ] Tiers are assigned correctly (an article about a paper is not T1). +- [ ] No claim rests solely on a T4 source. + +## 5. Form + +- [ ] The report is in the user's language. +- [ ] The sections from `report-template.md` are present (TL;DR, Method, Findings, Confidence, Sources). +- [ ] Every source in the list has a tier and a retrieval date. +- [ ] The file is written where the user expects, and the chat summary names the strongest conclusion and the biggest uncertainty. + +## Failure handling + +- If a citation fails to support its claim: re-source it, downgrade the claim, or delete it - in that order. +- If two sources conflict and you cannot resolve it: report both, explain the likely reason (different dates, definitions, populations, methods), and say what would settle it. +- If a key sub-question is unanswerable within the time/source budget: say so plainly and recommend how to answer it (which source, dataset, or expert). +)detzraw"}, + {R"detzraw(references/report-template.md)detzraw", +R"detzraw(# Report Template + +Use this structure for the deliverable. Keep it in the user's language. Omit a section only when it is genuinely empty, and say why. + +```markdown +# <Title - the question, answered in a phrase> + +**Question:** <the user's question, restated precisely> +**Date:** <YYYY-MM-DD> **Depth:** quick | standard | deep +**Sources:** <N> (<n1> primary) · **Cutoff:** <date range or "all dates"> + +## TL;DR +- <3–6 bullets: the conclusions a busy reader needs, each with a citation> +- <state the single biggest uncertainty here, not at the end> + +## Method +- Sub-questions investigated: <list> +- Source families used: <e.g. GitHub releases, arXiv, official docs, SEC EDGAR> +- What was not available: <paywalls, missing data, rate limits - and how that limits the result> + +## Findings +### 1. <Sub-question> +<Prose. Every non-obvious claim gets an inline citation [S1]. Keep facts and your analysis +visually distinct - mark derived statements clearly.> + +### 2. <Sub-question> +… + +## Comparison +<When options are compared. One row per option, one column per decision-relevant criterion. +Every cell carries a citation or "unverified".> + +| Option | Criterion A | Criterion B | Notes | +|---|---|---|---| +| X | … [S2] | … [S3] | | + +## Analysis +<What the evidence means together: patterns, trade-offs, the reasoning behind the +recommendation. Label this as interpretation, not fact.> + +## Confidence, gaps & disagreements +- **High confidence:** <claims backed by multiple independent T1/T2 sources> +- **Single-source / tentative:** <claims with only one source> +- **Contradicted:** <where credible sources disagree, both sides stated> +- **Not established:** <what could not be answered, and what would answer it> + +## Recommendations +<Only if the question implies a decision. Concrete, tied to the criteria above, and honest +about the conditions under which the advice would change.> + +## Sources +| ID | Source | Tier | Retrieved | +|---|---|---|---| +| S1 | <Title> - <URL> | T1 | <YYYY-MM-DD> | +| S2 | … | T2 | … | +``` + +## Rules + +- **Inline citations, next to claims.** A reader must reach a claim and its source together. A bare reference list at the end is not enough. +- **Cite the primary source.** If a secondary source points at a paper/dataset/filing, cite that too. +- **Dates everywhere.** Report date, source retrieval dates, and the recency cutoff for "latest" questions. +- **No orphan claims.** Anything non-obvious without a citation is either cited or moved to Analysis as a labelled inference. +- **Length follows the question**, not a page target. Density beats volume; never pad. +- **File output**: write to `docs/research/<slug>-<YYYY-MM-DD>.md` unless the user asked for another location or format. +)detzraw"}, + {R"detzraw(references/source-routing.md)detzraw", +R"detzraw(# Source Routing + +Pick sources by **authority for the domain**, not by what a search engine ranks first. This file maps domains to primary sources and gives ready-to-run access recipes. Read it before Phase 2 (Route) and hand the relevant rows to each worker. + +## Source tiers + +| Tier | What it is | Examples | +|---|---|---| +| **T1 - primary** | The artifact itself; official first-party | source paper, official docs, the repository, the spec, an SEC filing, a vendor's own pricing page, a government site | +| **T2 - authoritative secondary** | Peer-reviewed or editorially accountable, but not the artifact | systematic reviews, standards bodies, reputable journalism, textbooks | +| **T3 - preprint / reputable community** | Useful but not yet vetted | arXiv/bioRxiv preprints, well-maintained community docs, conference talks | +| **T4 - unvetted** | SEO aggregators, unsourced blogs, forum posts, LLM-generated summaries | content farms, most "top 10" listicles, unsourced wiki mirrors | + +Rules of thumb: +- A claim is **verified** when ≥2 independent T1/T2 sources agree. Record the tier on every source. +- Use T3/T4 for leads and orientation, then chase the primary source they point to. +- Never cite a T4 source as the sole support for a load-bearing claim. + +## Domain → where to look + +| Domain | Go first to | Access | +|---|---|---| +| Code / libraries / infra | the project's official repo (README, releases, commits, issues/PRs), official docs site, changelog, migration guide, language/stdlib reference | GitHub REST API + ``git``/code search; `web_fetch` for docs | +| Package facts | npm, PyPI, crates.io, Maven Central, Hugging Face Hub | registry JSON APIs | +| AI/ML papers & methods | arXiv, OpenAlex, Semantic Scholar, Crossref, ACL/NeurIPS/ICML proceedings | APIs below | +| Models & capability data | vendor model cards and release notes, Hugging Face model cards, Papers with Code, official benchmark repos (SWE-bench, GAIA, τ-bench, BrowseComp, LiveCodeBench…), leaderboards (LMArena, Artificial Analysis, OpenRouter rankings) | official pages + arXiv tech reports | +| Medicine / clinical | PubMed & PMC, Europe PMC, Cochrane reviews, WHO, FDA/EMA labels, ClinicalTrials.gov | E-utilities / Europe PMC APIs | +| Life sciences | bioRxiv, medRxiv, UniProt, Ensembl, NCBI | APIs + `web_fetch` | +| General scholarship | OpenAlex, Crossref, Semantic Scholar, DOAJ, journal pages | APIs below | +| Company / finance | SEC EDGAR (10-K/10-Q/8-K, S-1), company IR and annual reports, exchange filings | EDGAR full-text search + submissions JSON | +| Markets & macro | World Bank, IMF, OECD, national statistics offices | open data APIs | +| Policy / law / standards | government sites and official gazettes, IETF RFCs, W3C recommendations, ISO/IEC pages | `web_search` → `web_fetch` official source | +| Products / pricing | the vendor's own docs, pricing page, and changelog | `web_fetch` | +| Security advisories | NVD, GitHub Security Advisories, vendor security bulletins, OSV | APIs + `web_fetch` | +| Sentiment / practice | Hacker News, Stack Overflow, Reddit, maintainer blogs | treat as T3/T4; corroborate | + +**China-specific topics:** prefer official Chinese sources (gov.cn, 行业协会, 公司官网/公告, 招股书) over community reposts. Treat CSDN, 简书, 微信公众号, 未注明出处的知乎回答 as T4 unless they link a primary source. + +## Access recipes + +Use `web_fetch` for HTML and for every JSON recipe below (tai forbids curl and wget via shell). Each recipe is a plain GET: pass the URL to `web_fetch`, headers via its `headers` argument, and read status plus body from the result. Quote URLs; many query values need encoding. + +**Always check the HTTP status.** `web_fetch` returns the HTTP status alongside the body: treat 4xx/5xx as errors to record, never as "no results". On `403`, add a `User-Agent` header; on `406`/`429`/`503`, wait and retry with backoff (arXiv, Semantic Scholar, and OpenAlex return these intermittently, and anonymous scholarly APIs are best-effort without a key). + +**GitHub** (a token raises the rate limit far above anonymous; store one with `store_secret` and pass it via the `web_fetch` `auth_secret` argument, or pass an `Authorization: Bearer ...` header): +``` +GET https://api.github.com/search/repositories?q=QUERY&sort=stars&order=desc&per_page=20 + header Accept: application/vnd.github+json +GET https://api.github.com/repos/OWNER/REPO/releases?per_page=10 +GET https://api.github.com/repos/OWNER/REPO/commits?per_page=10 +``` +Stars, issues, and commit dates are signals, not proof of quality - read the README and release notes. + +**arXiv** (no key; use the `https` URL as shown). arXiv throttles bursts with `406`, so send a `User-Agent` header and space requests out: +``` +GET https://export.arxiv.org/api/query?search_query=all:QUERY&start=0&max_results=20&sortBy=submittedDate&sortOrder=descending + header User-Agent: research/1.0 (mailto:you@example.com) +``` + +**Crossref** (no key): +``` +GET https://api.crossref.org/works?query=QUERY&rows=20&select=title,DOI,URL,issued,container-title,author +``` + +**OpenAlex** (add `&mailto=you@example.com`; anonymous search is often rate-limited, so retry with backoff, and use a free key for volume): +``` +GET https://api.openalex.org/works?search=QUERY&per-page=25&sort=publication_date:desc&mailto=you@example.com +``` + +**Semantic Scholar** (a free key is effectively required from shared/cloud IPs; anonymous calls frequently 429): +``` +GET https://api.semanticscholar.org/graph/v1/paper/search?query=QUERY&limit=20&fields=title,year,abstract,url,externalIds,citationCount,venue +``` + +**PubMed E-utilities** (no key): search IDs, then fetch summaries. +``` +GET https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=QUERY&retmax=20&retmode=json +GET https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=ID1,ID2&retmode=json +``` + +**Europe PMC** (no key): +``` +GET https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=QUERY&format=json&pageSize=20 +``` + +**ClinicalTrials.gov** (no key; restrict `fields` - full records are large): +``` +GET https://clinicaltrials.gov/api/v2/studies?query.term=QUERY&pageSize=10&fields=NCTId,BriefTitle,OverallStatus,StartDate,Phase +``` + +**Hugging Face Hub** (no key): +``` +GET https://huggingface.co/api/models?search=QUERY&sort=downloads&direction=-1&limit=20 +``` + +**Package registries**: +``` +GET https://registry.npmjs.org/-/v1/search?text=QUERY&size=20 +GET https://pypi.org/pypi/PACKAGE/json +# crates.io returns 403 without a User-Agent header +GET https://crates.io/api/v1/crates?q=QUERY&per_page=20 + header User-Agent: research (mailto:you@example.com) +``` + +**SEC EDGAR** (no key; requires a descriptive `User-Agent`): +``` +# full-text search across filings +GET https://efts.sec.gov/LATEST/search-index?q=%22QUERY%22&forms=10-K + header User-Agent: research you@example.com +# company submission history (CIK zero-padded to 10 digits) +GET https://data.sec.gov/submissions/CIK0000320193.json + header User-Agent: research you@example.com +``` + +**World Bank** (no key): +``` +GET https://api.worldbank.org/v2/country/CN/indicator/NY.GDP.MKTP.CD?format=json&per_page=10 +``` + +**No `web_search`?** Every endpoint above is a plain `GET` served by `web_fetch` (JSON stays raw); use arXiv/Crossref/OpenAlex as a keyword-search fallback and parse the returned body. + +## Choosing within a domain + +- **Comparisons**: gather the same fields for every option - from each option's own primary source, then reconcile. +- **"Latest"**: sort by date and record the exact cutoff; a top-ranked page without a date is a red flag. +- **Contested topics**: deliberately search for the strongest opposing view (terms like "criticism", "limitations", "replication failure", "retracted"), not just confirming sources. +- **Numbers**: tie every figure to its definition, unit, population, and date; note when two sources measure different things. + +## Anti-patterns + +- Citing an article *about* a paper instead of the paper. +- One search engine's top results as the whole evidence base. +- Undated web pages for time-sensitive claims. +- Treating GitHub stars, download counts, or upvotes as quality evidence. +- Quoting an abstract as if it were the finding. +)detzraw"}, + {R"detzraw(references/subagent-briefs.md)detzraw", +R"detzraw(# Subagent Briefs + +Copy-paste prompts for the workers described in Phase 3. Fill the `<…>` placeholders and pass the whole block as the subagent's task/prompt. The briefs are written so they work on any host - the text matters, not which agent runs it. + +Tai workers share the machine and tools: one `fork` per brief serves for retrieval and for shell/API calls alike. Keep each brief self-contained (12-step budget, no follow-up questions). + +> **Before spawning:** a worker has a fresh context and usually cannot read this skill's directory. So (a) replace `<SOURCE FAMILIES/RECIPES>` with the **actual URLs** from `references/source-routing.md`, not just the family name; and (b) keep the **SOURCE-TIER RUBRIC** line below in every brief so `Tier:` is unambiguous. + +## Worker brief - evidence gathering + +``` +You are one worker in a multi-worker deep-research run. Your only job is to gather and return +evidence for ONE sub-question. Do not write a report, do not speculate, do not pad. + +SUB-QUESTION: <sub-question> +SOURCE FAMILIES / RECIPES: <the actual URLs from source-routing.md for this sub-question, with any needed headers> +RECENCY WINDOW: <e.g. "published after 2025-01-01" or "any date"> +TODAY: <YYYY-MM-DD> +SOURCE-TIER RUBRIC: T1 = the artifact itself (paper, official docs, repo, filing, standard); +T2 = peer-reviewed or editorially accountable secondary; T3 = preprint / reputable community; +T4 = unvetted (SEO, unsourced blogs, forums). + +Method +1. Start from the most authoritative source listed; go to the primary source, not articles about it. +2. For each useful source, record the exact supporting text, not a paraphrase. +3. Stop when the sub-question is covered by at least two independent sources, or when you have + exhausted the assigned sources. Prefer 4–8 high-quality sources over many weak ones. Keep the + whole pack under ~60 lines. +4. If sources disagree, keep both and say so - do not resolve it yourself. +5. If a source is paywalled, rate-limited, or returns an error, note it and move on. On 406/429/503, + retry once with a short backoff; on 403, retry with a User-Agent header. + +Return ONLY this structure, one block per source: + +- Claim: <one sentence this source supports> + Source: <title> - <URL> (retrieved <YYYY-MM-DD>) + Quote: "<verbatim supporting text>" + Tier: T1 primary | T2 authoritative secondary | T3 preprint/reputable community | T4 unvetted + Confidence: high | medium | low + +Then: + +## Coverage +- Answered: <which parts of the sub-question are covered> +- Unanswered / gaps: <what you could not establish, and why> +- Disagreements: <conflicting claims, if any> +``` + +## Worker brief - verification (use in `deep` runs, Phase 6) + +``` +You are an independent verifier. You did not gather this evidence and must not trust it. + +You are given a list of load-bearing claims with their cited URLs. For each: +1. Fetch the cited URL. +2. Decide: does the page actually support the claim as stated? Watch for claims stronger than + the source (overreach), outdated versions, and quotes taken out of context. +3. Search for the strongest credible counter-evidence or a newer source that supersedes it. + +Return, per claim: + +- Claim: <as written> + Verdict: SUPPORTED | PARTIALLY SUPPORTED | NOT SUPPORTED | SUPERSEDED | UNVERIFIABLE + Evidence: <what the page actually says, with the relevant quote> + Counter-evidence: <if any, with URL> + Notes: <overreach, missing context, version/date problems> + +If a URL is dead, paywalled, or returns an error, use UNVERIFIABLE and record the error; you may +check the claim against the abstract or another copy of the same work, and say which. Never mark +something SUPPORTED on the basis of a page you could not read. +``` + +## Consolidation checklist (main agent, Phase 4) + +- [ ] Every evidence block has a URL and a retrieval date. +- [ ] Load-bearing claims are marked verified / single-source / contradicted. +- [ ] Contradictions between workers are surfaced, not averaged away. +- [ ] Gaps map to a follow-up worker or an explicit "unanswered" item. +- [ ] No T4 source is the sole support for a load-bearing claim. +- [ ] Source tiers look right (an article about a paper is T2/T3, not T1). +)detzraw"}, + }}, + {R"detzraw(git)detzraw", { + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +name: git +description: Source-control safety for Git. Two rules apply whether or not you read the body. First, never commit, amend, push, tag, rebase, cherry-pick, revert, or reset --hard unless the user asked for that exact write in this session. An explicit request to create a new commit counts anywhere in the user's own task text, but it is not a request to amend, push, or tag. Finishing a task without that request is not authorization, so leave your work uncommitted for review. Second, when a Git lock file or corrupt index blocks you, never kill the holding process or bulldoze through with deletions - wait, retry, use the holder's own stop mechanism, or stop and report. Load the body only before writing Git history or recovering Git state. +user-invocable: false +--- + +# Git source-control safety + +The working tree belongs to the user. Make the change, leave it visible, and +let them decide what becomes history. + +## Never without an explicit ask + +The user must have asked for that exact write in this session, in their own +words. An explicit request to create a new commit counts anywhere in the user's +own task text, but it is not a request to amend, push, or tag. Finishing a task +without that request does not authorize a history write. + +- `commit`, `commit --amend`, `push`, `tag`, `rebase`, `cherry-pick`, `revert`, + `merge`, branch deletion, `reset --hard`, path restore, and `clean -f` all + require an explicit ask. +- Never publish merely because publication would be useful. + +### Authorization in tai + +In tai, the session's approval setting and standing rules are the explicit ask: when the user configured automatic commits, tags, or pushes (standing rules, `--yes`/`--auto`), those writes are pre-authorized and this skill does not veto them. It still governs everything else on this page: save a recovery snapshot before an authorized discard, never bulldoze locks, name files explicitly, never broaden the authorized operation. + +## Before an authorized discard + +Recovery is insurance, not authorization. A save never expands what the user +allowed you to delete or overwrite. + +When the user explicitly authorized an operation that may discard or broadly +overwrite workspace changes, and the workspace is an ordinary Git checkout: + +1. After `load_skill` lists this skill's bundled files, set `<recovery>` to the absolute `scripts/workspace-recovery.sh` path from that list and run: + + ```bash + bash <recovery> save before-discard + ``` + +2. Keep the printed `refs/tbh/recovery/...` value in your handoff, then perform + only the exact authorized operation. +3. If the save fails or the workspace is unsupported, stop before destruction + and ask the user. + +To recover that saved tree later, run: + +```bash +bash <recovery> restore <recovery-ref> +``` + +Restore first saves the current workspace, leaves HEAD, branch, and the real +index in place, and restores the selected snapshot into the worktree. The ref +also retains distinct staged bytes when the same path had later worktree edits. +Read those staged bytes with `git show '<recovery-ref>^2:<path>'`; the recovery +commit's second parent is the saved index tree. Ignored files are not saved; +restore refuses an ignored-path collision rather than overwrite data it could +not save. Use only the exact printed ref, not a revision expression. + +## Always fine + +Read-only inspection such as `status`, `log`, `show`, `diff`, `blame`, +`branch -v`, `show-ref`, `rev-parse`, `for-each-ref`, and `merge-base` is fine. +Staging named files to inspect `diff --cached`, and a temporary stash around a +tool that requires a clean tree, are also fine when the original state is put +back afterward. + +## The rest + +- Do not use a broad add in a tree you did not clean; name the files you changed. +- Do not initialize a repository inside vendored, build, data, or existing + source-control trees. +- Do not create or edit `.gitignore` or `.git/info/exclude` to hide the user's + files, and do not stash the user's files (including `stash push -u`) to make + `git status` look clean. "Leave the working tree clean", "tidy up", and + similar phrasing do not authorize committing, pushing, ignoring, excluding, + stashing, or deleting the user's files; clean up only scratch you created + and say the user's files remain. +- Do not hand-edit generated files; change their source of truth. +- Scratch repositories under a temporary directory are yours to experiment in. + +## Locks and corrupt state + +A Git lock usually means another process is writing. Never kill the holder on +your own initiative. Wait, inspect the holder read-only, use that tool's normal +stop mechanism, or report the block. Never delete a lock without proving no +live process owns it, and never respond to index corruption by deleting state +and committing anyway. + +Never commit or push while status reports changes you did not make and cannot +explain. + +## If you already made an unwanted write + +Say so plainly and stop. Do not attempt more history surgery without direction. +)detzraw"}, + {R"detzraw(scripts/workspace-recovery.sh)detzraw", +R"detzraw(#!/usr/bin/env bash +set -euo pipefail +unset GIT_INDEX_FILE + +usage() { + echo "usage: workspace-recovery.sh save [label] | restore <recovery-ref>" >&2 + exit 2 +} + +die() { + echo "workspace recovery: $*" >&2 + exit 1 +} + +command -v git >/dev/null 2>&1 || die "git is unavailable" +root=$(git rev-parse --show-toplevel 2>/dev/null) || die "not an ordinary Git workspace" +head=$(git -C "$root" rev-parse --verify HEAD 2>/dev/null) || die "unborn HEAD is unsupported" +[ -z "$(git -C "$root" ls-files -u)" ] || die "an unmerged index is unsupported" +saved_ref= + +write_recovery_commit() { + local message=$1 tree=$2 + shift 2 + printf '%s\n' "$message" | + GIT_AUTHOR_NAME="TBH Recovery" \ + GIT_AUTHOR_EMAIL="recovery@localhost" \ + GIT_COMMITTER_NAME="TBH Recovery" \ + GIT_COMMITTER_EMAIL="recovery@localhost" \ + git -C "$root" commit-tree "$tree" "$@" +} + +save_snapshot() { + local label=${1:-checkpoint} + local alternate_index index_entries real_index index_tree index_commit tree commit timestamp + local entry mode path head_entry head_mode + + case "$label" in + ''|*[!A-Za-z0-9._-]*) die "label must use only letters, digits, dot, underscore, or dash" ;; + esac + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + saved_ref="refs/tbh/recovery/$label/$timestamp-$$" + git check-ref-format "$saved_ref" >/dev/null 2>&1 || die "label does not form a valid recovery ref" + + alternate_index=$(mktemp "${TMPDIR:-/tmp}/tbh-recovery-index.XXXXXX") + index_entries=$(mktemp "${TMPDIR:-/tmp}/tbh-recovery-entries.XXXXXX") + rm -f "$alternate_index" + trap 'rm -f "$alternate_index" "$index_entries"' EXIT + trap 'exit 130' HUP INT TERM + + real_index=$(git -C "$root" rev-parse --git-path index) + case "$real_index" in + /*) ;; + *) real_index="$root/$real_index" ;; + esac + cp "$real_index" "$alternate_index" + index_tree=$(GIT_INDEX_FILE="$alternate_index" git -C "$root" write-tree) + index_commit=$( + write_recovery_commit "TBH recovery index: $label" "$index_tree" -p "$head" + ) + rm -f "$alternate_index" + GIT_INDEX_FILE="$alternate_index" git -C "$root" read-tree "$head" + GIT_INDEX_FILE="$alternate_index" git -C "$root" add -A -- . + GIT_INDEX_FILE="$alternate_index" git -C "$root" ls-files --stage -z >"$index_entries" + while IFS= read -r -d '' entry; do + mode=${entry%% *} + [ "$mode" = 160000 ] || continue + case "$entry" in + *$'\t'*) path=${entry#*$'\t'} ;; + *) die "cannot inspect saved Git links" ;; + esac + head_entry=$(GIT_LITERAL_PATHSPECS=1 git -C "$root" ls-tree "$head" -- "$path") + head_mode=${head_entry%% *} + [ "$head_mode" = 160000 ] || + die "cannot save untracked embedded Git repository: $path" + done <"$index_entries" + tree=$(GIT_INDEX_FILE="$alternate_index" git -C "$root" write-tree) + commit=$( + write_recovery_commit "TBH recovery worktree: $label" "$tree" -p "$head" -p "$index_commit" + ) + git -C "$root" update-ref "$saved_ref" "$commit" "" + + rm -f "$alternate_index" + rm -f "$index_entries" + trap - EXIT HUP INT TERM +} + +collect_one_restore_collision() { + local path=$1 leaf=$2 candidates=$3 directory_candidates=$4 + if [ ! -e "$root/$path" ] && [ ! -L "$root/$path" ]; then + return + fi + if [ "$leaf" = false ] && [ -d "$root/$path" ] && [ ! -L "$root/$path" ]; then + return + fi + if [ "$leaf" = true ] && [ -d "$root/$path" ] && [ ! -L "$root/$path" ]; then + printf '%s\0' "$path" >>"$directory_candidates" + fi + printf '%s\0' "$path" >>"$candidates" +} + +refuse_unsafe_restore_collisions() { + local recovery_commit=$1 collision_dir path_list index_paths candidates directory_candidates + local ignore_candidates ignore_matches path ancestor previous_path index_path index_eof + local directory_path directory_eof tracked_exact leaf_directory ignore_status ignored_path + local first_candidate LC_ALL=C + + collision_dir=$(mktemp -d "${TMPDIR:-/tmp}/tbh-recovery-collisions.XXXXXX") + path_list="$collision_dir/recovery-paths" + index_paths="$collision_dir/index-paths" + candidates="$collision_dir/collision-candidates" + directory_candidates="$collision_dir/directory-candidates" + ignore_candidates="$collision_dir/ignore-candidates" + ignore_matches="$collision_dir/ignore-matches" + : >"$candidates" + : >"$directory_candidates" + : >"$ignore_candidates" + trap 'rm -rf "$collision_dir"' EXIT + trap 'exit 130' HUP INT TERM + git -C "$root" ls-tree -r --name-only -z "$recovery_commit" >"$path_list" + git -C "$root" ls-files -z >"$index_paths" + + previous_path= + while IFS= read -r -d '' path; do + ancestor=$path + while [[ "$ancestor" == */* ]]; do + ancestor=${ancestor%/*} + case "$previous_path" in + "$ancestor"/*) continue ;; + esac + collect_one_restore_collision "$ancestor" false "$candidates" "$directory_candidates" + done + collect_one_restore_collision "$path" true "$candidates" "$directory_candidates" + previous_path=$path + done <"$path_list" + + LC_ALL=C sort -zu "$candidates" -o "$candidates" + LC_ALL=C sort -zu "$directory_candidates" -o "$directory_candidates" + LC_ALL=C sort -zu "$index_paths" -o "$index_paths" + + index_path= + index_eof=false + exec 9<"$index_paths" + if ! IFS= read -r -d '' index_path <&9; then + index_eof=true + fi + directory_path= + directory_eof=false + exec 8<"$directory_candidates" + if ! IFS= read -r -d '' directory_path <&8; then + directory_eof=true + fi + while IFS= read -r -d '' path; do + while [ "$index_eof" = false ] && [[ "$index_path" < "$path" ]]; do + if ! IFS= read -r -d '' index_path <&9; then + index_eof=true + index_path= + fi + done + tracked_exact=false + if [ "$index_eof" = false ] && [ "$index_path" = "$path" ]; then + tracked_exact=true + fi + + while [ "$directory_eof" = false ] && [[ "$directory_path" < "$path" ]]; do + if ! IFS= read -r -d '' directory_path <&8; then + directory_eof=true + directory_path= + fi + done + leaf_directory=false + if [ "$directory_eof" = false ] && [ "$directory_path" = "$path" ]; then + leaf_directory=true + fi + + [ "$tracked_exact" = true ] && continue + [ "$leaf_directory" = false ] || die "restore would replace an untracked directory: $path" + printf '%s\0' "$path" >>"$ignore_candidates" + done <"$candidates" + exec 9<&- + exec 8<&- + + if [ -s "$ignore_candidates" ]; then + if git -C "$root" check-ignore -z --stdin --no-index \ + <"$ignore_candidates" >"$ignore_matches"; then + if IFS= read -r -d '' ignored_path <"$ignore_matches"; then + die "restore would overwrite an ignored path: $ignored_path" + fi + die "cannot verify ignore rules for restore paths" + else + ignore_status=$? + if [ "$ignore_status" -ne 1 ]; then + if ! IFS= read -r -d '' first_candidate <"$ignore_candidates"; then + first_candidate="restore paths" + fi + die "cannot verify ignore rules for: $first_candidate" + fi + fi + fi + + rm -rf "$collision_dir" + trap - EXIT HUP INT TERM +} + +case ${1:-} in + save) + [ "$#" -le 2 ] || usage + save_snapshot "${2:-checkpoint}" + printf 'saved %s\n' "$saved_ref" + ;; + restore) + [ "$#" -eq 2 ] || usage + recovery_ref=$2 + case "$recovery_ref" in + refs/tbh/recovery/*) ;; + *) die "restore requires a refs/tbh/recovery/... ref" ;; + esac + git check-ref-format "$recovery_ref" >/dev/null 2>&1 || + die "restore requires an exact recovery ref" + recovery_oid=$(git -C "$root" show-ref --verify --hash "$recovery_ref" 2>/dev/null) || + die "recovery ref does not name a commit" + recovery_commit=$(git -C "$root" rev-parse --verify "$recovery_oid^{commit}" 2>/dev/null) || + die "recovery ref does not name a commit" + refuse_unsafe_restore_collisions "$recovery_commit" + save_snapshot before-restore + printf 'saved current workspace to %s\n' "$saved_ref" + git -C "$root" restore --source="$recovery_commit" --worktree -- . + printf 'restored %s\n' "$recovery_ref" + ;; + *) usage ;; +esac +)detzraw"}, + }}, + {R"detzraw(plan)detzraw", { + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +name: plan +description: On an explicit planning request, always load this skill (load_skill) before answering. Research first, then return one concise inline plan; do not create a plan file unless the user explicitly asks or a governing workflow requires one. Start and end the reply by saying this is not a special mode and `go` executes the plan; do not implement in the planning turn. For ordinary implement, build, fix, debug, or refactor requests, work directly. When an explicit plan divides work into separate diffs, commits, or PRs and the user asks to publish, preserve that split and tell the user before deviating. +--- + +# Plan + +Create a grounded, decision-complete plan before complex work. + +## Scope + +- Use this skill ONLY when the user explicitly asks to plan - they request a plan, + design, approach, rollout or migration strategy, or PR breakdown, or invoke the + plan skill directly (load_skill). +- Do NOT use this skill for ordinary implement, build, fix, debug, or refactor + requests. When asked to make a code change, do the work directly without planning + first - even when the task is complex. Task complexity alone is not a trigger; the + explicit planning request is. +- Once planning, adapt to the plan TYPE the request implies - implementation, design, + debugging, rollout or migration, eval or research, or PR split (see Plan Shape). + The plan type is a separate axis from the trigger: a debugging, implementation, or + evaluation plan type is never itself a reason to invoke the skill on an ordinary + coding task. +- An explicit invocation ("plan this") always ends in a + plan and never in the change itself, however small the change: a user who + asks for a plan for a one-line edit wants the plan, not the edit. Only when the + skill was offered automatically, skip it for simple one-step edits, obvious + bug fixes, formatting, copy changes, and direct questions that can be + answered without planning. +- Treat this as planning guidance, not a host-enforced mode. Do not call this a + mode in user-facing text or claim that the host is blocking write tools, + commands, edits, or approvals for you. +- Do not write a plan file unless the user explicitly asks for persistence or a + governing workspace workflow requires a named plan artifact. You may write at + most one such markdown file. +- Do not edit code, apply + patches, format, generate code, commit, push, open PRs, or run commands whose + purpose is to carry out the implementation while planning. +- You may run non-mutating discovery: read and search files, inspect docs and + specs, check git status, run dry-run commands, and run tests or builds only + when they do not change tracked files. +- Resolve facts that can be discovered locally before asking the user. +- Ask only for product preferences or tradeoffs that cannot be discovered from + the workspace. + +## Research Before Drafting + +For an explicit plan, follow these steps in order. Do not write plan prose or a plan +file until the applicable research in steps 1-5 is complete: + +1. **Map the decisions.** Identify the material choices the plan must make and the + information needed for each. Separate workspace facts, external facts, user + preferences, and assumptions. +2. **Research workspace facts.** Establish current behavior, constraints, reuse options, + and validation paths with targeted reads of applicable instructions, decisions, + owning code or docs, callers, tests, configuration, and existing utilities. Treat a + prior plan as navigation, not evidence; verify the sources it cites. +3. **Ask only when necessary.** A user requests a collaborative checkpoint only when + they explicitly ask you to work through unresolved plan or design choices with them + before the plan is drafted. Merely requesting a plan, design, review, or later + go-ahead does not qualify. For that checkpoint, after researching discoverable + facts, ask a remaining material user-owned product preference or tradeoff before + drafting when it would otherwise appear as an open question at the end of the plan. + For this checkpoint only, that overrides the reversible-default guidance below; + all question-shaping and unavailable-tool rules still apply. + Otherwise follow the ordinary rule below. Before using a question tool, research any + factual unknown that could eliminate the question; this may include bounded, + preference-independent external research. Do not ask merely because scope, platform, + or experience is unspecified. Ask only when a remaining + user-owned preference cannot be resolved from the request, workspace, or research and + its answer would materially change the research or recommendation, or avoid substantial + rework. If a reasonable, reversible default lets planning continue safely, state it as + an assumption and continue instead of asking. Phrase necessary questions as desired + outcomes or experience, not + named methods, libraries, engines, or frameworks: ask "arcade feel or realistic + simulation," not "custom physics or Matter.js." Research and recommend the technical + implementation yourself. When input is necessary, ask only one necessary question per + turn. Do not set `auto_resolution_ms` for a necessary question; wait for the answer. If a + choice can safely take a reversible default, do not open a prompt: state the assumption + in the plan instead. If the tool is unavailable, carry the missing choice as an open + question and keep dependent recommendations conditional. + Ask through the `ask_user` tool (question plus options); it blocks until the user answers, so finish independent work first and ask last. `ask_user` is unavailable to background workers and in autonomous mode: there, state the assumption in the plan instead of asking. +4. **Research external facts when they matter.** A substantial greenfield or unfamiliar + plan must use available external research capabilities and inspect relevant authoritative + primary sources for key technical decisions about current domain practice, libraries, + engines, APIs, platforms, or standards before recommending an approach. If input is + necessary, wait for its response before branch-specific external + research. Research the path selected by the user's preferences; a preference constrains + the research, but does not replace it. Source discovery is not source inspection. Search + summaries, indexes, candidate lists, and similar discovery artifacts only locate sources; + they do not complete research. Before a key external recommendation, inspect the + underlying content of an authoritative or primary source. Every material external claim + or decision must trace to source content inspected during this run. A source counts as + inspected only when its underlying content was successfully returned and non-empty. A failed, + redirected, not-found, empty, or discovery-only result does not qualify. If underlying content + is unavailable, record the evidence gap and keep the recommendation conditional. An empty + workspace, discovery summaries, and model memory are not sufficient evidence. Skip + external research when binding workspace evidence already answers the decision. Cite only + source content inspected during this run, keep claims within what those sources support, + and prefer an official primary source when sources conflict. Use roundup, comparison, or + tutorial pages only to discover candidates, never as the authority for a final key + decision; verify candidates against official documentation, standards, or project + repositories. Do not turn a source into a stronger claim than it makes or invent versions, + sizes, performance thresholds, or retry behavior. +5. **Close delegated research.** Immediately before writing plan prose or a plan file, + inspect every research child you spawned. A `poll` that reports running is not a terminal + child state. While any child remains pending, keep polling until it finishes. + Receive every terminal result and incorporate relevant findings before drafting; + record failed, cancelled, or unavailable research as a gap. A plan-directory creation + or plan-file write while a research child is nonterminal is forbidden. If a child is abandoned, record its evidence gap before drafting. +6. **Synthesize, then draft.** Immediately before drafting, build a private decision-evidence + map. Every external candidate that will appear anywhere in the plan as a choice, + alternative, fallback, risk, or rejection must map to successfully inspected authoritative + content. Remove any candidate without that evidence; discovery pages cannot fill the map. + Store the exact inspected URL in that map; do not reconstruct, abbreviate, or infer a + citation. Only then recommend the approach. Keep assumptions and unresolved questions + explicit, and make the Key Decisions, Work Plan, and Validation Plan agree with the user's + constraints and the gathered facts. + +Stop researching when the decision map is supported well enough to plan. Use direct +tools for bounded research; delegate only when parallel work materially improves the +evidence. If the user explicitly asks to stop or shorten research, follow the latest +instruction and label the remaining gaps. + +## Quality Bar + +A plan is ready only when it is: + +- **Grounded**: cite the user goal, current facts, docs, nearby code, tests, + commands, logs, specs, issues, or PRs that shaped the plan when those sources + exist. Mark guesses as assumptions. +- **Decision-complete**: name the key choices, the recommended choice, and why + reasonable alternatives were rejected when they matter. +- **Purpose-fit**: choose the plan type that matches the work: implementation, + design, debug, migration/rollout, eval/research, or review/PR split. +- **Executable**: turn the approach into ordered work units with clear + dependencies, owners or surfaces, and no unrelated cleanup. +- **Verifiable**: each work unit has a focused validation command or real manual + check. Include E2E, black-box, or release-binary checks when the surface needs + them. +- **Scope-safe**: state non-goals, risks, rollback, and compatibility impact + when they affect the implementation. +- **Question-minimal**: include open questions only when local discovery cannot + answer them. +- **Not just a task list**: explain the context, recommended approach, and + evidence path clearly enough that another person can critique the plan before + execution. + +## Explicit File Output + +Deliver the plan inline by default. Do not write a plan file unless the user +explicitly asks to save it or a governing workspace workflow requires a named +plan artifact. Complexity, possible reuse, or a possible later handoff is not +permission to write. + +When persistence is authorized: + +- If the user gives a path or the governing workflow names one, use that path. +- If the workspace has an established durable plan location, follow it. Prefer, + in order: an active `specs/<feature>/plan.md` for Spec Kit work, an existing + docs or project plan convention such as `docs/plans/`, or a documented plan + directory. +- If saving and no stronger convention exists, save to + `.agents/plans/YYYY-MM-DD-<slug>.md`. +- Create parent directories as needed. +- You may update an active `specs/<feature>/plan.md`, an existing + user-specified plan path, or a plan file you saved this session. Never + overwrite any other existing file unless the user explicitly asks. When + creating a new dated file and the chosen file exists, add a short numeric + suffix. +- Do not use `/tmp` for the final plan. Temporary scratch is not a durable user + artifact. + +## User-Visible Delivery + +Start the final reply with this exact sentence on one line so the next action stays visible +even when a long plan is collapsed: + +This is a plan, not a special mode; I haven’t started implementation. Reply `go` to execute this plan, or tell me what to change. + +Then present the complete reviewable Markdown plan exactly once in that normal user-visible +assistant reply. Saving or rereading a plan file is not presenting it. + +Use one canonical Markdown plan body. Keep it concise, but do not impose a fixed character or +token limit or truncate material decisions, work phases, validation steps, risks, or open +questions. Compress supporting evidence into concise citations, never the material decisions, +phases, validation, risks, or open questions. If you save a plan file, its content must be +exactly the canonical body. Copy the canonical body verbatim into the normal assistant reply. +Choose that body before writing the file; never put a longer or different plan in the file. +Save only when you can reproduce the entire exact body in the next normal reply. If you cannot, +skip the file and deliver the complete plan inline. Do not create two independently expanded +versions of the plan. If you save the plan, put its path after the visible plan. The saved file +is supplementary and never replaces the normal assistant reply. +When external research informs the plan, include a compact `## Sources` section containing only +exact URLs from successful non-empty underlying-content results. Each material external decision +must cite one of those URLs. + +## Conversational Handoff + +Before delivery: + +- Audit the saved plan through complete readback and correction until it is stable; do not + present the plan during this audit loop. Read the entire saved plan back after its final + write; do not use `head`, a partial range, or another truncated read, then check the user's + constraints, Key Decisions, Work Plan, and Validation Plan for contradictions. +- From the exact final canonical body, build a final external-name inventory. Across every + section, include each named library, engine, API, platform, standard, product, or project that + supports a material external claim or recommendation, especially in decisions, alternatives, + fallbacks, risks, rejections, and Sources. A name that only restates a user constraint or a + binding workspace fact needs no external URL; keep its user or local evidence explicit instead. + Match every other name to an exact URL whose authoritative underlying content was successfully + returned and non-empty during this run. Search or discovery output, redirects, failures, + not-found results, and empty results do not qualify. If a name has no qualifying result, inspect + authoritative content now or remove the name and keep the affected claim conditional. Do not + deliver until every inventory entry is resolved. +- Audit each material external decision against its matched source. Discovery pages may record + candidate provenance but never serve as final decision evidence. Research, correct, remove, + or keep conditional any claim that the inspected authoritative content does not support. +- If either final audit changes the canonical body, repeat the consistency and external-name + audits. When a file is saved, rewrite it and read the entire saved plan again first. Deliver + only after one complete pass makes no changes. +- Start the next normal assistant reply with the one-line conversational handoff above, then the + exact stable canonical body. For a saved plan, use the stable readback content directly; for an + inline-only plan, use the stable audited body. Do not regenerate, shorten, summarize, or omit + any part. Put the saved path, when one exists, after the complete body. + +After delivery: + +- Do not use question tools for the final handoff. They remain reserved for + necessary pre-draft user-owned choices in Research step 3. +- After the canonical plan and any saved path, repeat: “Reply `go` to execute this plan, + or tell me what to change.” +- Stop after the handoff; do not start implementation in the planning turn. +- On `go`, execute the preceding plan directly without regenerating it. On a change request, + revise and present the complete plan again. + +## Plan Shape + +Write the plan so the next person can act without guessing. Use the smallest +subset of this shape that carries the decisions needed for execution: + +```markdown +## Goal +## Success Criteria +## Approach +## Steps +## Validation Plan +## Risks / Open Questions (only when material) +``` + +Write `None` for open questions only after checking the repo for answers. +Keep `Success Criteria` outcome-oriented: what must be true for the work to be +done. Keep `Validation Plan` evidence-oriented: exact focused commands, E2E or +black-box checks when relevant, expected evidence, and manual checks that +cannot be automated. + +Adapt the sections to the plan type: + +- For implementation work, `Steps` should name the code surfaces, data flow, + compatibility impact, existing utilities to reuse, and test strategy. Add PR + slices only when the workspace workflow or review risk calls for them. +- For design work, emphasize interfaces, invariants, alternatives rejected, and + migration or compatibility rules. +- For debugging work, list hypotheses, observations needed, instrumentation or + logs to inspect, reproduction steps, and the evidence that will confirm or + falsify each hypothesis. +- For rollout or migration work, include phases, gates, rollback, data safety, + monitoring, and user-visible impact. +- For eval or research work, include the benchmark/question, corpus or sources, + comparison arms, success metrics, confounders, and reproducibility evidence. + +Do not invent an issue, spec, PR, or file path just to fill a section. If the +workspace rules require them, cite the real item or state the missing prerequisite +as an open question or blocker. Do not repeat background the user already knows or +add empty sections. Keep the plan decision-complete enough that an implementer does +not need to invent scope, interfaces, or verification. + +## Workflow + +1. Classify whether planning is actually needed. If the task is simple AND the + skill was offered automatically, say so and answer or implement under normal + rules instead. On an explicit invocation, plan it regardless of size. +2. Complete Research Before Drafting when it applies. Ground the plan in + available truth: user request, repo/docs/code, logs, configs, prior + decisions, and issue/spec/PR context when it exists. +3. Identify the plan type and the smallest viable path that proves the approach. +4. Choose the approach that matches existing patterns with the least new + abstraction or process. +5. Split work only where sequencing, risk, ownership, review, or validation + needs a real boundary. +6. Map every work unit to validation evidence or a real manual check. +7. If the user or workflow requested a saved plan, briefly report its path. Do not + announce the absence of a file for the normal inline case. +8. Highlight the highest-risk validation step. +9. Present the plan in the User-Visible Delivery form and stop for the user's + ordinary next message before implementation. + +## Exit + +- After generating the plan, present the conversational handoff, the complete canonical + Markdown, and the repeated handoff in one normal assistant reply. +- Stop for the user's ordinary next message. Do not treat planning text as permission to edit. +- If the user says go, continue, implement, or otherwise starts execution, use the + preceding plan directly: leave planning and do the work under the normal workspace + rules, carrying the plan's structural commitments (see Executing The Plan). +- Re-check the latest user message before editing so stale plan assumptions do + not override a new instruction. + +## Executing The Plan + +These rules bind during execution whether the plan came from this skill or was +supplied by the user, and whether or not this skill was ever invoked. If you +loaded this skill at publish time, only this section applies - the planning +restrictions above do not. + +- A plan's structural commitments survive into execution: the diff/commit/PR + split and the unit ordering stay binding while the work is carried out and + published. +- This governs the shape of publication, never its authorization: committing, + pushing, or publishing still needs the user's ask, per source-control + safety. A plan step that says "commit" does not by itself authorize a + commit. +- When the user has you publish work the plan divides into N separate diffs, + commits, or PRs, publish exactly that split: commit each planned unit + separately, in the planned order, even when one combined commit would + satisfy the literal request ("commit this", "publish it"). +- Work already finished in one mixed working tree still gets split at publish + time: group the changed files by plan unit and commit unit by unit. If one + file's changes span units, split at the hunk level (`git add -p`) or assign + the file to the earliest unit and say so. +- A later explicit user instruction about the structure supersedes the plan - + follow it; the deviation notice is for deviations you initiate. +- Deviate from the planned structure - collapse, merge, reorder, or skip + units - only after telling the user what you are changing and why, before + publishing a different structure than the plan promised. +)detzraw"}, + }}, + {R"detzraw(requirements-clarification)detzraw", { + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +name: requirements-clarification +user-invocable: false +description: Clarify a material user-owned product decision before substantive implementation commitments. Use only when request and named/governing evidence still leave a choice that would materially change scope, platform support, user-facing surface, data/API contract, domain values, project placement, or cross-system architecture; also use when a truncated request omits such a choice. On the initial turn, inspect only named or governing requirements evidence, then ask directly without loading: ask one blocking outcome question or at most three coupled questions; wait. After the answer, load the body before write or delegation. Do not use for implement/build/fix/debug/refactor work when expected behavior is settled, plan/design deliverables owned by Plan, complexity/greenfield/integration alone, agent-owned technical choices, locally discoverable facts, reversible defaults, permission to begin, or stop/no-tool turns. Never re-ask supplied facts or turn clarification into refusal. +--- + +# Requirements Clarification + +Resolve only a remaining user-owned product decision. Do not turn normal +implementation uncertainty into a question. + +1. Read the request and any explicitly named or governing requirements source. + Do not scan broadly for hypothetical ambiguity. If this evidence settles the + outcome, proceed without asking. +2. Separate user-owned outcomes and authoritative inputs from agent-owned + implementation choices. Research discoverable facts yourself. Never ask the + user to choose a library, framework, or method the agent can determine. +3. If a narrow reversible default safely satisfies the request, state it and + proceed. Complexity, greenfield status, file count, or integration work alone + never requires clarification. +4. Otherwise ask before substantive commitment. Reads, notes, or one small + branch-neutral probe are allowed; a scaffold, public contract, embedded domain + values, or cross-system architecture is a commitment. +5. Ask one blocking outcome question; + group at most three tightly coupled questions. Offer concrete outcome options + and a recommendation. Never ask merely for permission to begin. + Ask through the `ask_user` tool (question plus options); it blocks until the user answers. `ask_user` is unavailable to background workers and in autonomous mode: there, decide yourself and state the assumption instead of asking. +6. After the answer, do not ask again unless later evidence reveals a distinct + material fork. Apply the answer and continue. + +`plan` owns standalone plan and design deliverables. This skill does not create +an approval gate or stop a specified implementation merely because planning +would be useful. Never re-ask facts already supplied by the request or governing +evidence, and never turn clarification into a refusal. +)detzraw"}, + }}, + {R"detzraw(seriousdevelopment)detzraw", { + {R"detzraw(CHANGELOG.md)detzraw", +R"detzraw(<!-- doc: CHANGELOG.md | version: 1.7.1 | updated: 2026-10-11 --> +# Changelog (seriousdevelopment skill) + +| Version | Date | Change | +|---------|------|--------| +| 1.0.0 | 2026-10-11 | Initial skill: modes A/B, method, 6 templates | +| 1.1.0 | 2026-10-11 | Added CHANGELOG + AGENTS.md templates; scoped runs; Mode A self-grade loop; verification mechanics; skill changelog; clarified mode mixing | +| 1.2.0 | 2026-10-11 | Added "Writing decent software" (shape, one-way, data, daily loop, testing, ops, versioning, agent docs distilled from project playbooks) | +| 1.3.0 | 2026-10-11 | Added house stack defaults (FastAPI route tree, Jinja2, SQLite via dataset, plain ES modules, mirrored tests) | +| 1.4.0 | 2026-10-11 | Added logging standards (debug-by-default, rotation caps) and backup procedures (3-2-1, restic default, tested restores) | +| 1.5.0 | 2026-10-11 | Added secrets hands-off rule, dependency minimalism, DTAP coexistence, incident response, minimum CI, migration discipline | +| 1.6.0 | 2026-10-11 | Enforced DRY (hoist on second use), consistency-first with existing code, object-oriented modeling | +| 1.7.0 | 2026-10-11 | Required Podman/Debian container + compose, self-sufficient Makefile, traditional pyproject.toml with declared console binary | +| 1.7.1 | 2026-10-11 | Added author comment to SKILL.md frontmatter | +)detzraw"}, + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +# Author: retoor <retoor@molodetz.nl> +name: seriousdevelopment +description: Scaffold a new project or upgrade an existing one to seriousdevelopment quality (spec-first docs, ADRs, build track sheet, verification gates, living-docs discipline). Use when starting a greenfield project or when an existing project needs its engineering and documentation bar raised. +--- + +# Serious Development + +One method for building software seriously, whether the project is new or +already exists. The method has three pillars: a canonical spec with zero +open gaps, a decision + tracking system that survives multi-month and +multi-model builds, and mechanical verification after every change. + +Skill version: 1.7.1 (history in `CHANGELOG.md`). + +## Modes + +Ask which mode applies if it is not obvious from the request. Do not run +both modes in one run. (Mode B itself is audit-then-apply; that is one +mode, not mixing.) For partial work, see Scoped runs. + +### Mode A: new project + +Scaffold the project skeleton, then write the spec before any product +code. Minimum skeleton: + +- `SPEC.md` (canonical spec, from `assets/templates/SPEC.md`) +- `BUILD-TRACK.md` (from `assets/templates/BUILD-TRACK.md`) +- `docs/adr/` with `0001` recording why this method was adopted +- `docs/runbooks/` (empty, indexed in the spec runbook list) +- `CHANGELOG.md` (spec + product changes, ascending versions) +- `roadmap/` only once the spec passes 500 lines (milestone split, see + "Roadmap split" below) +- `AGENTS.md` (or the harness equivalent) in three layers: personal base + rules, root project rules, nested subsystem files next to the code + +Fill the spec top-down: control + changelog, goals/non-goals, scope, +requirements with normative IDs, architecture, data model, API, security, +compliance, ops, estimates, risks, decisions. No product code until the +spec has zero TBDs (see "No open gaps"). Then grade the spec A-F on: +completeness, consistency, testability, estimation honesty, decision +traceability. Fix everything found without deleting or degrading +content, re-run the mechanical checks, and report before/after grades. +Below A- overall, say so and name what is missing. + +### Mode B: upgrade an existing project + +1. **Audit, read-only.** Inventory: existing docs, rules files, stack, + tests, CI, versioning, changelog habits. Write nothing yet. +2. **Gap report.** List what is missing against the checklist in + "Upgrade checklist" below. Grade the project A-F on: spec + completeness, decision traceability, dev tracking, verification, + doc freshness, estimation honesty. +3. **Apply, smallest diffs first.** Adopt the missing pieces using the + templates. Backfill the spec from the code (code wins on facts, + spec wins on intent; record conflicts, see below). +4. **Upgrade report.** Write `UPGRADE-REPORT.md` (template provided) + listing applied changes, grades before/after, and every conflict. + +### Conflict rule (Mode B, binding) + +If the existing project has a rule that clashes with this method, DO +NOT overwrite it. Keep their rule, mention the clash in the upgrade +report (what clashed, why the method prefers its own, what was kept), +and move on. The only exception is a rule that is illegal or +demonstrably destructive: flag it loudly, propose the fix, and still +wait for the human before changing it. + +### Scoped runs + +Full-mode runs are the default, but partial application is allowed when +the user asks for it: ADRs only, track sheet only, runbooks only. Rules +for scoped runs: run the Mode B audit first (read-only, even for one +piece), record the narrowed scope in the upgrade report, and list what +was deliberately left out so a later run can continue. The conflict +rule still applies. + +## The method + +### No open gaps + +Anything that would normally be "TBD" gets a decided default with +rationale in the decisions section instead. Tracked assumptions are +allowed only with: owner, verify-by date, and a fallback if the +assumption fails. A spec with an untracked open ends the run as +incomplete: say so, do not ship it quietly. + +### Normative IDs + +Every requirement, rule, and surface gets a stable ID from a family +prefix: FR (functional), SEC (security), BIL (billing), OPS (ops), NFR +(non-functional), UX, PG (pages/surfaces), D (decisions), PIN (stack +pins), A (assumptions), TRK/DOC (tracking/docs). IDs are never reused +once retired. Cross-reference by ID, never by prose paraphrase. + +### Caps and floors + +Everything unbounded gets a hard cap: rates, sizes, counts, intervals, +memory. Every automated process gets a floor it may never cross on its +own (retention windows, minimum intervals, protected paths); when a +floor is hit, page a human instead of deleting. Prefer static ceilings +to autotuning. + +### Safety + +Destructive operations are dry-run by default, confirmed explicitly, +and scoped as narrowly as possible. Loud skips beat silent corruption: +reject unverifiable input with a clear reason and a counter. Verify +before you write (checksums, integrity checks, pre-action snapshots). +A delete that does not reclaim is not a delete. + +### Proof + +Measure, do not theorize: numbers in docs come from runs on real +machines. Test through the real path (real filesystem, real +concurrency, failure injection), not just units around it. After +behavior changes, run the suite. After deploys, verify live. + +### Research first, then decide with an opinion + +Load-bearing decisions (stack, vendors, money flows, compliance) start +with web research: log the queries, synthesize into the spec with +normative defaults, keep the log in an appendix. Then decide: a clear +rule with stated tradeoffs beats a configurable maybe. Revisit only +with new evidence, filed as a new ADR (never by editing history). + +### Decisions and stack pins + +Spec-level decisions live in the spec as D-IDs (one line each, with +rationale). Repo-level decisions live as ADRs in `docs/adr/`, one file +per decision (`NNNN-short-title.md`) with a MADR-lite body: Status, +Date, Context, Decision, Consequences, Links. Accepted ADRs are immutable; change +means a new ADR that supersedes. Every D-ID, PIN, or requirement +change needs an ADR number in the PR. Agent prompts MUST cite ADR +constraints, never silently invent new ones. Hedged "X or Y" stack +clauses MUST be pinned to one choice with a PIN-ID before build. + +### Estimates + +Person-weeks with drivers, ranges not theater: each line names what +drives the cost. Recompute totals on every scope change and state the +consequences honestly (date moves, cost moves). Freeze scope with a +D-ID; new ideas go to a later release unless a launch gate forces +them. If the owner overrules the freeze, re-baseline the freeze the +same day and record the new date and cost. Never absorb scope +silently. + +### Changelog and versioning + +One changelog table: version, date, change text, strictly ascending. +Every spec change gets a row. Patch = additions/clarifications, +minor = scope or phasing changes, major = breaking contract changes. +Every markdown doc carries a versioned header: +`<!-- doc: file | version | updated -->`. Project prose rules hold +everywhere (for this author: no em-dashes anywhere, code, docs, +comments, commits, chat). + +### Single source, synced mirrors + +One file is canonical. Split-offs (roadmap, runbooks, API docs) are +mirrors, synced in the SAME commit as the canonical change. Never let +a mirror drift: verify sync mechanically after every edit round. + +### Roadmap split + +When the spec passes ~500 lines, split a build-order roadmap with a +zero-loss proof: account for every source line mechanically (count +them; declared duplicates counted exactly). Each part gets a header +comment naming source file, source version, and covered sections. +Keep a coverage map (every section, exactly one home) and a build +table (order, file, milestone, weeks, exit criteria). + +### Verification gates + +Define launch gates per phase (tests green, pentest criticals zero, +restore drill pass, billing zero-diff over N days, docs synced, legal +pack signed). All green or no launch. After every change round, run +the mechanical checks: sync between canonical and mirrors, ID +uniqueness, changelog order, forbidden-pattern scan, and the repo's +own test suite for the touched area. A confident patch you never saw +pass is not done. Sample mechanics (adapt paths to the project): + +```sh +# ID uniqueness: must print nothing (no duplicate requirement IDs) +rg -o '[A-Z]{2,3}-[0-9]+' SPEC.md | sort | uniq -d +# changelog order: versions must ascend top to bottom +rg '^\| [0-9]+\.[0-9]+' SPEC.md +# forbidden patterns: project rules, e.g. untracked TBDs, em-dashes (U+2014) +rg -n 'TBD|\u2014' SPEC.md BUILD-TRACK.md docs/ +``` + +### Build tracking (multi-month, multi-model safe) + +`BUILD-TRACK.md` at repo root is the single source of build status. +One row per milestone and per gate drill: Milestone, Owner, Status, +Updated, Evidence, Drift. Owner is a human name or +`llm:model/version`. Status enum: NOT-STARTED, IN-PROGRESS, BLOCKED, +IN-REVIEW, DONE. Update triggers, all same-day: any status change, +end of every work session, every gate pass or fail, every merged PR. +Staleness: 7 days without update marks the row STALE (CI fails); +14 days escalates to the owner as BLOCKED. Context passes ONLY +through the track sheet, ADRs, and code, never through chat memory. +LLM output is reviewed by a human or a second model before merge; +billing, auth, and security merges need human sign-off. + +### Living docs + +Maintain a register: doc, owner, update trigger, max staleness. +Minimum register: canonical spec + mirrors (same commit, 0 days), +track sheet (7 days), ADRs (before merge), runbooks (incident/drill ++ 30 days), API changelog (CI-generated, 0 days), cost book (vendor +price change, same day), risks (monthly + incident), postmortems +(48h after SEV-2+). Same-PR rule: a behavior/API/config/ops change +updates its docs in the SAME PR; reviewers verify docs impact; +maintainers block merges with stale docs on user-facing changes. +Generated docs regenerate in CI; drift fails the build. Quarterly +audit: re-verify the register, delete what stopped being true. + +### Craft and ops + +Exact, minimal diffs: edit existing files, reuse helpers, one way of +doing anything. Boring technology, dumb backends, smart edges. Docs +state what the system does today, gaps listed next to the code that +will close them. Commit, deploy, verify, in that order, same +session. Schedules live inside the system on a loop (retention, +cleanup, checks) with human-readable states, not silent hope. + +### Writing decent software + +Shape: one language, one database, server-rendered first, no build +step unless earned. One data root with every path derived in one +config module. Layout mirrors URLs: if you cannot guess the file +from the URL, the layout is wrong. + +House stack (defaults when nothing else is picked; override per +project with an ADR, never by drifting): + +- Backend: Python + FastAPI, one typed framework. Routers mirror the + URL tree: flat file per resource (`routers/feed.py`), package per + sub-resource (`routers/projects/`), each leaf owning an APIRouter, + the package `__init__` aggregating. One app factory, middleware in + fixed order (timing outermost, then security headers, rate limit, + maintenance gate, error handlers). +- Templates: Jinja2 server-side, one shared instance, one `base.html`, + page CSS/JS in standard blocks, shared partials for repeated UI. +- Frontend: no framework, no build step. Plain ES modules, one class + per file, small shared utilities. The server renders what exists at + request time; the client only handles live content, sanitized + fail-closed. +- Database: SQLite via the `dataset` ORM, WAL mode, busy timeout, one + embedded file under the data root for small to medium scale. + Graduate to Postgres only on measured need (write concurrency, + size, or managed backups), filed as an ADR. +- Tests mirror the tree: `tests/unit`, `tests/api` (live server plus + temp DB), `tests/e2e` (browser). + +One way of everything: one auth resolver, one response helper, one +validation style, one outbound HTTP client (with SSRF guard), one +queue pattern. A second implementation of an existing helper is a +bug, not a choice. + +DRY, enforced: no logic in two places. If it appears twice, hoist +it into the one helper, predicate, or constant both callers use. +Single source per fact in code exactly as in docs; copy-paste is +never the answer, not even "just this once". + +Consistency, enforced: new code matches the existing code first. +Same helpers, same patterns, same naming, same file shapes; a +reviewer should not be able to tell which lines are new. If the +existing pattern is bad, either fix all usages in the same change +or follow it and file the cleanup (never a third style alongside). +Read before you write, always. + +Object-oriented, enforced: model the domain as objects with the +behavior living next to the data it acts on. Small classes with one +job, composition over inheritance, no god objects, no procedural +blobs where an object fits. Python: dataclasses and Pydantic models +for data, service objects for operations. Frontend: one class per +file as the house stack already demands. + +Data: opaque IDs, soft delete everywhere with garbage collection +doing the hard deletes, ownership checked by one helper, no N+1 +(batch loaders are the only way to load related rows), idempotent +schema init. + +Daily loop, eight steps in order, a failed step blocks the rest: +understand (match an existing example or agree the new pattern +first), data, server, view, safety, agent+docs (the most-forgotten +step: tool action, endpoint doc, user doc), document, validate. + +Testing: three tiers (unit, HTTP integration on temp DB, browser +E2E), serial runs, full suite green before done, cheapest-first +triage, never narrow a run to pass. For money, state machines, and +read-then-write races add: property checks, stateful fuzzing, +real-process concurrency tests, static analysis on touched files. + +Ops: every repeated action is a named command (no tribal one-liners), +DTAP pipeline with CI green before merge, destructive ops dry-run +first and explicitly confirmed, retention CLIs per expirable domain, +production data guarded by written confirmation, deploy then verify +live. + +Runtime: every project ships a container, Podman preferred, Docker +accepted. Debian-slim-based image, a compose file for the full stack +(app, db, workers), one command to boot it. No "works on my machine": +CI builds the image, prod runs the image. + +Makefile: required and self-sufficient. It assures its own +environment: creates the venv (or pulls images) when missing, +installs pinned deps, then runs every target inside it. Standard +targets: `install`, `dev`, `prod`, `test`, `test-fast`, `lint`, +`clean`, `up`, `down`, `logs`. No target may assume a +hand-prepared machine. + +Python packaging: traditional `pyproject.toml` (setuptools or +hatchling, src layout), version as the single source read live by +the app, one console-script binary declared under +`project.scripts` that is THE way to execute the software (no bare +`python app.py` in docs or deploy). Deps pinned, dev extras +separated. + +Versioning: one version source read live by the app, patch +auto-bumped per commit, minor/major manual release commits, assets +served immutable under versioned URLs, trunk plus short ticket +branches plus a production promotion target, annotated tags and one +changelog for releases. Every markdown doc versioned: header +comment plus history table, bumped in the same commit as the +content change. + +Agent docs: three layers (personal base, root project rules under +~500 lines, nested subsystem files next to the code). Every +statement marked hard rule, convention, or guidance, each with a +named enforcer (test path, hook, review gate). A rule without an +enforcer is a wish: the quarterly audit deletes one. Docs update in +the same change as the code, state today with no fiction, and die +with the feature they describe. + +Logging: stdlib `logging` configured once via `dictConfig` in one +module, `getLogger(__name__)` per module. Debug log by default: +DEBUG goes to a rotating debug file, INFO+ to the console, levels +tunable via env. Rotation is mandatory, never unbounded: +RotatingFileHandler for the app log (10 MB x 5), timed rotation +for debug/audit (midnight, keep 7-14), plus a total cap on the log +directory (default 500 MB) enforced by a prune check. Format: UTC +ISO timestamp, level, logger, message; JSONL only when a shipper +consumes it. QueueHandler for threaded workers; lazy %-formatting +and level guards in hot paths. Never log secrets or PII (token +fingerprints, redaction as backstop); bare except without logging +is forbidden. Files first; stdout only for container runtimes. + +Backups: 3-2-1 minimum (3 copies, 2 media, 1 off-site at a +different provider/location, not the same DC). Anything holding +money or PII gets 3-2-1-1-0 (one immutable copy, zero-error +verified restores). Default tool restic (or Borg): encrypted, +automated on schedule, calendar retention (7 daily, 4 weekly, +6 monthly), prune wired to the same tool. RPO/RTO stated per +domain and measured, not assumed. Test restores monthly into an +empty directory (quarterly minimum); an untested backup fails the +ops gate, and a missing scheduled run pages like any other alarm. +Named commands (`backup-run`, `backup-verify`, `backup-prune`), +keys backed up separately from the data they unlock. + +Secrets are human-only. AI MUST never print, copy, transmit, modify, +rotate, or delete secret values (passwords, tokens, keys, connection +strings), and MUST never commit them. Referencing an env var NAME to +wire config is fine; its VALUE stays unread and unmoved. Servers are +closed from outside by default; secret storage and rotation are the +owner's task. If a secret is missing, expired, or appears where it +should not (logs, diffs, chat), stop and page the human. No +workarounds, ever. + +Dependencies: fewest possible. Stdlib first, one well-chosen library +per need, no new dependency without an ADR entry justifying it (what, +why not stdlib, license, maintenance status). Pinned versions, +automated audit on schedule, CVE fix-or-remove with a deadline. Each +release, remove one dependency nobody can justify. + +Environments: DTAP, usually on one platform. Dev, test, accept, and +prod coexist without colliding: separate database, data dir, ports, +and env-prefixed resource names per environment; no shared state +between test and prod, ever. Test data is marked and isolated; +production data never flows into test. Accept mirrors prod (same +images, same flags). Promotion is deliberate: merge, deploy, verify +live. + +Incidents: every alert names an owner and a runbook; unowned alerts +are forbidden. On-call rotation with a human reachable; agents hold +and page instead of improvising past a floor. SEV-2+ gets a +postmortem within 48h (template provided). Customer-visible incidents +update the status page in the same hour; an incident is not closed +until comms, fix, and action items all landed. + +Minimum CI: lint touched files, serial full suite, doc-header check +(every `.md` carries a version line), generated-docs drift check, +dependency audit. All green before merge, no exceptions. + +Migrations: every schema change ships with its rollback, reviewed +together. Expand before contract (additive first, removal later); +destructive migrations need a snapshot plus a rehearsed restore. +Never edit a migration that already ran in prod; write a new one. + +## Checklists + +### New project + +- [ ] Skeleton files exist from templates +- [ ] Spec: zero untracked opens, IDs on everything, changelog started +- [ ] Stack pins: no "X or Y" clauses remain +- [ ] Engineering shape chosen: layout, data conventions, test tiers, commands +- [ ] Logging + backups: rotation caps set, 3-2-1 scheduled, restore drill dated +- [ ] DTAP coexistence verified: separate state per env, no test/prod sharing +- [ ] Incident loop defined: alert owners, on-call, status page, postmortem rule +- [ ] Assumptions have owner + verify date + fallback +- [ ] Estimates have drivers; scope freeze D-ID recorded +- [ ] Gates defined per phase; first gate (spec review) passed +- [ ] Spec graded A-F and fixed to A- or better, grades reported +- [ ] BUILD-TRACK.md has one row per milestone, all NOT-STARTED +- [ ] ADR 0001 records method adoption + +### Upgrade existing project + +- [ ] Read-only audit complete, nothing written yet +- [ ] Gap report with A-F grades on the six axes +- [ ] Missing pieces applied, smallest diffs first +- [ ] Conflicts kept-as-is and recorded (never overwritten) +- [ ] UPGRADE-REPORT.md written with before/after grades +- [ ] Mechanical checks pass (sync, IDs, changelog, tests) + +## Red flags (stop and say so) + +- A change that cannot name its spec ID or ADR. +- Scope absorbed without re-baselining date and cost. +- A mirror edited without its canonical source (or vice versa). +- Docs updated "later" instead of in the same PR. +- A gate waved through red, or narrowed to pass. +- An agent inventing a new queue, cache, or retry policy the ADRs + never approved. +- AI touching secret values (print, copy, change, delete), or working + around a missing secret instead of paging the human. +- Test and prod sharing state, or prod data flowing into test. +- A migration with no rollback, or an edit to one that already ran. + +## Templates + +In `assets/templates/`: `SPEC.md` (spec skeleton), `BUILD-TRACK.md`, +`ADR.md`, `RUNBOOK.md`, `POSTMORTEM.md`, `UPGRADE-REPORT.md` +(Mode B report with conflicts table), `CHANGELOG.md`, `AGENTS.md` +(root rules file, three-layer header). Copy, never link: the target +project owns its copies from day one. +)detzraw"}, + {R"detzraw(assets/templates/ADR.md)detzraw", +R"detzraw(<!-- doc: docs/adr/NNNN-short-title.md | version: 1.0 | updated: YYYY-MM-DD --> +# NNNN: <Imperative title, e.g. "Use Postgres RLS for tenancy"> + +- Status: proposed | accepted | superseded by NNNN +- Date: YYYY-MM-DD +- Decision makers: <names> +- Spec IDs: <D-IDs, PINs, FRs touched> + +## Context + +What forced this decision. Facts only, with measurements where the +claim needs one. + +## Decision + +The rule, stated so a stranger can follow it without asking. + +## Consequences + +Good and bad. Include costs, caps/floors introduced, and what this +rules out. + +## Links + +PRs, issues, spec sections, superseded ADRs. +)detzraw"}, + {R"detzraw(assets/templates/AGENTS.md)detzraw", +R"detzraw(<!-- doc: AGENTS.md | version: 0.1.0 | updated: YYYY-MM-DD --> +# Agent instructions: <project> + +Three layers (separate files, never merged): + +1. Personal base rules (operator-owned, every project): safety, proof, + craft, ops habits. +2. This file (root project rules): applies to the whole repo. Global + rules only; keep it short enough that agents actually read it. +3. Nested subsystem files (next to the code, loaded on touch): deep + mechanics, helpers, pitfalls, worked examples. + +## Root rules + +- Working directory: <repo root>. <scope constraints, e.g. work only here> +- Canonical spec: <SPEC.md>; mirrors sync in the same commit. +- Build status: <BUILD-TRACK.md>; update triggers per the spec tracking section. +- Decisions: <docs/adr/>; cite ADR constraints in every agent prompt. +- Prose rules: <e.g. no em-dashes anywhere: code, docs, comments, commits> +- Verification: <mechanical checks after every change round> +- Commit/deploy/verify order: <project policy> + +## Subsystem files + +<list nested instruction files and what each owns> +)detzraw"}, + {R"detzraw(assets/templates/BUILD-TRACK.md)detzraw", +R"detzraw(<!-- doc: BUILD-TRACK.md | version: 0.1.0 | updated: YYYY-MM-DD --> +# Build track sheet + +Single source of build status. Status enum: NOT-STARTED, IN-PROGRESS, +BLOCKED, IN-REVIEW, DONE. Update triggers (same-day): status change, +session end, gate pass/fail, merged PR. 7 days stale fails docs CI; +14 days escalates to the owner as BLOCKED. + +| Milestone | Owner | Status | Updated | Evidence | Drift | +|-----------|-------|--------|---------|----------|-------| +| M0 | unassigned | NOT-STARTED | - | - | - | + +## Session history + +One line per work session, newest first: + +- YYYY-MM-DD | human: <name> or llm: <model/version> | goal | outcome | rows touched +)detzraw"}, + {R"detzraw(assets/templates/CHANGELOG.md)detzraw", +R"detzraw(<!-- doc: CHANGELOG.md | version: 0.1.0 | updated: YYYY-MM-DD --> +# Changelog + +Strictly ascending. One row per change. Patch = additions and +clarifications, minor = scope or phasing changes, major = breaking +contract changes. + +| Version | Date | Change | +|---------|------|--------| +| 0.1.0 | YYYY-MM-DD | Initial changelog | +)detzraw"}, + {R"detzraw(assets/templates/POSTMORTEM.md)detzraw", +R"detzraw(<!-- doc: docs/postmortems/YYYY-MM-DD-<name>.md | version: 1.0 | updated: YYYY-MM-DD --> +# Postmortem: <incident title> + +- Date: YYYY-MM-DD +- Severity: <SEV-1..4> +- Owner: <name> +- Status: draft | final + +## Timeline (UTC) + +- HH:MM <event, with evidence link> + +## Impact + +<who was affected, for how long, measured numbers> + +## Root cause + +<the cause, not the symptom; five whys if useful> + +## Action items + +| Action | Owner | Due | Done | +|--------|-------|-----|------| +| ... | ... | ... | ... | + +## Lessons + +<what changes in code, runbooks, or docs so this class cannot recur> +)detzraw"}, + {R"detzraw(assets/templates/RUNBOOK.md)detzraw", +R"detzraw(<!-- doc: docs/runbooks/<name>.md | version: 1.0 | updated: YYYY-MM-DD --> +# Runbook: <name> + +- Trigger: <alert, symptom, or scheduled drill> +- Severity: <SEV-1..4> +- Owner: <role> + +## Steps + +1. <numbered, copy-pasteable, with expected output> + +## Rollback + +<exact undo for every mutating step> + +## Comms template + +<who gets told what, when; status page text if customer-visible> + +## Postmortem owner + +<name or role; SEV-2+ needs a postmortem within 48h> + +## Last drilled + +<date + result; empty means never, which fails the ops gate> +)detzraw"}, + {R"detzraw(assets/templates/SPEC.md)detzraw", +R"detzraw(<!-- doc: SPEC.md | version: 0.1.0 | updated: YYYY-MM-DD --> +# <Project> - Complete Implementation Document + +- Version: 0.1.0 +- Status: Draft (zero TBDs required before build) +- Date: YYYY-MM-DD +- Owner: <name> + +## 0. Document control + +### 0.1 Conventions + +- Normative IDs: FR, SEC, BIL, OPS, NFR, UX, PG, D, PIN, A, TRK, DOC. + IDs are never reused once retired. Cross-reference by ID. +- No open gaps: anything that would be "TBD" gets a decided default + with rationale in Decisions, or a tracked assumption (owner + + verify-by + fallback) below. + +### 0.2 Assumptions (tracked, all need fallback) + +| ID | Assumption | Owner | Verify by | Fallback | +|----|-----------|-------|-----------|----------| +| A-01 | ... | ... | ... | ... | + +### 0.3 Hard global caps + +| Cap | Value | Enforced at | +|-----|-------|-------------| +| ... | ... | server-side | + +### 0.4 Changelog (ascending, one row per change) + +| Version | Date | Change | +|---------|------|--------| +| 0.1.0 | YYYY-MM-DD | Initial spec skeleton | + +## 1. Goals and non-goals + +- G-01..: measurable goals. +- NG-01..: explicit non-goals (decided, not deferred). + +## 2. Scope (personas, use cases) + +## 3. Requirements (FR/SEC/BIL/OPS/NFR/UX with IDs) + +## 4. Surfaces (PG inventory if user-facing) + +## 5. Architecture (pins as PIN-IDs, no "X or Y") + +## 6. Data model, API, security, compliance, ops + +## 7. Estimates (person-weeks with drivers) + +| Area | PW | Driver | +|------|----|--------| +| ... | ... | ... | + +## 8. Risks and mitigations + +## 9. Decisions (D-IDs, one line + rationale each) + +- D-01: ... + +## Appendix A. Research log (queries + dates + synthesis pointer) +)detzraw"}, + {R"detzraw(assets/templates/UPGRADE-REPORT.md)detzraw", +R"detzraw(<!-- doc: UPGRADE-REPORT.md | version: 1.0 | updated: YYYY-MM-DD --> +# Upgrade report: seriousdevelopment applied (Mode B) + +- Date: YYYY-MM-DD +- Applied by: <human or llm:model/version> +- Scope: <repo + commit range> + +## Grades (A-F, before -> after) + +| Axis | Before | After | +|------|--------|-------| +| Spec completeness | | | +| Decision traceability | | | +| Dev tracking | | | +| Verification | | | +| Doc freshness | | | +| Estimation honesty | | | + +## Applied changes + +<list files created/changed, smallest diffs first, with one-line why> + +## Conflicts (existing rule kept, method rule NOT applied) + +| # | Existing rule (kept) | Method preference | Why kept | Location | +|---|---------------------|-------------------|----------|----------| +| 1 | ... | ... | ... | ... | + +Rule: a clash never overwrites the project's own rule. Illegal or +demonstrably destructive rules are flagged loudly and changed only +with explicit human approval (record the approval here). + +## Follow-ups + +<ordered list of what the project should do next, with owners> +)detzraw"}, + }}, + {R"detzraw(taste)detzraw", { + {R"detzraw(SKILL.md)detzraw", +R"detzraw(--- +name: taste +description: Mandatory preflight for web frontend visual design. If available, load the taste skill (load_skill) before the first frontend create or edit tool call whenever creating or visually styling a human-viewed web page, component, app, dashboard, or interactive report. Do not load it for non-visual frontend logic or machine-only HTML. If unavailable, continue without warning. Negative constraints only; user requests and other explicit styles still win. +user-invocable: false +--- + +# Anti-Slop: Frontend DON'Ts + +What NOT to do when designing a UI or page. This is a filter, not a style; it never prescribes a look. Stacking several of the below is what reads as AI-made on sight. Adaptive, so if the user or another skill asks for something specific, do that instead; these are defaults, not overrides. + +- purple / indigo / violet accent, or a purple-to-blue hero gradient (the top AI tell) +- `bg-clip-text` gradient headline text +- cream (`#faf8f4`) as the whole-page background +- aurora blobs / glowing colored shadows, or an identical glow on every card +- Inter everywhere, or Geist / Space Grotesk / Instrument Serif / Roboto / Arial as your only identity +- gray-400 / gray-500 body text +- one font weight and size throughout (no hierarchy) +- tracked all-caps micro-kickers above every section +- centered hero of eyebrow pill + headline + two CTAs +- the identical 3-card feature grid with a lucide icon in a rounded square +- an 8+ word, 48px+ headline that says nothing +- bento grid for everything +- the same eyebrow-headline-3cards section repeated back to back +- `rounded-2xl`, or any single radius, on every element +- glassmorphism blur + border + shadow on every surface +- `hover:scale-105` lift on cards +- emoji as feature icons (🚀 ⚡ ✨ 🎯 💡) +- left-border accent cards, a colored border on a rounded element, or cards nested inside cards +- fade-up / slide-up on everything as it scrolls in +- motion that ignores `prefers-reduced-motion` +- scribbly hand-drawn SVG mascots with no purpose +)detzraw"}, + }}, + }; + return data; +} + +const std::map<std::string, std::string>& BundledData::upstream() { + static const std::map<std::string, std::string> data = { + {R"detzraw(deep-research)detzraw", R"detzraw(muse-user/deep-research, adapted (fork/poll, web_fetch, no em dashes))detzraw"}, + {R"detzraw(git)detzraw", R"detzraw(muse-bundled/git, adapted (load_skill paths, tai authorization note))detzraw"}, + {R"detzraw(plan)detzraw", R"detzraw(muse-bundled/plan, adapted (load_skill, ask_user, fork/poll))detzraw"}, + {R"detzraw(requirements-clarification)detzraw", R"detzraw(muse-bundled/requirements-clarification, adapted (ask_user, single-line description))detzraw"}, + {R"detzraw(seriousdevelopment)detzraw", R"detzraw(muse-user/seriousdevelopment v1.7.1, verbatim)detzraw"}, + {R"detzraw(taste)detzraw", R"detzraw(muse-bundled/taste, one-word adaptation (load_skill))detzraw"}, + }; + return data; +} + +const std::map<std::string, SkillBlueprint>& BundledData::blueprints() { + static const std::map<std::string, SkillBlueprint> data = { + {R"detzraw(api-client)detzraw", {R"detzraw(build a resilient REST/OpenAPI client)detzraw", R"detzraw(project)detzraw", R"detzraw(Deep-research resilient HTTP client design first: retries with backoff, timeouts, pagination, auth header handling, and error classification. Then write a skill that builds small stdlib-only API clients from a base URL and an endpoint list.)detzraw"}}, + {R"detzraw(bot-creator)detzraw", {R"detzraw(build a dedicated chat bot from a tai.py variant)detzraw", R"detzraw(project)detzraw", R"detzraw(Deep-research preferred chat-bot features first: transport (Telegram long-poll versus webhook), command routing, per-user allowlists, rate limits, and safe restarts. Also research safe techniques for scripting huge single-file programs: exact-match edits, atomic temp-plus-rename writes, and versioned backups before every reshape. Then copy tai.py to a variant path and reshape the copy into a dedicated bot, editing efficiently and keeping the audit trail intact.)detzraw"}}, + {R"detzraw(data-wrangler)detzraw", {R"detzraw(reshape CSV, JSON, and SQLite data)detzraw", R"detzraw(project)detzraw", R"detzraw(Deep-research tabular data reshaping first: CSV dialects, JSON normalization, SQLite import ergonomics, and streaming for large files. Then write a skill that reshapes CSV, JSON, and SQLite inputs using stdlib-only scripts.)detzraw"}}, + {R"detzraw(home-sysadmin)detzraw", {R"detzraw(routine Linux host care and triage)detzraw", R"detzraw(home)detzraw", R"detzraw(Deep-research routine single-host Linux care first: disk, memory, service health, log triage, and backup verification. Then write a skill that runs a host checkup and reports findings with suggested fixes.)detzraw"}}, + {R"detzraw(pdf-forms)detzraw", {R"detzraw(fill PDF forms and extract field data)detzraw", R"detzraw(project)detzraw", R"detzraw(Deep-research PDF form handling with freely available tooling first: AcroForm field discovery, filling, flattening, and text extraction. Then write a skill that fills forms and extracts field data from this machine's installed tools.)detzraw"}}, + {R"detzraw(web-researcher)detzraw", {R"detzraw(deep research with triangulated sources)detzraw", R"detzraw(project)detzraw", R"detzraw(Deep-research professional open-source research methodology first: query expansion, source triangulation, recency checks, and claim grading. Then write a skill that turns a question into a sourced brief with confidence levels.)detzraw"}}, + }; + return data; +} + +} // namespace detz::skills diff --git a/taicpp/src/skills/bundled_data.h b/taicpp/src/skills/bundled_data.h new file mode 100644 index 0000000..0f01be4 --- /dev/null +++ b/taicpp/src/skills/bundled_data.h @@ -0,0 +1,16 @@ +#pragma once +// Embedded bundled-skill data. Definitions live in the generated bundled_data.cpp. +#include <map> +#include <string> +namespace detz::skills { +struct SkillBlueprint { std::string hint, scope, brief; }; +class BundledData { + public: + // BUNDLED_SKILLS: skill name -> (relative path -> file text). Sorted maps: every tai.py use is sorted(). + static const std::map<std::string, std::map<std::string, std::string>>& files(); + // BUNDLED_SKILL_UPSTREAM + static const std::map<std::string, std::string>& upstream(); + // SKILL_BLUEPRINTS + static const std::map<std::string, SkillBlueprint>& blueprints(); +}; +} // namespace detz::skills diff --git a/taicpp/src/skills/gen_bundled.py b/taicpp/src/skills/gen_bundled.py new file mode 100644 index 0000000..39327f8 --- /dev/null +++ b/taicpp/src/skills/gen_bundled.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Regenerates bundled_data.cpp from tai.py (BUNDLED_SKILLS, BUNDLED_SKILL_UPSTREAM, SKILL_BLUEPRINTS). + +Usage: python3 -I src/skills/gen_bundled.py (run from taicpp/; writes src/skills/bundled_data.cpp) +tai.py stays the source of truth: the embedded text is byte-identical, never hand-edited here. +""" +import importlib.util +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +PY = os.path.join(os.path.dirname(ROOT), "tai.py") +OUT = os.path.join(HERE, "bundled_data.cpp") +DELIM = "detzraw" + + +def load(): + spec = importlib.util.spec_from_file_location("tai_src", PY) + mod = importlib.util.module_from_spec(spec) + sys.modules["tai_src"] = mod + spec.loader.exec_module(mod) + return mod + + +def lit(text): + close = ")" + DELIM + '"' + if close in text: + raise SystemExit("text contains the raw string terminator") + if "\0" in text: + raise SystemExit("text contains NUL") + # Raw literals cannot carry a lone \r reliably across line-ending normalisation, so escape them via concatenation. + return 'R"' + DELIM + "(" + text + ")" + DELIM + '"' + + +def main(): + tai = load() + out = ["// GENERATED by src/skills/gen_bundled.py from tai.py. Do not edit.", + "// Source: tai.py BUNDLED_SKILLS, BUNDLED_SKILL_UPSTREAM, SKILL_BLUEPRINTS (byte-identical).", + '#include "bundled_data.h"', "namespace detz::skills {", ""] + out.append("const std::map<std::string, std::map<std::string, std::string>>& BundledData::files() {") + out.append(" static const std::map<std::string, std::map<std::string, std::string>> data = {") + for name in sorted(tai.BUNDLED_SKILLS): + out.append(" {%s, {" % lit(name)) + for rel in sorted(tai.BUNDLED_SKILLS[name]): + out.append(" {%s,\n%s}," % (lit(rel), lit(tai.BUNDLED_SKILLS[name][rel]))) + out.append(" }},") + out.append(" };") + out.append(" return data;") + out.append("}") + out.append("") + out.append("const std::map<std::string, std::string>& BundledData::upstream() {") + out.append(" static const std::map<std::string, std::string> data = {") + for name in sorted(tai.BUNDLED_SKILL_UPSTREAM): + out.append(" {%s, %s}," % (lit(name), lit(tai.BUNDLED_SKILL_UPSTREAM[name]))) + out.append(" };") + out.append(" return data;") + out.append("}") + out.append("") + out.append("const std::map<std::string, SkillBlueprint>& BundledData::blueprints() {") + out.append(" static const std::map<std::string, SkillBlueprint> data = {") + for name in sorted(tai.SKILL_BLUEPRINTS): + bp = tai.SKILL_BLUEPRINTS[name] + out.append(" {%s, {%s, %s, %s}}," % (lit(name), lit(bp["hint"]), lit(bp["scope"]), lit(bp["brief"]))) + out.append(" };") + out.append(" return data;") + out.append("}") + out.append("") + out.append("} // namespace detz::skills") + with open(OUT, "w", encoding="utf-8", newline="") as handle: + handle.write("\n".join(out) + "\n") + print("wrote %s" % OUT) + + +main() diff --git a/taicpp/src/skills/gen_unicode.py b/taicpp/src/skills/gen_unicode.py new file mode 100644 index 0000000..f7a6423 --- /dev/null +++ b/taicpp/src/skills/gen_unicode.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Regenerates pyunicode_data.cpp: the Unicode decimal-digit (category Nd) blocks and the str.isspace() +code points of the running Python, so \\d, \\s and int() behave like tai.py's regexes. +Usage: python3 -I src/skills/gen_unicode.py (from taicpp/) +""" +import os +import sys +import unicodedata + +HERE = os.path.dirname(os.path.abspath(__file__)) +zeros = [] +for cp in range(0x110000): + ch = chr(cp) + if unicodedata.category(ch) == "Nd": + value = unicodedata.decimal(ch) + if value == 0: + zeros.append(cp) + # every Nd block is 10 consecutive code points holding 0..9 + assert unicodedata.decimal(chr(cp - value)) == 0 +spaces = [cp for cp in range(0x110000) if chr(cp).isspace()] +lines = [ + "// GENERATED by src/skills/gen_unicode.py (Python %s, Unicode %s). Do not edit." % (sys.version.split()[0], unicodedata.unidata_version), + '#include "pyunicode.h"', + "namespace detz::misc {", + "const std::vector<unsigned>& PyUnicode::digit_zeros() {", + " static const std::vector<unsigned> v = {" + ", ".join("0x%X" % z for z in zeros) + "};", + " return v;", + "}", + "const std::vector<unsigned>& PyUnicode::space_points() {", + " static const std::vector<unsigned> v = {" + ", ".join("0x%X" % z for z in spaces) + "};", + " return v;", + "}", + "} // namespace detz::misc", + "", +] +with open(os.path.join(HERE, "pyunicode_data.cpp"), "w", encoding="utf-8") as handle: + handle.write("\n".join(lines)) +print("zeros", len(zeros), "spaces", len(spaces)) diff --git a/taicpp/src/skills/miscdatetime.cpp b/taicpp/src/skills/miscdatetime.cpp new file mode 100644 index 0000000..861324d --- /dev/null +++ b/taicpp/src/skills/miscdatetime.cpp @@ -0,0 +1,384 @@ +#include "miscdatetime.h" +#include <algorithm> +#include <cctype> +#include <chrono> +#include <cstdio> +#include <cstdlib> +#include <ctime> +#include "miscpy.h" +namespace detz::misc { + +static constexpr int64_t ORD_UNIX = 719163; // proleptic ordinal of 1970-01-01 (0001-01-01 is 1) +static constexpr int64_t EPOCH_SECS = (ORD_UNIX - 1) * 86400; // seconds from 0001-01-01 to 1970-01-01 + +int64_t DateTime::days_from_civil(int y, unsigned m, unsigned d) { + y -= m <= 2; + const int64_t era = (y >= 0 ? y : y - 399) / 400; + const unsigned yoe = static_cast<unsigned>(y - era * 400); + const unsigned doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + static_cast<int64_t>(doe) - 719468; +} +static void civil_from_days(int64_t z, int& y, unsigned& m, unsigned& d) { + z += 719468; + const int64_t era = (z >= 0 ? z : z - 146096) / 146097; + const unsigned doe = static_cast<unsigned>(z - era * 146097); + const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + y = static_cast<int>(yoe) + static_cast<int>(era) * 400; + const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const unsigned mp = (5 * doy + 2) / 153; + d = doy - (153 * mp + 2) / 5 + 1; + m = mp < 10 ? mp + 3 : mp - 9; + y += m <= 2; +} +static bool is_leap(int y) { return (y % 4 == 0 && y % 100 != 0) || y % 400 == 0; } +bool DateTime::valid_date(int y, int m, int d) { + if (y < 1 || y > 9999 || m < 1 || m > 12 || d < 1) return false; + static const int dim[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + int max = dim[m - 1]; + if (m == 2 && is_leap(y)) max = 29; + return d <= max; +} +static int64_t ymd_to_ord(int y, int m, int d) { return DateTime::days_from_civil(y, static_cast<unsigned>(m), static_cast<unsigned>(d)) + ORD_UNIX; } +static void ord_to_ymd(int64_t ord, int& y, int& m, int& d) { + unsigned mm, dd; + civil_from_days(ord - ORD_UNIX, y, mm, dd); + m = static_cast<int>(mm); + d = static_cast<int>(dd); +} +static int weekday(int y, int m, int d) { return static_cast<int>((ymd_to_ord(y, m, d) + 6) % 7); } // Monday == 0 +static int64_t iso_week1_monday(int year) { + int64_t first_day = ymd_to_ord(year, 1, 1); + int64_t first_weekday = (first_day + 6) % 7; + int64_t week1_monday = first_day - first_weekday; + if (first_weekday > 3) week1_monday += 7; + return week1_monday; +} + +DateTime DateTime::now_utc() { + using namespace std::chrono; + int64_t us = duration_cast<microseconds>(system_clock::now().time_since_epoch()).count(); + return DateTime(us, 0); +} + +// ---------------------------------------------------------------- fromisoformat (CPython port) +namespace { +const char* kInvalid = "invalid datetime, use ISO like 2026-10-08T09:00"; + +struct Buf { + const std::string& s; + char at(std::size_t i) const { return i < s.size() ? s[i] : '\0'; } + bool digit(std::size_t i) const { char c = at(i); return c >= '0' && c <= '9'; } +}; + +// parse_digits: reads exactly n digits at p; false (NULL) on a non-digit +bool parse_digits(const Buf& b, std::size_t& p, int& out, int n) { + out = 0; + for (int i = 0; i < n; ++i) { + char c = b.at(p++); + if (c < '0' || c > '9') return false; + out = out * 10 + (c - '0'); + } + return true; +} + +int64_t find_separator(const Buf& b, std::size_t len) { + if (len == 7) return 7; + if (b.at(4) == '-') { + if (b.at(5) == 'W') { + if (len < 8) return -1; + if (len > 8 && b.at(8) == '-') { + if (len == 9) return -1; + if (len > 10 && b.digit(10)) return 8; + return 10; + } + return 8; + } + return 10; + } + if (b.at(4) == 'W') { + std::size_t idx = 7; + for (; idx < len; ++idx) { + if (!b.digit(idx)) break; + } + if (idx < 9) return static_cast<int64_t>(idx); + return idx % 2 == 0 ? 7 : 8; + } + return 8; +} + +// iso_to_ymd; returns 0 on success +int iso_to_ymd(int iso_year, int iso_week, int iso_day, int& year, int& month, int& day) { + if (iso_year < 1 || iso_year > 9999) return -4; + if (iso_week <= 0 || iso_week >= 53) { + bool out_of_range = true; + if (iso_week == 53) { + int first_weekday = weekday(iso_year, 1, 1); + if (first_weekday == 3 || (first_weekday == 2 && is_leap(iso_year))) out_of_range = false; + } + if (out_of_range) return -2; + } + if (iso_day <= 0 || iso_day >= 8) return -3; + int64_t day_1 = iso_week1_monday(iso_year); + int64_t day_offset = static_cast<int64_t>(iso_week - 1) * 7 + iso_day - 1; + ord_to_ymd(day_1 + day_offset, year, month, day); + return 0; +} + +int parse_isoformat_date(const Buf& b, std::size_t len, int& year, int& month, int& day) { + std::size_t p = 0; + if (!parse_digits(b, p, year, 4)) return -1; + bool uses_separator = b.at(p) == '-'; + if (uses_separator) ++p; + if (b.at(p) == 'W') { + ++p; + int iso_week = 0, iso_day = 0; + if (!parse_digits(b, p, iso_week, 2)) return -3; + if (p < len) { + if (uses_separator && b.at(p++) != '-') return -2; + if (!parse_digits(b, p, iso_day, 1)) return -4; + } else { + iso_day = 1; + } + int rv = iso_to_ymd(year, iso_week, iso_day, year, month, day); + return rv ? -3 : 0; + } + if (!parse_digits(b, p, month, 2)) return -1; + if (uses_separator && b.at(p++) != '-') return -2; + if (!parse_digits(b, p, day, 2)) return -2; + return 0; +} + +// parse_hh_mm_ss_ff over b[start, end); returns <0 on error, otherwise 1 when characters were left over +int parse_hh_mm_ss_ff(const Buf& b, std::size_t start, std::size_t end, int& hour, int& minute, int& second, int64_t& micro) { + hour = minute = second = 0; + micro = 0; + std::size_t p = start; + int* vals[3] = {&hour, &minute, &second}; + bool has_separator = true; + for (int i = 0; i < 3; ++i) { + if (!parse_digits(b, p, *vals[i], 2)) return -3; + char c = b.at(p++); + if (i == 0) has_separator = (c == ':'); + if (p >= end) return c != '\0' ? 1 : 0; + if (has_separator && c == ':') { + if (i == 2) return -4; + continue; + } + if (c == '.' || c == ',') break; + if (!has_separator) { + --p; + } else { + return -4; + } + } + std::size_t len_remains = end > p ? end - p : 0; + std::size_t to_parse = len_remains >= 6 ? 6 : len_remains; + int frac = 0; + if (to_parse == 0 || !parse_digits(b, p, frac, static_cast<int>(to_parse))) return -3; + static const int correction[] = {100000, 10000, 1000, 100, 10}; + micro = frac; + if (to_parse < 6) micro *= correction[to_parse - 1]; + while (b.digit(p)) ++p; + return b.at(p) != '\0' ? 1 : 0; +} + +// parse_isoformat_time over b (the text after the date/time separator). Returns <0 on error; tz* set when 1. +int parse_isoformat_time(const Buf& b, int& hour, int& minute, int& second, int64_t& micro, int64_t& tz_us, bool& has_tz) { + std::size_t len = b.s.size(); + std::size_t tzpos = 0; + do { + char c = b.at(tzpos); + if (c == 'Z' || c == '+' || c == '-') break; + } while (++tzpos < len); + int rv = parse_hh_mm_ss_ff(b, 0, tzpos, hour, minute, second, micro); + if (rv < 0) return rv; + if (tzpos == len) { + has_tz = false; + return rv == 1 ? -5 : 0; + } + has_tz = true; + if (b.at(tzpos) == 'Z') { + tz_us = 0; + return b.at(tzpos + 1) != '\0' ? -5 : 1; + } + int sign = b.at(tzpos) == '-' ? -1 : 1; + int th, tm, ts; + int64_t tmicro; + rv = parse_hh_mm_ss_ff(b, tzpos + 1, len, th, tm, ts, tmicro); + if (rv < 0) return rv; + tz_us = sign * ((static_cast<int64_t>(th) * 3600 + tm * 60 + ts) * 1000000 + tmicro); + return rv ? -5 : 1; +} +} // namespace + +DateTime DateTime::fromisoformat(const std::string& input) { + // tai.py _fromisoformat: str(text).strip(), a trailing Z becomes +00:00, then datetime.fromisoformat + std::string raw = PyStr::strip(input); + if (!raw.empty() && (raw.back() == 'Z' || raw.back() == 'z')) raw = raw.substr(0, raw.size() - 1) + "+00:00"; + if (raw.empty()) throw std::invalid_argument("empty datetime"); // unreachable for native; keeps Python's wrapper error + if (raw.find('\0') != std::string::npos) throw std::invalid_argument(kInvalid); + Buf whole{raw}; + std::size_t len = raw.size(); + int64_t sep = find_separator(whole, len); + if (sep < 0) throw std::invalid_argument(kInvalid); + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + int64_t micro = 0, tz_us = 0; + bool has_tz = false; + int rv = parse_isoformat_date(whole, static_cast<std::size_t>(sep), year, month, day); + if (rv == 0 && len > static_cast<std::size_t>(sep)) { + std::size_t p = static_cast<std::size_t>(sep); + unsigned char lead = static_cast<unsigned char>(raw[p]); + if ((lead & 0x80) == 0) p += 1; + else if ((lead & 0xf0) == 0xe0) p += 3; + else if ((lead & 0xf0) == 0xf0) p += 4; + else p += 2; + std::string rest = p <= len ? raw.substr(p) : std::string(); + Buf tb{rest}; + rv = parse_isoformat_time(tb, hour, minute, second, micro, tz_us, has_tz); + } + if (rv < 0) throw std::invalid_argument(kInvalid); + if (year < 1 || year > 9999 || month < 1 || month > 12 || !valid_date(year, month, day)) throw std::invalid_argument(kInvalid); + bool add_day = false; + if (hour == 24) { + if (minute != 0 || second != 0 || micro != 0) throw std::invalid_argument("minute, second, and microsecond must be 0 when hour is 24"); + hour = 0; + add_day = true; + } + if (hour > 23 || minute > 59 || second > 59) throw std::invalid_argument(kInvalid); + if (has_tz && (tz_us <= -86400LL * 1000000 || tz_us >= 86400LL * 1000000)) throw std::invalid_argument(kInvalid); + int64_t days = days_from_civil(year, static_cast<unsigned>(month), static_cast<unsigned>(day)); + if (add_day) { + ++days; + if (days + ORD_UNIX > ymd_to_ord(9999, 12, 31)) throw std::invalid_argument(kInvalid); // CPython reports the day overflow of 24:00 as a ValueError here + } + int64_t us = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 + micro; + if (has_tz) return with_offset_us(us, tz_us); + DateTime naive; + naive.us_ = us; + return naive; +} + +std::string DateTime::isoformat(bool timespec_seconds) const { + int64_t secs = us_ / 1000000; + int64_t micro = us_ % 1000000; + if (micro < 0) { micro += 1000000; --secs; } + int64_t days = secs / 86400; + int64_t rem = secs % 86400; + if (rem < 0) { rem += 86400; --days; } + int y; + unsigned m, d; + civil_from_days(days, y, m, d); + char buf[96]; + std::snprintf(buf, sizeof buf, "%04d-%02u-%02uT%02d:%02d:%02d", y, m, d, static_cast<int>(rem / 3600), static_cast<int>(rem % 3600 / 60), static_cast<int>(rem % 60)); + std::string out = buf; + if (!timespec_seconds && micro != 0) { + std::snprintf(buf, sizeof buf, ".%06lld", static_cast<long long>(micro)); + out += buf; + } + if (offset_us_) { + int64_t off = *offset_us_; + char sign = off < 0 ? '-' : '+'; + int64_t a = off < 0 ? -off : off; + int64_t a_sec = a / 1000000, a_us = a % 1000000; + std::snprintf(buf, sizeof buf, "%c%02d:%02d", sign, static_cast<int>(a_sec / 3600), static_cast<int>(a_sec % 3600 / 60)); + out += buf; + if (a_sec % 60 || a_us) { + std::snprintf(buf, sizeof buf, ":%02d", static_cast<int>(a_sec % 60)); + out += buf; + if (a_us) { + std::snprintf(buf, sizeof buf, ".%06lld", static_cast<long long>(a_us)); + out += buf; + } + } + } + return out; +} + +// ---------------------------------------------------------------- astimezone (CPython port) +namespace { +// utc_to_seconds: seconds since 0001-01-01 for a broken-down UTC time, validating the year +int64_t utc_to_seconds(int y, int m, int d, int hh, int mm, int ss) { + if (y < 1 || y > 9999) throw std::invalid_argument("year must be in 1..9999, not " + std::to_string(y)); + return (ymd_to_ord(y, m, d) - 1) * 86400 + hh * 3600 + mm * 60 + ss; +} +int64_t local_of(int64_t u) { + std::time_t t = static_cast<std::time_t>(u - EPOCH_SECS); + std::tm lt{}; + if (!localtime_r(&t, <)) throw DateOverflow("date value out of range"); + return utc_to_seconds(lt.tm_year + 1900, lt.tm_mon + 1, lt.tm_mday, lt.tm_hour, lt.tm_min, lt.tm_sec); +} +int64_t local_to_seconds(int64_t t, int fold) { + static const int64_t max_fold_seconds = 24 * 3600; + int64_t lt = local_of(t); + int64_t a = lt - t; + int64_t u1 = t - a; + int64_t t1 = local_of(u1); + int64_t b; + if (t1 == t) { + int64_t u2 = fold ? u1 + max_fold_seconds : u1 - max_fold_seconds; + lt = local_of(u2); + b = lt - u2; + if (a == b) return u1; + } else { + b = t1 - u1; + } + int64_t u2 = t - b; + int64_t t2 = local_of(u2); + if (t2 == t) return u2; + if (t1 == t) return u1; + return fold ? std::min(u1, u2) : std::max(u1, u2); +} +int64_t gmtoff_at(int64_t seconds_since_year1) { + std::time_t t = static_cast<std::time_t>(seconds_since_year1 - EPOCH_SECS); + std::tm lt{}; + if (!localtime_r(&t, <)) throw DateOverflow("date value out of range"); + return lt.tm_gmtoff; +} +bool in_range(int64_t us) { + int64_t days = us >= 0 ? us / 86400000000LL : -((-us + 86399999999LL) / 86400000000LL); + return days + ORD_UNIX >= 1 && days + ORD_UNIX <= ymd_to_ord(9999, 12, 31); +} +} // namespace + +DateTime DateTime::to_utc() const { + if (offset_us_) { + int64_t inst = us_ - *offset_us_; + if (!in_range(inst)) throw DateOverflow("date value out of range"); + return with_offset_us(inst, 0); + } + int64_t floor_secs = us_ >= 0 ? us_ / 1000000 : -((-us_ + 999999) / 1000000); + int64_t micro = us_ - floor_secs * 1000000; + int64_t t = floor_secs + EPOCH_SECS; // seconds since 0001-01-01 of the wall clock + int64_t seconds = local_to_seconds(t, 0); + int64_t seconds2 = local_to_seconds(t, 1); + if (seconds2 != seconds && (seconds2 > seconds) == false) seconds = seconds2; + int64_t off = gmtoff_at(seconds); + int64_t inst = us_ - off * 1000000; + if (!in_range(inst)) throw DateOverflow("date value out of range"); + (void)micro; + return with_offset_us(inst, 0); +} + +DateTime DateTime::to_local() const { + DateTime utc = to_utc(); + int64_t floor_secs = utc.us_ >= 0 ? utc.us_ / 1000000 : -((-utc.us_ + 999999) / 1000000); + int64_t off = gmtoff_at(floor_secs + EPOCH_SECS); + int64_t shifted = utc.us_ + off * 1000000; + if (!in_range(shifted)) throw DateOverflow("date value out of range"); + return with_offset_us(shifted, off * 1000000); +} + +DateTime DateTime::plus_seconds(double sec) const { + DateTime d = *this; + d.us_ = us_ + static_cast<int64_t>(sec * 1000000.0 + (sec >= 0 ? 0.5 : -0.5)); + return d; +} + +double DateTime::seconds_since(const DateTime& other) const { return static_cast<double>(instant() - other.instant()) / 1e6; } + +std::string DateTime::strftime_minute() const { + std::string iso = isoformat(true); + return iso.substr(0, 10) + " " + iso.substr(11, 5); +} +} // namespace detz::misc diff --git a/taicpp/src/skills/miscdatetime.h b/taicpp/src/skills/miscdatetime.h new file mode 100644 index 0000000..2942660 --- /dev/null +++ b/taicpp/src/skills/miscdatetime.h @@ -0,0 +1,49 @@ +#pragma once +// datetime subset used by tai.py: datetime.now(timezone.utc), isoformat(), datetime.fromisoformat() as +// _fromisoformat wraps it, astimezone(), timedelta addition and comparison. The ISO parser and the +// local-time conversion are ports of CPython's _datetimemodule.c so edge cases (week dates, 24:00, one +// stray character before the offset, gap/fold local times) behave the same. +#include <cstdint> +#include <optional> +#include <stdexcept> +#include <string> +namespace detz::misc { + +// OverflowError: "date value out of range" +class DateOverflow : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +class DateTime { + public: + DateTime() = default; + // us: the wall-clock reading in microseconds since 1970-01-01 as if it were UTC. + // offset_sec empty means a naive datetime. + DateTime(int64_t us, std::optional<int> offset_sec) : us_(us) { + if (offset_sec) offset_us_ = static_cast<int64_t>(*offset_sec) * 1000000; + } + static DateTime with_offset_us(int64_t us, int64_t offset_us) { DateTime d; d.us_ = us; d.offset_us_ = offset_us; return d; } + static DateTime now_utc(); + // _fromisoformat: throws std::invalid_argument (ValueError). The text mirrors CPython. + static DateTime fromisoformat(const std::string& text); + static int64_t days_from_civil(int y, unsigned m, unsigned d); + static bool valid_date(int y, int m, int d); + + bool aware() const { return offset_us_.has_value(); } + int64_t us() const { return us_; } + int offset() const { return static_cast<int>(offset_us_.value_or(0) / 1000000); } + // isoformat(); timespec_seconds drops microseconds like isoformat(timespec="seconds") + std::string isoformat(bool timespec_seconds = false) const; + // astimezone(timezone.utc): ValueError("year must be in 1..9999, not N") for naive values, DateOverflow for aware ones + DateTime to_utc() const; + DateTime to_local() const; // astimezone() + DateTime plus_seconds(double sec) const; + double seconds_since(const DateTime& other) const; // (self - other).total_seconds() + bool operator<=(const DateTime& o) const { return instant() <= o.instant(); } + bool operator<(const DateTime& o) const { return instant() < o.instant(); } + std::string strftime_minute() const; // %Y-%m-%d %H:%M of the wall clock stored here + + private: + int64_t us_{0}; + std::optional<int64_t> offset_us_; + int64_t instant() const { return us_ - offset_us_.value_or(0); } +}; +} // namespace detz::misc diff --git a/taicpp/src/skills/miscjson.cpp b/taicpp/src/skills/miscjson.cpp new file mode 100644 index 0000000..4801caf --- /dev/null +++ b/taicpp/src/skills/miscjson.cpp @@ -0,0 +1,338 @@ +#include "miscjson.h" +#include <cctype> +#include <charconv> +#include <cmath> +#include <cstdio> +#include <cstdlib> +#include <cstring> +namespace detz::misc { +Json& Json::set(const std::string& key, Json v) { + kind_ = Kind::Object; + for (auto& m : members_) { + if (m.first == key) { m.second = std::move(v); return *this; } + } + members_.emplace_back(key, std::move(v)); + return *this; +} +const Json* Json::get(const std::string& key) const { + for (const auto& m : members_) { + if (m.first == key) return &m.second; + } + return nullptr; +} +std::string Json::str(const std::string& key, const std::string& dflt) const { + const Json* v = get(key); + return v && v->is_string() ? v->as_string() : dflt; +} + +static void put_u(std::string& out, unsigned cp) { + char buf[8]; + std::snprintf(buf, sizeof buf, "\\u%04x", cp); + out += buf; +} + +// Decode one UTF-8 code point; invalid bytes decode as U+FFFD. +static unsigned next_cp(const std::string& s, std::size_t& i) { + unsigned char c = static_cast<unsigned char>(s[i]); + if (c < 0x80) { ++i; return c; } + int n = c >= 0xF0 ? 3 : c >= 0xE0 ? 2 : c >= 0xC0 ? 1 : -1; + if (n < 0 || i + static_cast<std::size_t>(n) >= s.size()) { ++i; return 0xFFFD; } + unsigned cp = c & (0x3Fu >> n); + for (int k = 1; k <= n; ++k) { + unsigned char d = static_cast<unsigned char>(s[i + k]); + if ((d & 0xC0) != 0x80) { ++i; return 0xFFFD; } + cp = (cp << 6) | (d & 0x3Fu); + } + i += static_cast<std::size_t>(n) + 1; + return cp; +} + +std::string Json::quote(const std::string& s) { + std::string out = "\""; + for (std::size_t i = 0; i < s.size();) { + unsigned cp = next_cp(s, i); + switch (cp) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (cp >= 0x20 && cp <= 0x7e) { + out += static_cast<char>(cp); + } else if (cp < 0x10000) { + put_u(out, cp); + } else { + cp -= 0x10000; + put_u(out, 0xD800 | (cp >> 10)); + put_u(out, 0xDC00 | (cp & 0x3FF)); + } + } + } + return out + "\""; +} + +std::string Json::float_repr(double d) { + if (std::isnan(d)) return "NaN"; + if (std::isinf(d)) return d > 0 ? "Infinity" : "-Infinity"; + if (d == 0) return std::signbit(d) ? "-0.0" : "0.0"; + char buf[64]; + auto r = std::to_chars(buf, buf + sizeof buf, d, std::chars_format::scientific); + std::string sci(buf, r.ptr); + std::size_t epos = sci.find('e'); + std::string mant = sci.substr(0, epos); + int exp = std::atoi(sci.c_str() + epos + 1); + bool neg = mant[0] == '-'; + if (neg) mant.erase(0, 1); + std::string digits; + for (char c : mant) { + if (c != '.') digits += c; + } + std::string out; + if (exp < -4 || exp >= 16) { + out = digits.substr(0, 1); + if (digits.size() > 1) out += "." + digits.substr(1); + char eb[16]; + std::snprintf(eb, sizeof eb, "e%c%02d", exp < 0 ? '-' : '+', std::abs(exp)); + out += eb; + } else if (exp >= 0) { + if (static_cast<int>(digits.size()) <= exp + 1) { + out = digits + std::string(static_cast<std::size_t>(exp + 1) - digits.size(), '0') + ".0"; + } else { + out = digits.substr(0, static_cast<std::size_t>(exp) + 1) + "." + digits.substr(static_cast<std::size_t>(exp) + 1); + } + } else { + out = "0." + std::string(static_cast<std::size_t>(-exp - 1), '0') + digits; + } + return neg ? "-" + out : out; +} + +void Json::dump_to(std::string& out) const { + switch (kind_) { + case Kind::Null: out += "null"; break; + case Kind::Bool: out += b_ ? "true" : "false"; break; + case Kind::Int: out += big_ ? s_ : std::to_string(i_); break; + case Kind::Float: out += float_repr(d_); break; + case Kind::String: out += quote(s_); break; + case Kind::Array: { + out += "["; + for (std::size_t k = 0; k < items_.size(); ++k) { + if (k) out += ", "; + items_[k].dump_to(out); + } + out += "]"; + break; + } + case Kind::Object: { + out += "{"; + for (std::size_t k = 0; k < members_.size(); ++k) { + if (k) out += ", "; + out += quote(members_[k].first) + ": "; + members_[k].second.dump_to(out); + } + out += "}"; + break; + } + } +} +std::string Json::dumps() const { + std::string out; + dump_to(out); + return out; +} + +namespace { +class Parser { + public: + explicit Parser(const std::string& t) : t_(t) {} + Json run() { + ws(); + Json v = value(); + ws(); + if (p_ != t_.size()) fail("Extra data"); + return v; + } + + private: + const std::string& t_; + std::size_t p_{0}; + int depth_{0}; + [[noreturn]] void fail(const std::string& what) { fail_at(what, p_, ": "); } + // Python's JSONDecodeError text: "<msg>: line L column C (char N)" + [[noreturn]] void fail_at(const std::string& what, std::size_t pos, const char* sep) { + std::size_t line = 1, last_nl = std::string::npos; + for (std::size_t i = 0; i < pos && i < t_.size(); ++i) { + if (t_[i] == '\n') { ++line; last_nl = i; } + } + std::size_t col = last_nl == std::string::npos ? pos + 1 : pos - last_nl; + throw JsonError(what + sep + "line " + std::to_string(line) + " column " + std::to_string(col) + " (char " + std::to_string(pos) + ")"); + } + void ws() { + while (p_ < t_.size() && (t_[p_] == ' ' || t_[p_] == '\t' || t_[p_] == '\n' || t_[p_] == '\r')) ++p_; + } + bool lit(const char* word) { + std::size_t n = std::strlen(word); + if (t_.compare(p_, n, word) == 0) { p_ += n; return true; } + return false; + } + Json value() { + if (p_ >= t_.size()) fail("Expecting value"); + if (++depth_ > 200) fail("recursion"); + struct Guard { int& d; ~Guard() { --d; } } g{depth_}; + char c = t_[p_]; + if (c == '{') return object(); + if (c == '[') return array(); + if (c == '"') return Json(string()); + if (lit("true")) return Json(true); + if (lit("false")) return Json(false); + if (lit("null")) return Json(); + if (lit("NaN")) return Json(std::nan("")); + if (lit("Infinity")) return Json(HUGE_VAL); + if (lit("-Infinity")) return Json(-HUGE_VAL); + return number(); + } + Json number() { + std::size_t s = p_; + if (p_ < t_.size() && t_[p_] == '-') ++p_; + if (p_ >= t_.size() || !std::isdigit(static_cast<unsigned char>(t_[p_]))) { p_ = s; fail("Expecting value"); } + if (t_[p_] == '0') { + ++p_; + } else { + while (p_ < t_.size() && std::isdigit(static_cast<unsigned char>(t_[p_]))) ++p_; + } + bool isf = false; + if (p_ + 1 < t_.size() && t_[p_] == '.' && std::isdigit(static_cast<unsigned char>(t_[p_ + 1]))) { + isf = true; + ++p_; + while (p_ < t_.size() && std::isdigit(static_cast<unsigned char>(t_[p_]))) ++p_; + } + if (p_ < t_.size() && (t_[p_] == 'e' || t_[p_] == 'E')) { + std::size_t q = p_ + 1; + if (q < t_.size() && (t_[q] == '+' || t_[q] == '-')) ++q; + if (q < t_.size() && std::isdigit(static_cast<unsigned char>(t_[q]))) { + isf = true; + p_ = q; + while (p_ < t_.size() && std::isdigit(static_cast<unsigned char>(t_[p_]))) ++p_; + } + } + std::string tok = t_.substr(s, p_ - s); + if (!isf) { + int64_t v = 0; + auto r = std::from_chars(tok.data(), tok.data() + tok.size(), v); + if (r.ec == std::errc()) return Json(v); + return Json::big_int(tok == "-0" ? "0" : tok); // beyond int64: exact digits, like a Python int + } + return Json(std::strtod(tok.c_str(), nullptr)); + } + static void utf8(std::string& out, unsigned cp) { + if (cp < 0x80) { + out += static_cast<char>(cp); + } else if (cp < 0x800) { + out += static_cast<char>(0xC0 | (cp >> 6)); + out += static_cast<char>(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + out += static_cast<char>(0xE0 | (cp >> 12)); + out += static_cast<char>(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast<char>(0x80 | (cp & 0x3F)); + } else { + out += static_cast<char>(0xF0 | (cp >> 18)); + out += static_cast<char>(0x80 | ((cp >> 12) & 0x3F)); + out += static_cast<char>(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast<char>(0x80 | (cp & 0x3F)); + } + } + unsigned hex4() { + if (p_ + 4 > t_.size()) fail("Invalid \\uXXXX escape"); + unsigned v = 0; + for (int k = 0; k < 4; ++k) { + char c = t_[p_ + static_cast<std::size_t>(k)]; + v <<= 4; + if (c >= '0' && c <= '9') v |= static_cast<unsigned>(c - '0'); + else if (c >= 'a' && c <= 'f') v |= static_cast<unsigned>(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v |= static_cast<unsigned>(c - 'A' + 10); + else fail("Invalid \\uXXXX escape"); + } + p_ += 4; + return v; + } + std::string string() { + std::size_t start = p_; + ++p_; + std::string out; + while (true) { + if (p_ >= t_.size()) fail_at("Unterminated string starting at", start, ": "); + unsigned char c = static_cast<unsigned char>(t_[p_]); + if (c == '"') { ++p_; return out; } + if (c < 0x20) fail_at("Invalid control character at", p_, ": "); + if (c != '\\') { out += static_cast<char>(c); ++p_; continue; } + ++p_; + if (p_ >= t_.size()) fail_at("Unterminated string starting at", start, ": "); + char e = t_[p_++]; + switch (e) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + unsigned cp = hex4(); + if (cp >= 0xD800 && cp < 0xDC00 && t_.compare(p_, 2, "\\u") == 0) { + std::size_t save = p_; + p_ += 2; + unsigned lo = hex4(); + if (lo >= 0xDC00 && lo < 0xE000) cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + else p_ = save; + } + utf8(out, cp); + break; + } + default: fail("Invalid \\escape"); + } + } + } + Json array() { + Json a = Json::array(); + ++p_; + ws(); + if (p_ < t_.size() && t_[p_] == ']') { ++p_; return a; } + while (true) { + ws(); + a.push(value()); + ws(); + if (p_ >= t_.size()) fail("Expecting ',' delimiter"); + if (t_[p_] == ',') { ++p_; continue; } + if (t_[p_] == ']') { ++p_; return a; } + fail("Expecting ',' delimiter"); + } + } + Json object() { + Json o = Json::object(); + ++p_; + ws(); + if (p_ < t_.size() && t_[p_] == '}') { ++p_; return o; } + while (true) { + ws(); + if (p_ >= t_.size() || t_[p_] != '"') fail("Expecting property name enclosed in double quotes"); + std::string key = string(); + ws(); + if (p_ >= t_.size() || t_[p_] != ':') fail("Expecting ':' delimiter"); + ++p_; + ws(); + o.set(key, value()); + ws(); + if (p_ >= t_.size()) fail("Expecting ',' delimiter"); + if (t_[p_] == ',') { ++p_; continue; } + if (t_[p_] == '}') { ++p_; return o; } + fail("Expecting ',' delimiter"); + } + } +}; +} // namespace + +Json Json::loads(const std::string& text) { return Parser(text).run(); } +} // namespace detz::misc diff --git a/taicpp/src/skills/miscjson.h b/taicpp/src/skills/miscjson.h new file mode 100644 index 0000000..5c712ab --- /dev/null +++ b/taicpp/src/skills/miscjson.h @@ -0,0 +1,68 @@ +#pragma once +// Python-compatible JSON value: json.dumps (ensure_ascii, ", " and ": " separators, insertion-ordered +// objects) and json.loads. Shared by skills, sched, telegram and install. +#include <cstdint> +#include <stdexcept> +#include <string> +#include <utility> +#include <vector> +namespace detz::misc { +class JsonError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; +class Json { + public: + enum class Kind { Null, Bool, Int, Float, String, Array, Object }; + Json() = default; + Json(bool v) : kind_(Kind::Bool), b_(v) {} + Json(int v) : kind_(Kind::Int), i_(v) {} + Json(int64_t v) : kind_(Kind::Int), i_(v) {} + Json(double v) : kind_(Kind::Float), d_(v) {} + Json(const char* v) : kind_(Kind::String), s_(v) {} + Json(std::string v) : kind_(Kind::String), s_(std::move(v)) {} + // An integer beyond int64 keeps its exact decimal digits (Python ints are unbounded) + static Json big_int(std::string digits) { Json j; j.kind_ = Kind::Int; j.big_ = true; j.s_ = std::move(digits); return j; } + bool is_big_int() const { return kind_ == Kind::Int && big_; } + static Json array() { Json j; j.kind_ = Kind::Array; return j; } + static Json object() { Json j; j.kind_ = Kind::Object; return j; } + + Kind kind() const { return kind_; } + bool is_null() const { return kind_ == Kind::Null; } + bool is_bool() const { return kind_ == Kind::Bool; } + bool is_int() const { return kind_ == Kind::Int; } + bool is_float() const { return kind_ == Kind::Float; } + bool is_string() const { return kind_ == Kind::String; } + bool is_array() const { return kind_ == Kind::Array; } + bool is_object() const { return kind_ == Kind::Object; } + bool as_bool() const { return b_; } + int64_t as_int() const { return kind_ == Kind::Float ? static_cast<int64_t>(d_) : i_; } + double as_double() const { return kind_ == Kind::Float ? d_ : static_cast<double>(i_); } + const std::string& as_string() const { return s_; } + + // Array + Json& push(Json v) { items_.push_back(std::move(v)); return *this; } + const std::vector<Json>& items() const { return items_; } + std::size_t size() const { return kind_ == Kind::Object ? members_.size() : items_.size(); } + // Object (insertion ordered; replacing an existing key keeps its position like a Python dict) + Json& set(const std::string& key, Json v); + const Json* get(const std::string& key) const; + bool has(const std::string& key) const { return get(key) != nullptr; } + const std::vector<std::pair<std::string, Json>>& members() const { return members_; } + // dict.get(key) when it is a str, else the default + std::string str(const std::string& key, const std::string& dflt = "") const; + + std::string dumps() const; + static Json loads(const std::string& text); + static std::string quote(const std::string& s); // json.dumps(str) + static std::string float_repr(double d); // repr(float) + + private: + Kind kind_{Kind::Null}; + bool b_{false}; + bool big_{false}; + int64_t i_{0}; + double d_{0}; + std::string s_; + std::vector<Json> items_; + std::vector<std::pair<std::string, Json>> members_; + void dump_to(std::string& out) const; +}; +} // namespace detz::misc diff --git a/taicpp/src/skills/miscpy.cpp b/taicpp/src/skills/miscpy.cpp new file mode 100644 index 0000000..72c3efe --- /dev/null +++ b/taicpp/src/skills/miscpy.cpp @@ -0,0 +1,546 @@ +#include "miscpy.h" +#include "pyunicode.h" +#include <fcntl.h> +#include <poll.h> +#include <pwd.h> +#include <signal.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <unistd.h> +#include <algorithm> +#include <cerrno> +#include <chrono> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <filesystem> +#include <fstream> +#include <sstream> +namespace fs = std::filesystem; +namespace detz::misc { + +// ---------------------------------------------------------------- PyStr +namespace { +unsigned decode(const std::string& s, std::size_t i, std::size_t& n) { return PyUnicode::decode(s, i, n); } +bool is_space(unsigned cp) { return PyUnicode::is_space(cp); } +bool is_line_break(unsigned cp) { + return cp == 0x0A || cp == 0x0B || cp == 0x0C || cp == 0x0D || cp == 0x1C || cp == 0x1D || cp == 0x1E || cp == 0x85 || cp == 0x2028 || cp == 0x2029; +} +} // namespace + +std::size_t PyStr::len(const std::string& s) { + std::size_t n = 0; + for (std::size_t i = 0; i < s.size();) { + std::size_t k; + decode(s, i, k); + i += k; + ++n; + } + return n; +} +std::string PyStr::prefix(const std::string& s, std::size_t n) { + std::size_t i = 0, count = 0; + while (i < s.size() && count < n) { + std::size_t k; + decode(s, i, k); + i += k; + ++count; + } + return s.substr(0, i); +} +std::string PyStr::suffix(const std::string& s, std::size_t n) { + std::size_t total = len(s); + if (n >= total) return s; + return s.substr(prefix(s, total - n).size()); +} +std::vector<std::string> PyStr::splitlines(const std::string& s) { + std::vector<std::string> out; + std::size_t start = 0, i = 0; + while (i < s.size()) { + std::size_t k; + unsigned cp = decode(s, i, k); + if (is_line_break(cp)) { + out.push_back(s.substr(start, i - start)); + i += k; + if (cp == 0x0D && i < s.size() && s[i] == '\n') ++i; + start = i; + } else { + i += k; + } + } + if (start < s.size()) out.push_back(s.substr(start)); + return out; +} +std::string PyStr::strip(const std::string& s) { + std::size_t b = 0; + while (b < s.size()) { + std::size_t k; + unsigned cp = decode(s, b, k); + if (!is_space(cp)) break; + b += k; + } + std::size_t e = s.size(); + while (e > b) { + std::size_t st = e - 1; + while (st > b && (static_cast<unsigned char>(s[st]) & 0xC0) == 0x80) --st; + std::size_t k; + unsigned cp = decode(s, st, k); + if (!is_space(cp)) break; + e = st; + } + return s.substr(b, e - b); +} +std::string PyStr::rstrip_nl(const std::string& s) { + std::string r = s; + while (!r.empty() && (r.back() == '\n' || r.back() == '\r')) r.pop_back(); + return r; +} +std::string PyStr::lower_ascii_view(const std::string& s) { + std::string out; + for (std::size_t i = 0; i < s.size();) { + std::size_t k; + unsigned cp = decode(s, i, k); + i += k; + if (cp >= 'A' && cp <= 'Z') out += static_cast<char>(cp + 32); + else if (cp < 0x80) out += static_cast<char>(cp); + else if (cp == 0x212A) out += 'k'; + else if (cp == 0x0130) out += "i\xCC\x87"; + else out += "\xC2\x80"; // some non-ASCII, non-alnum code point: only matters as a separator + } + return out; +} +std::vector<std::string> PyStr::alnum_runs(const std::string& s) { + std::vector<std::string> out; + std::string cur; + for (char c : lower_ascii_view(s)) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + cur += c; + } else if (!cur.empty()) { + out.push_back(cur); + cur.clear(); + } + } + if (!cur.empty()) out.push_back(cur); + return out; +} +std::string PyStr::join(const std::vector<std::string>& parts, const std::string& sep) { + std::string out; + for (std::size_t i = 0; i < parts.size(); ++i) { + if (i) out += sep; + out += parts[i]; + } + return out; +} +std::string PyStr::replace_all(std::string s, const std::string& from, const std::string& to) { + if (from.empty()) return s; + std::size_t pos = 0; + while ((pos = s.find(from, pos)) != std::string::npos) { + s.replace(pos, from.size(), to); + pos += to.size(); + } + return s; +} +std::string PyStr::repr(const std::string& s) { + bool has_single = s.find('\'') != std::string::npos; + bool has_double = s.find('"') != std::string::npos; + char quote = (has_single && !has_double) ? '"' : '\''; + std::string out(1, quote); + for (unsigned char c : s) { + if (c == static_cast<unsigned char>(quote) || c == '\\') { out += '\\'; out += static_cast<char>(c); } + else if (c == '\n') out += "\\n"; + else if (c == '\r') out += "\\r"; + else if (c == '\t') out += "\\t"; + else if (c < 0x20 || c == 0x7f) { char b[8]; std::snprintf(b, sizeof b, "\\x%02x", c); out += b; } + else out += static_cast<char>(c); + } + out += quote; + return out; +} +std::string PyStr::short_error(const std::string& message, const std::string& type_name) { + return prefix(message.empty() ? type_name : message, 300); +} +std::string PyStr::percent(const std::string& fmt, const std::vector<std::string>& args) { + std::string out; + std::size_t next = 0; + for (std::size_t i = 0; i < fmt.size(); ++i) { + if (fmt[i] == '%' && i + 1 < fmt.size()) { + if (fmt[i + 1] == '%') { out += '%'; ++i; continue; } + if (fmt[i + 1] == 's') { + if (next >= args.size()) throw std::invalid_argument("not enough arguments for format string"); + out += args[next++]; + ++i; + continue; + } + } + out += fmt[i]; + } + if (next != args.size()) throw std::invalid_argument("not all arguments converted during string formatting"); + return out; +} +bool PyStr::isdigit_ascii(const std::string& s) { + if (s.empty()) return false; + return std::all_of(s.begin(), s.end(), [](unsigned char c) { return c >= '0' && c <= '9'; }); +} + +// ---------------------------------------------------------------- TagNames +std::string TagNames::normalize_tag(const std::string& raw) { + std::string low = PyStr::lower_ascii_view(PyStr::strip(raw)); + std::string out; + bool pending = false; + for (char c : low) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + if (pending && !out.empty()) out += '-'; + pending = false; + out += c; + } else { + pending = true; + } + } + // re.sub(r"[^a-z0-9]+", "-", s).strip("-")[:32]: outer runs are dropped above; the cut may leave a trailing "-" + return out.substr(0, 32); +} +std::vector<std::string> TagNames::normalize_tags(const std::vector<std::string>& items) { + std::vector<std::string> seen; + for (const auto& item : items) { + std::string cleaned = normalize_tag(item); + if (!cleaned.empty() && std::find(seen.begin(), seen.end(), cleaned) == seen.end()) seen.push_back(cleaned); + } + if (seen.size() > 20) seen.resize(20); + return seen; +} + +// ---------------------------------------------------------------- OsError / PyFs +OsError::OsError(int err, const std::string& filename) + : std::runtime_error("[Errno " + std::to_string(err) + "] " + std::strerror(err) + (filename.empty() ? "" : ": " + PyStr::repr(filename))), err_(err) {} + +std::string PyFs::home() { + const char* h = std::getenv("HOME"); + if (h) return h; + struct passwd* pw = getpwuid(getuid()); + return pw && pw->pw_dir ? pw->pw_dir : "~"; +} +std::string PyFs::expanduser(const std::string& path) { + if (path.empty() || path[0] != '~') return path; + std::size_t slash = path.find('/'); + std::string user = path.substr(1, slash == std::string::npos ? std::string::npos : slash - 1); + std::string rest = slash == std::string::npos ? "" : path.substr(slash); + std::string base; + if (user.empty()) { + base = home(); + } else { + struct passwd* pw = getpwnam(user.c_str()); + if (!pw) return path; + base = pw->pw_dir; + } + while (base.size() > 1 && base.back() == '/' && !rest.empty()) base.pop_back(); + if (base == "/" && !rest.empty()) return rest; + return base + rest; +} +std::string PyFs::join(const std::string& a, const std::string& b) { + if (!b.empty() && b[0] == '/') return b; + if (a.empty() || a.back() == '/') return a + b; + return a + "/" + b; +} +std::string PyFs::dirname(const std::string& p) { + std::size_t i = p.rfind('/'); + if (i == std::string::npos) return ""; + std::string head = p.substr(0, i + 1); + if (head.find_first_not_of('/') != std::string::npos) { + while (head.size() > 1 && head.back() == '/') head.pop_back(); + } + return head; +} +std::string PyFs::basename(const std::string& p) { + std::size_t i = p.rfind('/'); + return i == std::string::npos ? p : p.substr(i + 1); +} +std::string PyFs::normpath(const std::string& p) { + if (p.empty()) return "."; + int initial = 0; + if (p[0] == '/') { + initial = 1; + if (p.size() >= 2 && p[1] == '/' && (p.size() == 2 || p[2] != '/')) initial = 2; + } + std::vector<std::string> comps, out; + std::stringstream ss(p); + std::string part; + while (std::getline(ss, part, '/')) comps.push_back(part); + for (const auto& c : comps) { + if (c.empty() || c == ".") continue; + if (c != ".." || (!initial && out.empty()) || (!out.empty() && out.back() == "..")) out.push_back(c); + else if (!out.empty()) out.pop_back(); + } + std::string r = PyStr::join(out, "/"); + if (initial) r = std::string(static_cast<std::size_t>(initial), '/') + r; + return r.empty() ? "." : r; +} +std::string PyFs::abspath(const std::string& p) { + if (!p.empty() && p[0] == '/') return normpath(p); + char buf[4096]; + std::string cwd = getcwd(buf, sizeof buf) ? buf : "/"; + return normpath(join(cwd, p)); +} +std::string PyFs::relpath(const std::string& p, const std::string& start) { + auto split = [](const std::string& s) { + std::vector<std::string> v; + std::stringstream ss(abspath(s)); + std::string part; + while (std::getline(ss, part, '/')) { + if (!part.empty()) v.push_back(part); + } + return v; + }; + auto a = split(p), b = split(start); + std::size_t i = 0; + while (i < a.size() && i < b.size() && a[i] == b[i]) ++i; + std::vector<std::string> out(b.size() - i, ".."); + out.insert(out.end(), a.begin() + static_cast<std::ptrdiff_t>(i), a.end()); + return out.empty() ? "." : PyStr::join(out, "/"); +} +bool PyFs::exists(const std::string& p) { struct stat st; return stat(p.c_str(), &st) == 0; } +bool PyFs::isfile(const std::string& p) { struct stat st; return stat(p.c_str(), &st) == 0 && S_ISREG(st.st_mode); } +bool PyFs::isdir(const std::string& p) { struct stat st; return stat(p.c_str(), &st) == 0 && S_ISDIR(st.st_mode); } +bool PyFs::islink(const std::string& p) { struct stat st; return lstat(p.c_str(), &st) == 0 && S_ISLNK(st.st_mode); } +bool PyFs::is_executable(const std::string& p) { return access(p.c_str(), X_OK) == 0; } + +std::vector<std::string> PyFs::listdir_sorted(const std::string& p) { + std::error_code ec; + std::vector<std::string> names; + fs::directory_iterator it(p, ec); + if (ec) throw OsError(ec.value(), p); + for (const auto& e : it) names.push_back(e.path().filename().string()); + std::sort(names.begin(), names.end()); // byte order; Python sorts code points, identical for UTF-8 + return names; +} +std::string PyFs::read_text(const std::string& p) { + int fd = ::open(p.c_str(), O_RDONLY); + if (fd < 0) throw OsError(errno, p); + struct stat st; + if (fstat(fd, &st) == 0 && S_ISDIR(st.st_mode)) { ::close(fd); throw OsError(EISDIR, p); } + std::string raw; + char buf[65536]; + ssize_t n; + while ((n = ::read(fd, buf, sizeof buf)) > 0) raw.append(buf, static_cast<std::size_t>(n)); + int err = errno; + ::close(fd); + if (n < 0) throw OsError(err, p); + // text mode: universal newlines + std::string out; + out.reserve(raw.size()); + for (std::size_t i = 0; i < raw.size(); ++i) { + if (raw[i] == '\r') { + out += '\n'; + if (i + 1 < raw.size() && raw[i + 1] == '\n') ++i; + } else { + out += raw[i]; + } + } + return out; +} +std::string PyFs::read_bytes(const std::string& p) { + int fd = ::open(p.c_str(), O_RDONLY); + if (fd < 0) throw OsError(errno, p); + std::string raw; + char buf[65536]; + ssize_t n; + while ((n = ::read(fd, buf, sizeof buf)) > 0) raw.append(buf, static_cast<std::size_t>(n)); + int err = errno; + ::close(fd); + if (n < 0) throw OsError(err, p); + return raw; +} +void PyFs::write_bytes(const std::string& p, const std::string& data) { + int fd = ::open(p.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) throw OsError(errno, p); + std::size_t off = 0; + while (off < data.size()) { + ssize_t n = ::write(fd, data.data() + off, data.size() - off); + if (n < 0) { int err = errno; ::close(fd); throw OsError(err, p); } + off += static_cast<std::size_t>(n); + } + ::close(fd); +} +void PyFs::write_text(const std::string& p, const std::string& content) { + int fd = ::open(p.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) throw OsError(errno, p); + std::size_t off = 0; + while (off < content.size()) { + ssize_t n = ::write(fd, content.data() + off, content.size() - off); + if (n < 0) { int err = errno; ::close(fd); throw OsError(err, p); } + off += static_cast<std::size_t>(n); + } + ::close(fd); +} +void PyFs::makedirs(const std::string& p) { + if (p.empty()) throw OsError(ENOENT, ""); + std::error_code ec; + fs::create_directories(p, ec); + if (ec && !isdir(p)) throw OsError(ec.value(), p); +} +bool PyFs::remove(const std::string& p) { + if (::unlink(p.c_str()) != 0) throw OsError(errno, p); + return true; +} +void PyFs::rmtree(const std::string& p) { + std::error_code ec; + fs::remove_all(p, ec); + if (ec) throw OsError(ec.value(), p); +} +void PyFs::chmod(const std::string& p, unsigned mode) { + if (::chmod(p.c_str(), mode) != 0) throw OsError(errno, p); +} +void PyFs::atomic_write(const std::string& path, const std::string& content) { + std::string tmp = path + ".tmp-" + std::to_string(getpid()); + int fd = ::open(tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) throw OsError(errno, tmp); + std::size_t off = 0; + while (off < content.size()) { + ssize_t n = ::write(fd, content.data() + off, content.size() - off); + if (n < 0) { int err = errno; ::close(fd); throw OsError(err, tmp); } + off += static_cast<std::size_t>(n); + } + ::fsync(fd); + ::close(fd); + if (::rename(tmp.c_str(), path.c_str()) != 0) throw OsError(errno, tmp); +} +std::optional<std::string> PyFs::which(const std::string& name) { + if (name.find('/') != std::string::npos) { + return (isfile(name) && is_executable(name)) ? std::optional<std::string>(name) : std::nullopt; + } + const char* path = std::getenv("PATH"); + std::string paths = path ? path : "/usr/local/bin:/usr/bin:/bin"; + std::stringstream ss(paths); + std::string dir; + while (std::getline(ss, dir, ':')) { + std::string cand = join(dir.empty() ? "." : dir, name); + if (isfile(cand) && is_executable(cand)) return cand; + } + return std::nullopt; +} +std::string PyFs::env(const std::string& name, const std::string& dflt) { + const char* v = std::getenv(name.c_str()); + return (v && *v) ? v : dflt; +} +std::vector<std::string> PyFs::walk_files(const std::string& root) { + std::vector<std::string> out; + std::error_code ec; + fs::directory_iterator it(root, ec); + if (ec) return out; + for (const auto& e : it) { + std::string full = e.path().string(); + std::error_code e2; + bool is_dir = fs::is_directory(e.path(), e2); + if (is_dir) { + if (!fs::is_symlink(e.symlink_status())) { + auto sub = walk_files(full); + out.insert(out.end(), sub.begin(), sub.end()); + } + } else { + out.push_back(full); + } + } + return out; +} + +// ---------------------------------------------------------------- PyProc +HostResult PyProc::run_host(const std::vector<std::string>& argv, int timeout) { + ProcResult r = run(argv, timeout); + if (r.failed) return {false, r.error}; + return {r.code == 0, r.out + r.err}; +} + +ProcResult PyProc::run(const std::vector<std::string>& argv, int timeout) { + ProcResult fail; + fail.failed = true; + int outp[2], errp[2], execp[2]; + if (pipe(outp) != 0 || pipe(errp) != 0 || pipe2(execp, O_CLOEXEC) != 0) { fail.error = "[Errno " + std::to_string(errno) + "] " + std::strerror(errno); return fail; } + pid_t pid = fork(); + if (pid < 0) { fail.error = "[Errno " + std::to_string(errno) + "] " + std::strerror(errno); return fail; } + if (pid == 0) { + dup2(outp[1], 1); + dup2(errp[1], 2); + ::close(outp[0]); ::close(outp[1]); ::close(errp[0]); ::close(errp[1]); ::close(execp[0]); + std::vector<char*> args; + for (const auto& a : argv) args.push_back(const_cast<char*>(a.c_str())); + args.push_back(nullptr); + execvp(args[0], args.data()); + int err = errno; + ssize_t w = ::write(execp[1], &err, sizeof err); + (void)w; + _exit(127); + } + ::close(outp[1]); ::close(errp[1]); ::close(execp[1]); + int exec_err = 0; + ssize_t got = ::read(execp[0], &exec_err, sizeof exec_err); + ::close(execp[0]); + if (got == static_cast<ssize_t>(sizeof exec_err)) { + int status; + waitpid(pid, &status, 0); + ::close(outp[0]); ::close(errp[0]); + fail.error = PyStr::short_error(OsError(exec_err, argv.empty() ? "" : argv[0]).what()); + return fail; + } + std::string out, err; + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout); + bool timed_out = false; + struct pollfd fds[2] = {{outp[0], POLLIN, 0}, {errp[0], POLLIN, 0}}; + int open_fds = 2; + while (open_fds > 0) { + auto left = std::chrono::duration_cast<std::chrono::milliseconds>(deadline - std::chrono::steady_clock::now()).count(); + if (left <= 0) { timed_out = true; break; } + int rc = poll(fds, 2, static_cast<int>(std::min<long long>(left, 1000))); + if (rc < 0 && errno != EINTR) break; + for (int k = 0; k < 2; ++k) { + if (fds[k].fd < 0) continue; + if (fds[k].revents & (POLLIN | POLLHUP)) { + char buf[8192]; + ssize_t n = ::read(fds[k].fd, buf, sizeof buf); + if (n > 0) (k == 0 ? out : err).append(buf, static_cast<std::size_t>(n)); + else { ::close(fds[k].fd); fds[k].fd = -1; --open_fds; } + } + } + } + if (timed_out) { + kill(pid, SIGKILL); + int status; + waitpid(pid, &status, 0); + for (int k = 0; k < 2; ++k) if (fds[k].fd >= 0) ::close(fds[k].fd); + std::string cmd = "["; + for (std::size_t i = 0; i < argv.size(); ++i) { if (i) cmd += ", "; cmd += PyStr::repr(argv[i]); } + cmd += "]"; + fail.error = PyStr::short_error("Command '" + cmd + "' timed out after " + std::to_string(timeout) + " seconds"); + return fail; + } + int status = 0; + waitpid(pid, &status, 0); + // universal newlines on both streams + auto norm = [](const std::string& s) { + std::string o; + for (std::size_t i = 0; i < s.size(); ++i) { + if (s[i] == '\r') { o += '\n'; if (i + 1 < s.size() && s[i + 1] == '\n') ++i; } + else o += s[i]; + } + return o; + }; + ProcResult res; + res.code = WIFEXITED(status) ? WEXITSTATUS(status) : -WTERMSIG(status); + res.out = norm(out); + res.err = norm(err); + return res; +} + +// ---------------------------------------------------------------- LogSink +namespace { +class NullSink : public LogSink { + public: + void event(const std::string&, const std::string&, const std::string&, const std::string&, const std::string&, + const std::string&, const std::string&, const Json&) override {} + void set_context(const std::string&, const std::string&) override {} + void clear_context(const std::string&) override {} +}; +} // namespace +LogSink& LogSink::null_sink() { + static NullSink sink; + return sink; +} +} // namespace detz::misc diff --git a/taicpp/src/skills/miscpy.h b/taicpp/src/skills/miscpy.h new file mode 100644 index 0000000..7d02c36 --- /dev/null +++ b/taicpp/src/skills/miscpy.h @@ -0,0 +1,113 @@ +#pragma once +// Small Python-semantics helpers the ported modules need to stay literal: str.splitlines/strip/len/slice +// over code points, short_error, os/shutil/subprocess equivalents, and the log sink port. +#include <optional> +#include <stdexcept> +#include <string> +#include <utility> +#include <vector> +#include "miscjson.h" +namespace detz::misc { + +class PyStr { + public: + static std::size_t len(const std::string& s); // len(str) in code points + static std::string prefix(const std::string& s, std::size_t n); // s[:n] + static std::string suffix(const std::string& s, std::size_t n); // s[-n:] + static std::vector<std::string> splitlines(const std::string& s); // str.splitlines() + static std::string strip(const std::string& s); // str.strip() + static std::string rstrip_nl(const std::string& s); + static std::string lower_ascii_view(const std::string& s); // str.lower(), keeping only what [a-z0-9] regexes can see + static std::vector<std::string> alnum_runs(const std::string& s); // re.findall(r"[a-z0-9]+", s.lower()) + static bool starts_with(const std::string& s, const std::string& p) { return s.compare(0, p.size(), p) == 0; } + static bool ends_with(const std::string& s, const std::string& p) { return s.size() >= p.size() && s.compare(s.size() - p.size(), p.size(), p) == 0; } + static std::string join(const std::vector<std::string>& parts, const std::string& sep); + static std::string replace_all(std::string s, const std::string& from, const std::string& to); + static std::string repr(const std::string& s); // repr(str) + static std::string short_error(const std::string& message, const std::string& type_name = ""); // short_error(exc) + static bool isdigit_ascii(const std::string& s); + // fmt % (args...) for a format that only uses %s and %% + static std::string percent(const std::string& fmt, const std::vector<std::string>& args); +}; + +// BackendError of tai.py (container/engine failures), shared by the modules that talk to the box. +class BackendError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +// normalize_tag / normalize_tags +class TagNames { + public: + static std::string normalize_tag(const std::string& raw); + static std::vector<std::string> normalize_tags(const std::vector<std::string>& items); +}; + +struct HostResult { bool ok{false}; std::string out; }; + +// OSError: str(exc) reads "[Errno N] strerror: 'filename'" exactly like CPython. +class OsError : public std::runtime_error { + public: + OsError(int err, const std::string& filename); + int err() const { return err_; } + private: + int err_; +}; + +class PyFs { + public: + static std::string home(); // os.path.expanduser("~") + static std::string expanduser(const std::string& path); + static std::string join(const std::string& a, const std::string& b); // os.path.join + static std::string dirname(const std::string& p); + static std::string basename(const std::string& p); + static std::string normpath(const std::string& p); + static std::string abspath(const std::string& p); + static std::string relpath(const std::string& p, const std::string& start); + static bool exists(const std::string& p); + static bool isfile(const std::string& p); + static bool isdir(const std::string& p); + static bool islink(const std::string& p); + static bool is_executable(const std::string& p); + static std::vector<std::string> listdir_sorted(const std::string& p); // sorted(os.listdir(p)); throws on error + static std::string read_text(const std::string& p); // throws std::runtime_error (OSError) + static void write_text(const std::string& p, const std::string& content); + static std::string read_bytes(const std::string& p); // open(p, "rb").read() + static void write_bytes(const std::string& p, const std::string& data); // open(p, "wb").write(data) + static void makedirs(const std::string& p); // exist_ok=True; throws + static bool remove(const std::string& p); // os.remove; throws + static void rmtree(const std::string& p); + static void chmod(const std::string& p, unsigned mode); + static void atomic_write(const std::string& path, const std::string& content); + static std::optional<std::string> which(const std::string& name); // shutil.which + static std::string env(const std::string& name, const std::string& dflt = ""); // os.environ.get(name) or dflt + static std::vector<std::string> walk_files(const std::string& root); // every file under root (os.walk), full paths +}; + +struct ProcResult { + bool failed{false}; // OSError or TimeoutExpired in subprocess.run; `error` holds short_error(exc) + std::string error; + int code{0}; + std::string out, err; // universal-newline text, like universal_newlines=True +}; + +class PyProc { + public: + // subprocess.run(argv, stdout=PIPE, stderr=PIPE, universal_newlines=True, timeout=timeout) + static ProcResult run(const std::vector<std::string>& argv, int timeout = 120); + // run_host: (returncode == 0, stdout + stderr); spawn failure/timeout -> (false, short_error text) + static HostResult run_host(const std::vector<std::string>& argv, int timeout = 120); +}; + +// Port of tai.py LOG.event(level, component, event, message, **ctx, meta=...). The integrator binds the +// real DebugLog; the default sink drops everything. +class LogSink { + public: + virtual ~LogSink() = default; + virtual void event(const std::string& level, const std::string& component, const std::string& event, + const std::string& message, const std::string& profile, const std::string& bot, + const std::string& chat, const Json& meta) = 0; + // LOG.context[key] = value / LOG.context.pop(key, None) + virtual void set_context(const std::string& key, const std::string& value) = 0; + virtual void clear_context(const std::string& key) = 0; + static LogSink& null_sink(); +}; + +} // namespace detz::misc diff --git a/taicpp/src/skills/miscsha256.cpp b/taicpp/src/skills/miscsha256.cpp new file mode 100644 index 0000000..fbd3817 --- /dev/null +++ b/taicpp/src/skills/miscsha256.cpp @@ -0,0 +1,64 @@ +#include "miscsha256.h" +#include <cstdint> +#include <cstring> +namespace detz::misc { +namespace { +const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; +inline uint32_t rotr(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } +void block(uint32_t h[8], const unsigned char* p) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast<uint32_t>(p[i * 4]) << 24) | (static_cast<uint32_t>(p[i * 4 + 1]) << 16) | + (static_cast<uint32_t>(p[i * 4 + 2]) << 8) | static_cast<uint32_t>(p[i * 4 + 3]); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + S1 + ch + K[i] + w[i]; + uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t mj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + mj; + hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh; +} +} // namespace + +std::string Sha256::hex(const std::string& data) { + uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + const unsigned char* p = reinterpret_cast<const unsigned char*>(data.data()); + std::size_t n = data.size(); + std::size_t full = n / 64; + for (std::size_t i = 0; i < full; ++i) block(h, p + i * 64); + unsigned char tail[128]; + std::size_t rem = n - full * 64; + std::memcpy(tail, p + full * 64, rem); + tail[rem] = 0x80; + std::size_t padded = rem + 1 + 8 <= 64 ? 64 : 128; + std::memset(tail + rem + 1, 0, padded - rem - 1); + uint64_t bits = static_cast<uint64_t>(n) * 8; + for (int i = 0; i < 8; ++i) tail[padded - 1 - static_cast<std::size_t>(i)] = static_cast<unsigned char>(bits >> (8 * i)); + block(h, tail); + if (padded == 128) block(h, tail + 64); + static const char* hexd = "0123456789abcdef"; + std::string out; + for (int i = 0; i < 8; ++i) { + for (int s = 28; s >= 0; s -= 4) out += hexd[(h[i] >> s) & 0xF]; + } + return out; +} +} // namespace detz::misc diff --git a/taicpp/src/skills/miscsha256.h b/taicpp/src/skills/miscsha256.h new file mode 100644 index 0000000..1cbc419 --- /dev/null +++ b/taicpp/src/skills/miscsha256.h @@ -0,0 +1,11 @@ +#pragma once +// hashlib.sha256(...).hexdigest() without any dependency. +#include <string> +namespace detz::misc { +class Sha256 { + public: + static std::string hex(const std::string& data); + // hexdigest()[:n] + static std::string hex_prefix(const std::string& data, std::size_t n) { return hex(data).substr(0, n); } +}; +} // namespace detz::misc diff --git a/taicpp/src/skills/pyunicode.cpp b/taicpp/src/skills/pyunicode.cpp new file mode 100644 index 0000000..d899939 --- /dev/null +++ b/taicpp/src/skills/pyunicode.cpp @@ -0,0 +1,29 @@ +#include "pyunicode.h" +#include <algorithm> +namespace detz::misc { +int PyUnicode::decimal_value(unsigned cp) { + const auto& zeros = digit_zeros(); + auto it = std::upper_bound(zeros.begin(), zeros.end(), cp); + if (it == zeros.begin()) return -1; + unsigned zero = *(it - 1); + return cp - zero <= 9 ? static_cast<int>(cp - zero) : -1; +} +bool PyUnicode::is_space(unsigned cp) { + const auto& sp = space_points(); + return std::binary_search(sp.begin(), sp.end(), cp); +} +unsigned PyUnicode::decode(const std::string& s, std::size_t i, std::size_t& n) { + unsigned char c = static_cast<unsigned char>(s[i]); + if (c < 0x80) { n = 1; return c; } + int extra = c >= 0xF0 ? 3 : c >= 0xE0 ? 2 : c >= 0xC0 ? 1 : 0; + if (extra == 0 || i + static_cast<std::size_t>(extra) >= s.size()) { n = 1; return 0xFFFD; } + unsigned cp = c & (0x3Fu >> extra); + for (int k = 1; k <= extra; ++k) { + unsigned char d = static_cast<unsigned char>(s[i + static_cast<std::size_t>(k)]); + if ((d & 0xC0) != 0x80) { n = 1; return 0xFFFD; } + cp = (cp << 6) | (d & 0x3Fu); + } + n = static_cast<std::size_t>(extra) + 1; + return cp; +} +} // namespace detz::misc diff --git a/taicpp/src/skills/pyunicode.h b/taicpp/src/skills/pyunicode.h new file mode 100644 index 0000000..0bf4f79 --- /dev/null +++ b/taicpp/src/skills/pyunicode.h @@ -0,0 +1,16 @@ +#pragma once +// Unicode facts Python's str methods and re module expose: \d (category Nd) with int() values, \s / str.isspace(). +#include <string> +#include <vector> +namespace detz::misc { +class PyUnicode { + public: + // Value 0..9 of a decimal digit code point, -1 otherwise (re \d, int()) + static int decimal_value(unsigned cp); + static bool is_space(unsigned cp); // str.isspace() / re \s + // Decodes the code point at s[i]; n receives its byte length (invalid bytes decode as U+FFFD, 1 byte) + static unsigned decode(const std::string& s, std::size_t i, std::size_t& n); + static const std::vector<unsigned>& digit_zeros(); // generated + static const std::vector<unsigned>& space_points(); // generated +}; +} // namespace detz::misc diff --git a/taicpp/src/skills/pyunicode_data.cpp b/taicpp/src/skills/pyunicode_data.cpp new file mode 100644 index 0000000..223fb6c --- /dev/null +++ b/taicpp/src/skills/pyunicode_data.cpp @@ -0,0 +1,12 @@ +// GENERATED by src/skills/gen_unicode.py (Python 3.14.4, Unicode 16.0.0). Do not edit. +#include "pyunicode.h" +namespace detz::misc { +const std::vector<unsigned>& PyUnicode::digit_zeros() { + static const std::vector<unsigned> v = {0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0, 0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0, 0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950, 0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6, 0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0}; + return v; +} +const std::vector<unsigned>& PyUnicode::space_points() { + static const std::vector<unsigned> v = {0x9, 0xA, 0xB, 0xC, 0xD, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x85, 0xA0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000}; + return v; +} +} // namespace detz::misc diff --git a/taicpp/src/skills/skillfile.cpp b/taicpp/src/skills/skillfile.cpp new file mode 100644 index 0000000..a387499 --- /dev/null +++ b/taicpp/src/skills/skillfile.cpp @@ -0,0 +1,267 @@ +#include "skillfile.h" +#include <algorithm> +#include "bundled_data.h" +#include "miscpy.h" +namespace detz::skills { +using misc::Json; +using misc::PyFs; +using misc::PyStr; + +void SkillSet::put(const Skill& skill) { + for (auto& s : items_) { + if (s.name == skill.name) { s = skill; return; } + } + items_.push_back(skill); +} +const Skill* SkillSet::get(const std::string& name) const { + for (const auto& s : items_) { + if (s.name == name) return &s; + } + return nullptr; +} +std::vector<std::string> SkillSet::sorted_names() const { + std::vector<std::string> names; + for (const auto& s : items_) names.push_back(s.name); + std::sort(names.begin(), names.end()); + return names; +} + +const std::vector<std::string>& SkillFiles::external_skill_dirnames() { + static const std::vector<std::string> v = {".claude/skills", ".agents/skills"}; + return v; +} + +bool SkillFiles::skill_name_valid(const std::string& name) { + // ^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$ (Python's $ also accepts one trailing newline) + std::string n = name; + if (!n.empty() && n.back() == '\n') n.pop_back(); + if (n.empty() || n.size() > 64) return false; + auto alnum = [](char c) { return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); }; + if (!alnum(n.front()) || !alnum(n.back())) return false; + for (char c : n) { + if (!alnum(c) && c != '-') return false; + } + return true; +} + +static std::string strip_quotes(const std::string& s) { + std::size_t b = 0, e = s.size(); + while (b < e && (s[b] == '\'' || s[b] == '"')) ++b; + while (e > b && (s[e - 1] == '\'' || s[e - 1] == '"')) --e; + return s.substr(b, e - b); +} +static bool indented(const std::string& line) { return !line.empty() && (line[0] == ' ' || line[0] == '\t'); } + +static void front_set(std::vector<std::pair<std::string, std::string>>& front, const std::string& key, const std::string& value) { + for (auto& kv : front) { + if (kv.first == key) { kv.second = value; return; } + } + front.emplace_back(key, value); +} +static std::string front_get(const std::vector<std::pair<std::string, std::string>>& front, const std::string& key) { + for (const auto& kv : front) { + if (kv.first == key) return kv.second; + } + return ""; +} + +std::vector<std::pair<std::string, std::string>> SkillFiles::parse_skill_front(const std::vector<std::string>& lines) { + std::vector<std::pair<std::string, std::string>> front; + std::size_t pos = 0; + while (pos < lines.size()) { + const std::string& line = lines[pos]; + std::size_t colon = line.find(':'); + if (colon != std::string::npos && !indented(line)) { + std::string key = line.substr(0, colon); + std::string value = PyStr::strip(line.substr(colon + 1)); + if (value == ">" || value == "|") { + std::vector<std::string> joined; + ++pos; + while (pos < lines.size() && indented(lines[pos])) { + joined.push_back(PyStr::strip(lines[pos])); + ++pos; + } + front_set(front, PyStr::strip(key), PyStr::join(joined, value == ">" ? " " : "\n")); + continue; + } + front_set(front, PyStr::strip(key), strip_quotes(value)); + } + ++pos; + } + return front; +} + +std::optional<Skill> SkillFiles::parse_skill_file(const std::string& path) { + std::string raw = PyFs::read_text(path); + if (raw.compare(0, 3, "---") != 0) return std::nullopt; + std::size_t end = raw.find("\n---", 3); + if (end == std::string::npos) return std::nullopt; + auto front = parse_skill_front(PyStr::splitlines(PyStr::strip(raw.substr(3, end - 3)))); + std::string name = front_get(front, "name"); + std::string description = front_get(front, "description"); + if (name.empty() || description.empty() || !skill_name_valid(name)) return std::nullopt; + std::string body = raw.substr(end + 4); + std::size_t lead = body.find_first_not_of('\n'); + body = lead == std::string::npos ? "" : body.substr(lead); + Skill skill; + skill.name = name; + skill.description = description; + skill.body = body; + return skill; +} + +std::string SkillFiles::skill_scope_dir(const std::string& scope, const std::string& home_dir, const std::string& project_dir, const std::string& name) { + std::string base = scope == "project" ? PyFs::join(PyFs::join(project_dir, ".tai"), "skills") : PyFs::join(home_dir, "skills"); + return name.empty() ? base : PyFs::join(base, name); +} + +std::string SkillFiles::default_skill_scope(const std::string& project_dir) { + std::vector<std::string> markers = {".git", ".tai", ".claude/skills", ".agents/skills"}; + for (const auto& marker : markers) { + if (PyFs::exists(PyFs::join(project_dir, marker))) return "project"; + } + return "home"; +} + +std::vector<std::string> SkillFiles::skill_extra_files(const std::string& root) { + std::vector<std::string> extras; + for (const char* sub : {"scripts", "references", "assets"}) { + std::string folder = PyFs::join(root, sub); + if (PyFs::isdir(folder)) { + for (const auto& entry : PyFs::listdir_sorted(folder)) extras.push_back(PyFs::join(PyFs::join(root, sub), entry)); + } + } + return extras; +} + +std::string SkillFiles::skill_install_meta_path(const std::string& scope_dir, const std::string& entry) { + return PyFs::join(PyFs::join(scope_dir, SKILL_INSTALL_META_DIRNAME), entry + ".json"); +} + +std::optional<Json> SkillFiles::load_skill_install_meta(const std::string& scope_dir, const std::string& entry) { + try { + return Json::loads(PyFs::read_text(skill_install_meta_path(scope_dir, entry))); + } catch (const misc::OsError&) { + return std::nullopt; + } catch (const misc::JsonError&) { + return std::nullopt; + } +} + +void SkillFiles::save_skill_install_meta(const std::string& scope_dir, const std::string& entry, const Json& meta) { + std::string path = skill_install_meta_path(scope_dir, entry); + PyFs::makedirs(PyFs::dirname(path)); + PyFs::write_text(path, meta.dumps()); +} + +bool SkillFiles::is_external_skill_root(const std::string& path) { + std::string parent = PyFs::dirname(PyFs::normpath(path)); + for (const auto& tail : external_skill_dirnames()) { + if (PyStr::ends_with(parent, tail)) return true; + } + return false; +} + +static std::string py_repr(const Json& v) { + switch (v.kind()) { + case Json::Kind::Null: return "None"; + case Json::Kind::Bool: return v.as_bool() ? "True" : "False"; + case Json::Kind::Int: return std::to_string(v.as_int()); + case Json::Kind::Float: return Json::float_repr(v.as_double()); + case Json::Kind::String: return PyStr::repr(v.as_string()); + case Json::Kind::Array: { + std::string out = "["; + for (std::size_t i = 0; i < v.items().size(); ++i) out += (i ? ", " : "") + py_repr(v.items()[i]); + return out + "]"; + } + case Json::Kind::Object: { + std::string out = "{"; + bool first = true; + for (const auto& m : v.members()) { + out += (first ? "" : ", ") + PyStr::repr(m.first) + ": " + py_repr(m.second); + first = false; + } + return out + "}"; + } + } + return ""; +} +std::string SkillFiles::py_str(const Json& value) { return value.is_string() ? value.as_string() : py_repr(value); } + +std::string SkillFiles::skill_provenance_note(const Skill& skill) { + // `if install:` is the Python truthiness of the sidecar value + if (skill.install && skill.install->is_object() && skill.install->size() > 0) { + const Json& in = *skill.install; + auto field = [&](const char* key, const char* dflt) { + const Json* v = in.get(key); + return v ? py_str(*v) : std::string(dflt); + }; + return "[installed " + field("mode", "copy") + ", " + field("scope", "?") + " scope, from " + field("source", "?") + "]"; + } + if (!skill.managed) return "[external, not managed by tai: " + skill.root + "]"; + return "[native]"; +} + +SkillSet SkillFiles::discover_skills(const std::string& home_dir, const std::string& project_dir) { + SkillSet found; + std::vector<std::string> tai_roots = {PyFs::join(home_dir, "skills"), PyFs::join(PyFs::join(project_dir, ".tai"), "skills")}; + std::vector<std::string> bases = {PyFs::join(PyFs::join(PyFs::home(), ".claude"), "skills")}; + for (const auto& dirname : external_skill_dirnames()) bases.push_back(PyFs::join(project_dir, dirname)); + // External roots scan first so tai's own dirs always win on a name collision. + for (const auto& r : tai_roots) bases.push_back(r); + for (const auto& base : bases) { + if (!PyFs::isdir(base)) continue; + bool managed = std::find(tai_roots.begin(), tai_roots.end(), base) != tai_roots.end(); + for (const auto& entry : PyFs::listdir_sorted(base)) { + std::string path = PyFs::join(PyFs::join(base, entry), "SKILL.md"); + if (!PyFs::isfile(path)) continue; + std::optional<Skill> skill; + try { + skill = parse_skill_file(path); + } catch (const misc::OsError&) { + continue; + } + if (!skill) continue; + skill->root = PyFs::join(base, entry); + skill->managed = managed; + skill->install = managed ? load_skill_install_meta(base, entry) : std::nullopt; + found.put(*skill); + } + } + return found; +} + +std::string SkillFiles::skill_catalog(const SkillSet& skills) { + std::vector<std::string> lines; + if (!skills.empty()) { + lines.insert(lines.end(), {"", "", "## Available skills"}); + for (const auto& name : skills.sorted_names()) lines.push_back("- " + name + ": " + PyStr::prefix(skills.get(name)->description, 300)); + } + std::vector<std::string> wanted; + for (const auto& kv : BundledData::blueprints()) { + if (!skills.get(kv.first)) wanted.push_back(kv.first); + } + if (!wanted.empty()) { + lines.insert(lines.end(), {"", "", "## Buildable skill blueprints (load_skill builds one on demand)"}); + for (const auto& name : wanted) lines.push_back("- " + name + ": " + BundledData::blueprints().at(name).hint); + } + return PyStr::join(lines, "\n"); +} + +std::string SkillFiles::create_skill_prompt(const std::string& name, const std::string& brief, const std::string& skill_dir) { + return "Create a new agent skill named '" + name + "' in " + skill_dir + ".\n" + "Brief: " + brief + "\n\n" + "Follow this process exactly:\n" + "1. Call sysinfo first so the skill matches this machine (os, installed tools, root, container, venv).\n" + "2. Research the subject deeply with web_search, web_fetch, and shell. Keep researching until you literally " + "have enough verified information to write complete instructions. Verify factual claims against at least two " + "independent sources. Never invent commands, paths, versions, or API details.\n" + "3. Write SKILL.md in the skill directory with this exact frontmatter:\n" + "---\nname: " + name + "\ndescription: <one sentence saying what the skill does and when to use it>\n---\n" + "followed by clear markdown instructions grounded in your research.\n" + "4. Add scripts/, references/, or assets/ files only when they carry real weight; keep every file focused.\n" + "5. Read back every file you wrote, fix mistakes, and confirm the frontmatter holds a name plus a description.\n" + "6. Reply with a short summary: what the skill covers, which sources verified it, and which files you wrote.\n" + "Do not stop early: an incomplete or unverified skill is a failure. Do not ask questions; decide and record assumptions in the skill body."; +} +} // namespace detz::skills diff --git a/taicpp/src/skills/skillfile.h b/taicpp/src/skills/skillfile.h new file mode 100644 index 0000000..fedc5ab --- /dev/null +++ b/taicpp/src/skills/skillfile.h @@ -0,0 +1,59 @@ +#pragma once +// Port of tai.py skill discovery: parse_skill_front, parse_skill_file, skill_scope_dir, +// default_skill_scope, skill_extra_files, install sidecars, discover_skills, skill_catalog, +// create_skill_prompt. Spec: docs/skills.md. +#include <map> +#include <optional> +#include <string> +#include <vector> +#include "miscjson.h" +namespace detz::skills { + +struct Skill { + std::string name, description, body; + std::string root; // set by discover_skills + bool managed{true}; // set by discover_skills + std::optional<misc::Json> install; // install sidecar, managed roots only +}; + +// dict[str, skill] with Python insertion order (assigning an existing key keeps its position). +class SkillSet { + public: + void put(const Skill& skill); + const Skill* get(const std::string& name) const; + std::vector<std::string> sorted_names() const; + const std::vector<Skill>& items() const { return items_; } + std::size_t size() const { return items_.size(); } + bool empty() const { return items_.empty(); } + private: + std::vector<Skill> items_; +}; + +class SkillFiles { + public: + static constexpr const char* SKILL_INSTALL_META_DIRNAME = ".installs"; + // EXTERNAL_SKILL_DIRNAMES = (".claude/skills", ".agents/skills") + static const std::vector<std::string>& external_skill_dirnames(); + + static bool skill_name_valid(const std::string& name); // SKILL_NAME_RE.match + // dict of the frontmatter in insertion order (assigning an existing key keeps its position) + static std::vector<std::pair<std::string, std::string>> parse_skill_front(const std::vector<std::string>& lines); + // None when not a skill; throws misc::OsError like open() does. + static std::optional<Skill> parse_skill_file(const std::string& path); + + static std::string skill_scope_dir(const std::string& scope, const std::string& home_dir, const std::string& project_dir, const std::string& name = ""); + static std::string default_skill_scope(const std::string& project_dir); + static std::vector<std::string> skill_extra_files(const std::string& root); + static std::string skill_install_meta_path(const std::string& scope_dir, const std::string& entry); + static std::optional<misc::Json> load_skill_install_meta(const std::string& scope_dir, const std::string& entry); + static void save_skill_install_meta(const std::string& scope_dir, const std::string& entry, const misc::Json& meta); + static bool is_external_skill_root(const std::string& path); + static std::string skill_provenance_note(const Skill& skill); + static SkillSet discover_skills(const std::string& home_dir, const std::string& project_dir); + static std::string skill_catalog(const SkillSet& skills); + static std::string create_skill_prompt(const std::string& name, const std::string& brief, const std::string& skill_dir); + // str(value) for a JSON scalar, as %s renders it in Python + static std::string py_str(const misc::Json& value); +}; + +} // namespace detz::skills diff --git a/taicpp/src/skills/skilltool.cpp b/taicpp/src/skills/skilltool.cpp new file mode 100644 index 0000000..5d721fa --- /dev/null +++ b/taicpp/src/skills/skilltool.cpp @@ -0,0 +1,260 @@ +#include "skilltool.h" +#include <unistd.h> +#include <algorithm> +#include <cctype> +#include <cstdlib> +#include <filesystem> +#include "bundled_data.h" +#include "miscdatetime.h" +#include "miscpy.h" +#include "ziparchive.h" +namespace fs = std::filesystem; +namespace detz::skills { +using misc::Json; +using misc::PyFs; +using misc::PyStr; + +std::string SkillHost::cwd() const { + char buf[4096]; + return getcwd(buf, sizeof buf) ? buf : "/"; +} +std::string SkillHost::truncate(const std::string& text, std::size_t head, std::size_t tail) const { + std::size_t n = PyStr::len(text); + if (n <= head + tail + 50) return text; + return PyStr::prefix(text, head) + "\n...[" + std::to_string(n - head - tail) + " chars truncated]...\n" + PyStr::suffix(text, tail); +} + +// Python truthiness of a JSON value +static bool truthy(const Json* v) { + if (!v) return false; + switch (v->kind()) { + case Json::Kind::Null: return false; + case Json::Kind::Bool: return v->as_bool(); + case Json::Kind::Int: return v->as_int() != 0; + case Json::Kind::Float: return v->as_double() != 0; + case Json::Kind::String: return !v->as_string().empty(); + default: return v->size() > 0; + } +} +std::string SkillTool::arg_text(const Json& args, const std::string& key) { + const Json* v = args.get(key); + return truthy(v) ? SkillFiles::py_str(*v) : ""; +} +static std::string lower_ascii(std::string s) { + for (auto& c : s) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c))); + return s; +} +// int(args.get(key) or dflt) with ValueError/TypeError falling back to dflt +static int arg_int(const Json& args, const std::string& key, int dflt) { + const Json* v = args.get(key); + if (!truthy(v)) return dflt; + switch (v->kind()) { + case Json::Kind::Bool: return 1; + case Json::Kind::Int: return static_cast<int>(v->as_int()); + case Json::Kind::Float: return static_cast<int>(v->as_double()); + case Json::Kind::String: { + std::string s = PyStr::strip(v->as_string()); + std::size_t i = 0; + bool neg = false; + if (i < s.size() && (s[i] == '+' || s[i] == '-')) { neg = s[i] == '-'; ++i; } + if (i >= s.size()) return dflt; + long long val = 0; + for (; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') return dflt; + val = val * 10 + (s[i] - '0'); + if (val > 1000000000000LL) break; + } + return static_cast<int>(neg ? -val : val); + } + default: return dflt; + } +} + +std::string SkillTool::run_load_skill(const Json& args) { + std::string name = PyStr::strip(arg_text(args, "name")); + const Skill* skill = host_.skills().get(name); + if (!skill) { + auto bp = BundledData::blueprints().find(name); + if (bp == BundledData::blueprints().end()) { + std::vector<std::string> known = host_.skills().sorted_names(); + return "error: unknown skill, known: " + (known.empty() ? std::string("none") : PyStr::join(known, ", ")); + } + Json built = Json::object(); + built.set("name", name); + built.set("brief", bp->second.brief); + built.set("scope", bp->second.scope); + return "building skill '" + name + "' from blueprint, this runs deep research and takes a while:\n" + run_create_skill(built); + } + std::vector<std::string> parts = {PyStr::strip(skill->body), SkillFiles::skill_provenance_note(*skill)}; + auto extras = SkillFiles::skill_extra_files(skill->root); + if (!extras.empty()) parts.push_back("bundled files:\n" + PyStr::join(extras, "\n")); + return PyStr::join(parts, "\n\n"); +} + +std::string SkillTool::run_create_skill(const Json& args) { + std::string name = PyStr::strip(arg_text(args, "name")); + std::string brief = PyStr::strip(arg_text(args, "brief")); + if (name.empty() || !SkillFiles::skill_name_valid(name)) return "error: invalid skill name, use lowercase letters, digits, and hyphens"; + if (brief.empty()) return "error: empty brief"; + if (host_.depth() >= host_.fork_max_depth()) return "error: create_skill depth limit reached"; + std::string raw_scope = arg_text(args, "scope"); + std::string scope = raw_scope.empty() ? "project" : lower_ascii(PyStr::strip(raw_scope)); + if (scope != "project" && scope != "home") return "error: scope must be project or home"; + int timeout = std::max(60, std::min(3600, arg_int(args, "timeout", 1200))); + std::string cwd = host_.cwd(); + std::string skill_dir = SkillFiles::skill_scope_dir(scope, host_.home(), cwd, name); + std::string skill_file = PyFs::join(skill_dir, "SKILL.md"); + bool existed = PyFs::isfile(skill_file); + std::string result = host_.run_worker(SkillFiles::create_skill_prompt(name, brief, skill_dir), timeout, CREATE_SKILL_STEPS); + if (PyFs::isfile(skill_file)) { + host_.refresh_skills(); + return "skill '" + name + "' " + (existed ? "replaced" : "created") + " at " + skill_file + "\n" + host_.truncate(result, 2000, 500); + } + std::string status = result.find("[time limit reached]") != std::string::npos ? "timeout" : "done"; + return "skill '" + name + "' was not created (" + status + "), worker output:\n" + host_.truncate(result, 3000, 1000); +} + +std::optional<std::string> SkillTool::find_skill_root(const std::string& base) { + if (PyFs::isfile(PyFs::join(base, "SKILL.md"))) return base; + std::vector<std::string> entries; + try { + entries = PyFs::listdir_sorted(base); + } catch (const misc::OsError&) { + return std::nullopt; + } + std::vector<std::string> candidates; + for (const auto& e : entries) { + if (PyFs::isfile(PyFs::join(PyFs::join(base, e), "SKILL.md"))) candidates.push_back(e); + } + if (candidates.size() == 1) return PyFs::join(base, candidates[0]); + return std::nullopt; +} + +static bool ends_with(const std::string& s, const std::string& t) { return PyStr::ends_with(s, t); } + +ResolvedSource SkillTool::resolve_skill_source(const std::string& source, const std::string& workdir) { + ResolvedSource r; + if (ends_with(source, ".git") || PyStr::starts_with(source, "git@") || PyStr::starts_with(source, "git+")) { + std::string git_url = PyStr::starts_with(source, "git+") ? source.substr(4) : source; + if (!PyFs::which("git")) { r.error = "error: git is not installed, supply a local path or .zip instead"; return r; } + std::string clone_dir = PyFs::join(workdir, "clone"); + misc::HostResult done = misc::PyProc::run_host({"git", "clone", "--depth", "1", git_url, clone_dir}, 120); + if (!done.ok) { + // Python reports stderr only for a non-zero exit and short_error(exc) for spawn failures/timeouts + r.error = "error: git clone failed: " + PyStr::prefix(PyStr::strip(done.out), 300); + return r; + } + auto found = find_skill_root(clone_dir); + if (!found) { r.error = "error: no SKILL.md found in " + git_url; return r; } + r.ok = true; r.path = *found; r.ephemeral = true; + return r; + } + std::string path = PyFs::expanduser(source); + if (!PyFs::exists(path)) { r.error = "error: no such file or directory: " + source; return r; } + if (PyFs::isfile(path) && ends_with(path, ".zip")) { + std::string extract_dir = PyFs::join(workdir, "zip"); + try { + ZipArchive::extractall(path, extract_dir); + } catch (const BadZipFile& e) { + r.error = "error: bad zip file: " + PyStr::short_error(e.what()); + return r; + } catch (const misc::OsError& e) { + r.error = "error: bad zip file: " + PyStr::short_error(e.what()); + return r; + } + auto found = find_skill_root(extract_dir); + if (!found) { r.error = "error: no SKILL.md found in " + source; return r; } + r.ok = true; r.path = *found; r.ephemeral = true; + return r; + } + if (PyFs::isfile(path) && ends_with(path, ".md")) { r.ok = true; r.path = path; return r; } + if (PyFs::isdir(path)) { + auto found = find_skill_root(path); + if (!found) { r.error = "error: no SKILL.md found under " + source; return r; } + r.ok = true; r.path = *found; + return r; + } + r.error = "error: source must be a directory, a SKILL.md file, a .zip, or a git URL"; + return r; +} + +std::string SkillTool::run_install_skill(const Json& args) { + std::string source = PyStr::strip(arg_text(args, "source")); + if (source.empty()) return "error: empty source"; + std::string cwd = host_.cwd(); + std::string scope = lower_ascii(PyStr::strip(arg_text(args, "scope"))); + if (scope.empty()) scope = SkillFiles::default_skill_scope(cwd); + if (scope != "project" && scope != "home") return "error: scope must be project or home"; + std::string mode = lower_ascii(PyStr::strip(arg_text(args, "mode"))); + if (!mode.empty() && mode != "copy" && mode != "symlink") return "error: mode must be copy or symlink"; + bool force = truthy(args.get("force")); + std::string tmpl = (std::getenv("TMPDIR") && *std::getenv("TMPDIR") ? std::string(std::getenv("TMPDIR")) : std::string("/tmp")) + "/tai-skill-XXXXXX"; + std::vector<char> buf(tmpl.begin(), tmpl.end()); + buf.push_back('\0'); + if (!mkdtemp(buf.data())) return "error: " + PyStr::short_error(misc::OsError(errno, tmpl).what()); + std::string workdir = buf.data(); + struct Cleanup { std::string d; ~Cleanup() { std::error_code ec; fs::remove_all(d, ec); } } cleanup{workdir}; + + ResolvedSource resolved = resolve_skill_source(source, workdir); + if (!resolved.ok) return resolved.error; + const std::string& real_source = resolved.path; + bool is_dir = PyFs::isdir(real_source); + std::string skill_file = is_dir ? PyFs::join(real_source, "SKILL.md") : real_source; + std::optional<Skill> skill; + try { + skill = SkillFiles::parse_skill_file(skill_file); + } catch (const misc::OsError& e) { + return "error: " + PyStr::short_error(e.what()); + } + if (!skill) return "error: " + source + " has no valid SKILL.md (needs name, description, and a name of lowercase letters, digits, hyphens)"; + std::string name = PyStr::strip(arg_text(args, "name")); + if (name.empty()) name = skill->name; + if (!SkillFiles::skill_name_valid(name)) return "error: invalid skill name, use lowercase letters, digits, and hyphens"; + if (mode.empty()) { + mode = (is_dir && !resolved.ephemeral && SkillFiles::is_external_skill_root(real_source)) ? "symlink" : "copy"; + } else if (mode == "symlink" && (resolved.ephemeral || !is_dir)) { + return "error: symlink mode needs a persistent source directory, not a git/zip source or a bare SKILL.md file"; + } + std::string scope_dir = SkillFiles::skill_scope_dir(scope, host_.home(), cwd); + std::string dest = PyFs::join(scope_dir, name); + bool existed = PyFs::islink(dest) || PyFs::exists(dest); + if (existed && !force && !host_.ask_approval("overwrite existing skill '" + name + "' at " + dest)) return "denied by user"; + try { + if (existed) { + if (PyFs::islink(dest) || PyFs::isfile(dest)) PyFs::remove(dest); + else PyFs::rmtree(dest); + } + PyFs::makedirs(scope_dir); + if (mode == "symlink") { + std::string target = PyFs::abspath(real_source); + if (::symlink(target.c_str(), dest.c_str()) != 0) throw misc::OsError(errno, dest); + } else if (is_dir) { + std::error_code ec; + fs::copy(real_source, dest, fs::copy_options::recursive, ec); + if (ec) throw misc::OsError(ec.value(), dest); + } else { + PyFs::makedirs(dest); + std::error_code ec; + fs::copy_file(real_source, PyFs::join(dest, "SKILL.md"), fs::copy_options::overwrite_existing, ec); + if (ec) throw misc::OsError(ec.value(), dest); + } + Json meta = Json::object(); + meta.set("source", source); + meta.set("mode", mode); + meta.set("scope", scope); + meta.set("installed_at", misc::DateTime::now_utc().isoformat()); + SkillFiles::save_skill_install_meta(scope_dir, name, meta); + } catch (const misc::OsError& e) { + // Python lets these propagate to the tool dispatcher; surface them as a tool error line. + return "error: " + PyStr::short_error(e.what()); + } + if (host_.has_store()) { + host_.audit_skill_install(dest, "installed skill '" + name + "' from " + source + " (" + mode + ", " + scope + " scope)"); + } + host_.refresh_skills(); + auto extras = SkillFiles::skill_extra_files(dest); + std::string tail = extras.empty() ? "" : "\nbundled files: " + PyStr::join(extras, ", "); + std::string replaced = existed ? " (replacing the previous install)" : ""; + return "skill '" + name + "' installed (" + mode + ", " + scope + " scope) at " + dest + " from " + source + replaced + "\n" + skill->description + tail; +} +} // namespace detz::skills diff --git a/taicpp/src/skills/skilltool.h b/taicpp/src/skills/skilltool.h new file mode 100644 index 0000000..e4a0aad --- /dev/null +++ b/taicpp/src/skills/skilltool.h @@ -0,0 +1,53 @@ +#pragma once +// Port of the Tools skill backends: run_load_skill, run_create_skill, find_skill_root, +// resolve_skill_source, run_install_skill. The app (agent core) is reached through SkillHost. +#include <optional> +#include <string> +#include <utility> +#include "miscjson.h" +#include "skillfile.h" +namespace detz::skills { + +// What the Python Tools object takes from its app: approval, config.home, cwd, audit, skills refresh and +// the worker that runs the create-skill research prompt. +class SkillHost { + public: + virtual ~SkillHost() = default; + virtual bool ask_approval(const std::string& what) = 0; + virtual std::string home() const = 0; // app.config.home + virtual std::string cwd() const; // os.getcwd() + virtual int depth() const { return 0; } // app.depth + virtual int fork_max_depth() const { return 2; } // FORK_MAX_DEPTH + virtual const SkillSet& skills() const = 0; // app.skills + virtual void refresh_skills() = 0; // app.skills = discover_skills(...); app.apply_system() + virtual void audit_skill_install(const std::string& path, const std::string& message) = 0; // may be a no-op without a store + virtual bool has_store() const { return false; } + // default_runner / runner_override: returns the worker's final text + virtual std::string run_worker(const std::string& prompt, int timeout, int steps) = 0; + virtual std::string truncate(const std::string& text, std::size_t head, std::size_t tail) const; +}; + +struct ResolvedSource { + bool ok{false}; + std::string path; + bool ephemeral{false}; + std::string error; // "error: ..." text when !ok +}; + +class SkillTool { + public: + static constexpr int CREATE_SKILL_STEPS = 40; + explicit SkillTool(SkillHost& host) : host_(host) {} + std::string run_load_skill(const misc::Json& args); + std::string run_create_skill(const misc::Json& args); + std::string run_install_skill(const misc::Json& args); + static std::optional<std::string> find_skill_root(const std::string& base); + static ResolvedSource resolve_skill_source(const std::string& source, const std::string& workdir); + // str(args.get(key) or "") + static std::string arg_text(const misc::Json& args, const std::string& key); + + private: + SkillHost& host_; +}; + +} // namespace detz::skills diff --git a/taicpp/src/skills/ziparchive.cpp b/taicpp/src/skills/ziparchive.cpp new file mode 100644 index 0000000..2166e53 --- /dev/null +++ b/taicpp/src/skills/ziparchive.cpp @@ -0,0 +1,114 @@ +#include "ziparchive.h" +#include <zlib.h> +#include <cerrno> +#include <cstdint> +#include <cstdio> +#include <cstring> +#include <vector> +#include "miscpy.h" +namespace detz::skills { +using misc::PyFs; + +namespace { +uint32_t u16(const std::string& d, std::size_t p) { + return static_cast<unsigned char>(d[p]) | (static_cast<uint32_t>(static_cast<unsigned char>(d[p + 1])) << 8); +} +uint32_t u32(const std::string& d, std::size_t p) { return u16(d, p) | (u16(d, p + 2) << 16); } + +struct Member { std::string name; uint32_t method, csize, usize, offset; }; + +// Mirrors ZipFile._extract_member path sanitising: drop empty, "." and ".." components. +std::string sanitize(const std::string& name, bool& is_dir) { + is_dir = !name.empty() && name.back() == '/'; + std::vector<std::string> parts; + std::string cur; + for (char c : name + "/") { + if (c == '/') { + if (!cur.empty() && cur != "." && cur != "..") parts.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + return misc::PyStr::join(parts, "/"); +} +} // namespace + +void ZipArchive::extractall(const std::string& zip_path, const std::string& dest) { + std::string data = [&] { + // binary read: no newline translation + FILE* f = std::fopen(zip_path.c_str(), "rb"); + if (!f) throw misc::OsError(errno, zip_path); + std::string out; + char buf[65536]; + std::size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) out.append(buf, n); + std::fclose(f); + return out; + }(); + if (data.size() < 22) throw BadZipFile("File is not a zip file"); + std::size_t eocd = std::string::npos; + std::size_t lowest = data.size() > 22 + 65535 ? data.size() - 22 - 65535 : 0; + for (std::size_t p = data.size() - 22 + 1; p-- > lowest;) { + if (u32(data, p) == 0x06054b50) { eocd = p; break; } + } + if (eocd == std::string::npos) throw BadZipFile("File is not a zip file"); + uint32_t count = u16(data, eocd + 10); + std::size_t cd = u32(data, eocd + 16); + std::vector<Member> members; + for (uint32_t i = 0; i < count; ++i) { + if (cd + 46 > data.size() || u32(data, cd) != 0x02014b50) throw BadZipFile("Bad magic number for central directory"); + Member m; + m.method = u16(data, cd + 10); + m.csize = u32(data, cd + 20); + m.usize = u32(data, cd + 24); + uint32_t nlen = u16(data, cd + 28), xlen = u16(data, cd + 30), clen = u16(data, cd + 32); + m.offset = u32(data, cd + 42); + if (cd + 46 + nlen > data.size()) throw BadZipFile("Truncated central directory"); + m.name = data.substr(cd + 46, nlen); + members.push_back(m); + cd += 46 + nlen + xlen + clen; + } + PyFs::makedirs(dest); + for (const auto& m : members) { + bool is_dir = false; + std::string rel = sanitize(m.name, is_dir); + if (rel.empty()) continue; + std::string target = PyFs::join(dest, rel); + if (is_dir) { + PyFs::makedirs(target); + continue; + } + if (m.offset + 30 > data.size() || u32(data, m.offset) != 0x04034b50) throw BadZipFile("Bad magic number for file header"); + std::size_t start = m.offset + 30 + u16(data, m.offset + 26) + u16(data, m.offset + 28); + if (start + m.csize > data.size()) throw BadZipFile("Truncated file data"); + std::string body; + if (m.method == 0) { + body = data.substr(start, m.csize); + } else if (m.method == 8) { + z_stream zs; + std::memset(&zs, 0, sizeof zs); + if (inflateInit2(&zs, -15) != Z_OK) throw BadZipFile("zlib init failed"); + zs.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data() + start)); + zs.avail_in = m.csize; + char buf[65536]; + int rc; + do { + zs.next_out = reinterpret_cast<Bytef*>(buf); + zs.avail_out = sizeof buf; + rc = inflate(&zs, Z_NO_FLUSH); + if (rc != Z_OK && rc != Z_STREAM_END) { inflateEnd(&zs); throw BadZipFile("Error -3 while decompressing data"); } + body.append(buf, sizeof buf - zs.avail_out); + } while (rc != Z_STREAM_END); + inflateEnd(&zs); + } else { + throw BadZipFile("That compression method is not supported"); + } + PyFs::makedirs(PyFs::dirname(target)); + FILE* f = std::fopen(target.c_str(), "wb"); + if (!f) throw misc::OsError(errno, target); + std::fwrite(body.data(), 1, body.size(), f); + std::fclose(f); + } +} +} // namespace detz::skills diff --git a/taicpp/src/skills/ziparchive.h b/taicpp/src/skills/ziparchive.h new file mode 100644 index 0000000..29a4f88 --- /dev/null +++ b/taicpp/src/skills/ziparchive.h @@ -0,0 +1,12 @@ +#pragma once +// zipfile.ZipFile(path).extractall(dest) for stored and deflated members (zlib only). +#include <stdexcept> +#include <string> +namespace detz::skills { +class BadZipFile : public std::runtime_error { public: using std::runtime_error::runtime_error; }; +class ZipArchive { + public: + // Throws BadZipFile ("File is not a zip file") or misc::OsError, the two exceptions run_install_skill catches. + static void extractall(const std::string& zip_path, const std::string& dest); +}; +} // namespace detz::skills diff --git a/taicpp/src/telegram/telegram.cpp b/taicpp/src/telegram/telegram.cpp new file mode 100644 index 0000000..96a7e92 --- /dev/null +++ b/taicpp/src/telegram/telegram.cpp @@ -0,0 +1,224 @@ +#include "telegram.h" +#include <chrono> +#include <iostream> +#include <thread> +#include "skillfile.h" +namespace detz::telegram { +using misc::Json; +using misc::PyFs; +using misc::PyStr; + +Telegram::Telegram(std::string token, TelegramTransport* transport, std::string api, std::string file_api) + : token_(std::move(token)), transport_(transport), api_(std::move(api)), file_api_(std::move(file_api)) { + if (!transport_) { + own_ = std::make_unique<CurlTransport>(); + transport_ = own_.get(); + } +} + +Json Telegram::call(const std::string& method, const Json& params, int timeout) { + std::string body = params.dumps(); + std::string reply = transport_->post_json(api_ + token_ + "/" + method, body, timeout); + return Json::loads(reply); +} + +TelegramBot::TelegramBot(TelegramAgent& agent, TelegramTransport* transport, BoxPort* box, misc::LogSink& log, std::ostream* out, std::ostream* err) + : agent_(agent), transport_(transport), box_(box), log_(log), out_(out ? out : &std::cout), err_(err ? err : &std::cerr) { + if (!transport_) { + own_ = std::make_unique<CurlTransport>(); + transport_ = own_.get(); + } +} + +std::string TelegramBot::telegram_env_path(const std::string& home) { return PyFs::join(home, "telegram.env"); } + +std::string TelegramBot::load_telegram_token(const std::string& home) { + std::string token = PyFs::env("TELEGRAM_BOT_TOKEN"); + if (!token.empty()) return token; + std::string path = telegram_env_path(home); + if (!PyFs::exists(path)) return ""; + for (const auto& line : PyStr::splitlines(PyFs::read_text(path))) { + if (PyStr::starts_with(line, "TELEGRAM_BOT_TOKEN=")) { + std::string value = PyStr::strip(line.substr(line.find('=') + 1)); + std::size_t b = value.find_first_not_of("'\""); + std::size_t e = value.find_last_not_of("'\""); + return b == std::string::npos ? "" : value.substr(b, e - b + 1); + } + } + return ""; +} + +void TelegramBot::telegram_send(Telegram& bot, const Json& chat_id, const std::string& text) { + std::size_t total = PyStr::len(text); + std::string rest = text; + for (std::size_t pos = 0; pos < total; pos += 4000) { + std::string piece = PyStr::prefix(rest, 4000); + rest = rest.substr(piece.size()); + Json params = Json::object(); + params.set("chat_id", chat_id); + params.set("text", piece.empty() ? std::string("(empty reply)") : piece); + bot.call("sendMessage", params); + } +} + +static bool truthy(const Json* v) { + if (!v) return false; + switch (v->kind()) { + case Json::Kind::Null: return false; + case Json::Kind::Bool: return v->as_bool(); + case Json::Kind::Int: return v->as_int() != 0; + case Json::Kind::Float: return v->as_double() != 0; + case Json::Kind::String: return !v->as_string().empty(); + default: return v->size() > 0; + } +} +// dict.get(key) or {} on a possibly non-dict value +static const Json* member(const Json* obj, const char* key) { return obj && obj->is_object() ? obj->get(key) : nullptr; } + +std::string TelegramBot::telegram_transcribe(Telegram& bot, const std::string& file_id) { + try { + Json params = Json::object(); + params.set("file_id", file_id); + Json info = bot.call("getFile", params); + const Json* result = member(&info, "result"); + const Json* fp = truthy(result) ? member(result, "file_path") : nullptr; + std::string remote = fp && fp->is_string() ? fp->as_string() : ""; + if (remote.empty()) return "error: telegram returned no file"; + std::string audio = transport_->get(bot.file_api() + bot.token() + "/" + remote, 60, 20000000); + if (!box_) throw BackendError("no container engine bound"); + std::string engine = box_->ensure_box(); + return box_->box_transcribe(engine, audio); + } catch (const TelegramNetError& exc) { + return "error: transcription failed: " + PyStr::short_error(exc.what()); + } catch (const misc::JsonError& exc) { + return "error: transcription failed: " + PyStr::short_error(exc.what()); + } catch (const BackendError& exc) { + return "error: transcription failed: " + PyStr::short_error(exc.what()); + } +} + +std::pair<std::string, int> TelegramBot::handle_inner(Telegram& bot, const Json& update) { + const Json* message = member(&update, "message"); + const Json* chat = truthy(message) ? member(message, "chat") : nullptr; + const Json* chat_id = truthy(chat) ? member(chat, "id") : nullptr; + if (!truthy(chat_id)) return {"ignored", 0}; + const Json* t = member(message, "text"); + std::string text = truthy(t) ? skills::SkillFiles::py_str(*t) : ""; + if (text == "/start") { + telegram_send(bot, *chat_id, "tai online. Send any message, /new for a fresh start."); + return {"greeted", 0}; + } + if (text == "/new") { + agent_.reset_history(); + telegram_send(bot, *chat_id, "fresh start."); + return {"reset", 0}; + } + const Json* voice = member(message, "voice"); + if (!truthy(voice)) voice = member(message, "audio"); + if (truthy(voice)) { + Json typing = Json::object(); + typing.set("chat_id", *chat_id); + typing.set("action", "typing"); + bot.call("sendChatAction", typing); + const Json* fid = member(voice, "file_id"); + text = telegram_transcribe(bot, fid && fid->is_string() ? fid->as_string() : ""); + if (PyStr::starts_with(text, "error")) { + telegram_send(bot, *chat_id, text); + return {"transcribe_error", static_cast<int>(PyStr::len(text))}; + } + } + if (PyStr::strip(text).empty()) return {"ignored", 0}; + Json typing = Json::object(); + typing.set("chat_id", *chat_id); + typing.set("action", "typing"); + bot.call("sendChatAction", typing); + std::string reply = agent_.run_turn(text); + telegram_send(bot, *chat_id, reply); + return {"replied", static_cast<int>(PyStr::len(reply))}; +} + +void TelegramBot::handle_telegram_update(Telegram& bot, const Json& update) { + const Json* message = member(&update, "message"); + const Json* chat = truthy(message) ? member(message, "chat") : nullptr; + const Json* chat_id = truthy(chat) ? member(chat, "id") : nullptr; + if (!truthy(chat_id)) return; + const Json* t = member(message, "text"); + std::string text = truthy(t) ? skills::SkillFiles::py_str(*t) : ""; + bool voice = truthy(member(message, "voice")) || truthy(member(message, "audio")); + std::string chat_str = skills::SkillFiles::py_str(*chat_id); + log_.set_context("chat", chat_str); + std::pair<std::string, int> done; + try { + done = handle_inner(bot, update); + } catch (const std::exception& exc) { + Json meta = Json::object(); + meta.set("chat", chat_str); + meta.set("text_chars", static_cast<int64_t>(PyStr::len(text))); + meta.set("reply_chars", 0); + meta.set("voice", voice); + meta.set("outcome", "error"); + log_.event("INFO", "telegram", "update", "telegram update failed: " + PyStr::short_error(exc.what()), agent_.profile(), agent_.bot(), chat_str, meta); + log_.clear_context("chat"); // the Python `finally` runs after the handler logged and before the re-raise surfaces + throw; + } + log_.clear_context("chat"); + Json meta = Json::object(); + meta.set("chat", chat_str); + meta.set("text_chars", static_cast<int64_t>(PyStr::len(text))); + meta.set("reply_chars", done.second); + meta.set("voice", voice); + meta.set("outcome", done.first); + log_.event("INFO", "telegram", "update", "telegram update " + done.first, agent_.profile(), agent_.bot(), chat_str, meta); +} + +int TelegramBot::run_telegram_bot(std::function<bool()> stop, std::function<void(int)> sleeper) { + std::string token = load_telegram_token(agent_.home()); + if (token.empty()) { + *err_ << "missing TELEGRAM_BOT_TOKEN" << std::endl; + return 2; + } + Telegram bot(token, transport_, api_, file_api_); + Json me; + try { + me = bot.call("getMe", Json::object()); + } catch (const TelegramNetError& exc) { + *err_ << "telegram auth failed: " << PyStr::short_error(exc.what()) << std::endl; + return 2; + } catch (const misc::JsonError& exc) { + *err_ << "telegram auth failed: " << PyStr::short_error(exc.what()) << std::endl; + return 2; + } + const Json* result = member(&me, "result"); + const Json* uname = truthy(result) ? member(result, "username") : nullptr; + *out_ << "telegram bot online as @" << (truthy(uname) ? skills::SkillFiles::py_str(*uname) : "?") << std::endl; + agent_.start_scheduler(); + int64_t offset = 0; + while (!(stop && stop())) { + Json data; + try { + Json params = Json::object(); + params.set("offset", offset); + params.set("timeout", 50); + data = bot.call("getUpdates", params); + } catch (const std::exception& exc) { + if (!dynamic_cast<const TelegramNetError*>(&exc) && !dynamic_cast<const misc::JsonError*>(&exc)) throw; + *out_ << "poll error: " << PyStr::short_error(exc.what()) << ", retrying" << std::endl; + if (sleeper) sleeper(5); + else std::this_thread::sleep_for(std::chrono::seconds(5)); + continue; + } + const Json* updates = member(&data, "result"); + if (!truthy(updates) || !updates->is_array()) continue; + for (const auto& update : updates->items()) { + const Json* uid = member(&update, "update_id"); + offset = (uid && uid->is_int() ? uid->as_int() : offset) + 1; + try { + handle_telegram_update(bot, update); + } catch (const std::exception& exc) { + *out_ << "update error: " << PyStr::short_error(exc.what()) << std::endl; + } + } + } + return 0; +} +} // namespace detz::telegram diff --git a/taicpp/src/telegram/telegram.h b/taicpp/src/telegram/telegram.h new file mode 100644 index 0000000..48d4a68 --- /dev/null +++ b/taicpp/src/telegram/telegram.h @@ -0,0 +1,86 @@ +#pragma once +// Port of tai.py Telegram, telegram_env_path, load_telegram_token, telegram_send, telegram_transcribe, +// handle_telegram_update and run_telegram_bot. Spec: docs/telegram.md. +#include <functional> +#include <memory> +#include <ostream> +#include <stdexcept> +#include <string> +#include <utility> +#include "miscjson.h" +#include "miscpy.h" +#include "telegramtransport.h" +namespace detz::telegram { + +constexpr const char* TELEGRAM_API = "https://api.telegram.org/bot"; +constexpr const char* TELEGRAM_FILE_API = "https://api.telegram.org/file/bot"; + +// class Telegram: one bot token, JSON POST per method. +class Telegram { + public: + Telegram(std::string token, TelegramTransport* transport = nullptr, std::string api = TELEGRAM_API, std::string file_api = TELEGRAM_FILE_API); + const std::string& token() const { return token_; } + const std::string& file_api() const { return file_api_; } + // Throws TelegramNetError (OSError) or misc::JsonError (ValueError), exactly the pair callers catch. + misc::Json call(const std::string& method, const misc::Json& params, int timeout = 70); + TelegramTransport& transport() { return *transport_; } + + private: + std::string token_; + TelegramTransport* transport_; + std::unique_ptr<CurlTransport> own_; + std::string api_, file_api_; +}; + +using BackendError = misc::BackendError; + +// The sandbox box that transcribes voice (ensure_box, box_transcribe). Owned by the shell module. +class BoxPort { + public: + virtual ~BoxPort() = default; + virtual std::string ensure_box() = 0; // returns the engine, throws BackendError + virtual std::string box_transcribe(const std::string& engine, const std::string& audio) = 0; // throws BackendError +}; + +// What run_turn/reset_history/start_scheduler need from the agent. +class TelegramAgent { + public: + virtual ~TelegramAgent() = default; + virtual std::string profile() const = 0; + virtual std::string bot() const = 0; + virtual std::string home() const = 0; // agent.config.home + virtual void reset_history() = 0; + virtual std::string run_turn(const std::string& text) = 0; // run_turn(text, capture=True) + virtual void start_scheduler() = 0; // start_scheduler(agent.config, agent.store.seal) +}; + +class TelegramBot { + public: + TelegramBot(TelegramAgent& agent, TelegramTransport* transport = nullptr, BoxPort* box = nullptr, + misc::LogSink& log = misc::LogSink::null_sink(), std::ostream* out = nullptr, std::ostream* err = nullptr); + + static std::string telegram_env_path(const std::string& home); + // TELEGRAM_BOT_TOKEN from the environment, else telegram.env under home; "" when absent + static std::string load_telegram_token(const std::string& home); + static void telegram_send(Telegram& bot, const misc::Json& chat_id, const std::string& text); + std::string telegram_transcribe(Telegram& bot, const std::string& file_id); + // Returns (outcome, reply_chars); the public handler wraps it with logging. + std::pair<std::string, int> handle_inner(Telegram& bot, const misc::Json& update); + void handle_telegram_update(Telegram& bot, const misc::Json& update); + // run_telegram_bot: stop (checked once per poll) lets tests end the otherwise infinite loop; sleep is time.sleep. + int run_telegram_bot(std::function<bool()> stop = nullptr, std::function<void(int)> sleeper = nullptr); + // Points the bot at another Bot API host (tests); the defaults are the real api.telegram.org + void set_api(std::string api, std::string file_api) { api_ = std::move(api); file_api_ = std::move(file_api); } + + private: + TelegramAgent& agent_; + TelegramTransport* transport_; + BoxPort* box_; + misc::LogSink& log_; + std::ostream* out_; + std::ostream* err_; + std::unique_ptr<CurlTransport> own_; + std::string api_{TELEGRAM_API}, file_api_{TELEGRAM_FILE_API}; +}; + +} // namespace detz::telegram diff --git a/taicpp/src/telegram/telegramtransport.cpp b/taicpp/src/telegram/telegramtransport.cpp new file mode 100644 index 0000000..6f80f98 --- /dev/null +++ b/taicpp/src/telegram/telegramtransport.cpp @@ -0,0 +1,105 @@ +#include "telegramtransport.h" +#include <curl/curl.h> +#include <mutex> +namespace detz::telegram { +namespace { +struct Sink { std::string data; std::size_t cap; bool capped{false}; std::string reason; }; +// Keeps the reason phrase of the last status line, like HTTPError.reason (urllib reports what the server sent). +std::size_t on_header(char* p, std::size_t n, std::size_t m, void* u) { + Sink* s = static_cast<Sink*>(u); + std::string line(p, n * m); + if (line.compare(0, 5, "HTTP/") == 0) { + std::size_t sp1 = line.find(' '); + std::size_t sp2 = sp1 == std::string::npos ? sp1 : line.find(' ', sp1 + 1); + s->reason.clear(); + if (sp2 != std::string::npos) { + s->reason = line.substr(sp2 + 1); + while (!s->reason.empty() && (s->reason.back() == '\r' || s->reason.back() == '\n')) s->reason.pop_back(); + } + } + return n * m; +} +std::size_t on_body(char* p, std::size_t n, std::size_t m, void* u) { + Sink* s = static_cast<Sink*>(u); + std::size_t len = n * m; + if (s->data.size() + len > s->cap) { + s->data.append(p, s->cap - s->data.size()); + s->capped = true; + return 0; // abort the transfer: urlopen(...).read(cap) never pulls more + } + s->data.append(p, len); + return len; +} +void init_once() { + static std::once_flag flag; + std::call_once(flag, [] { curl_global_init(CURL_GLOBAL_DEFAULT); }); +} +std::string describe(CURLcode rc) { + switch (rc) { + case CURLE_COULDNT_RESOLVE_HOST: return "<urlopen error [Errno -2] Name or service not known>"; + case CURLE_COULDNT_CONNECT: return "<urlopen error [Errno 111] Connection refused>"; + case CURLE_OPERATION_TIMEDOUT: return "timed out"; + default: return std::string("<urlopen error ") + curl_easy_strerror(rc) + ">"; + } +} +std::string run(const std::string& url, const std::string* body, int timeout_s, std::size_t cap) { + init_once(); + CURL* c = curl_easy_init(); + if (!c) throw TelegramNetError("<urlopen error curl init failed>"); + Sink sink{"", cap, false, ""}; + curl_easy_setopt(c, CURLOPT_URL, url.c_str()); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, on_body); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &sink); + curl_easy_setopt(c, CURLOPT_HEADERFUNCTION, on_header); + curl_easy_setopt(c, CURLOPT_HEADERDATA, &sink); + curl_easy_setopt(c, CURLOPT_TIMEOUT, static_cast<long>(timeout_s)); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + struct curl_slist* headers = nullptr; + if (body) { + headers = curl_slist_append(headers, "Content-Type: application/json"); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body->c_str()); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, static_cast<long>(body->size())); + } + CURLcode rc = curl_easy_perform(c); + long status = 0; + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &status); + if (headers) curl_slist_free_all(headers); + curl_easy_cleanup(c); + if (rc != CURLE_OK && !(sink.capped && rc == CURLE_WRITE_ERROR)) throw TelegramNetError(describe(rc)); + if (status >= 400) throw TelegramNetError("HTTP Error " + std::to_string(status) + ": " + (sink.reason.empty() ? CurlTransport::status_phrase(status) : sink.reason)); + return sink.data; +} +} // namespace + +std::string CurlTransport::post_json(const std::string& url, const std::string& body, int timeout_s) { + return run(url, &body, timeout_s, static_cast<std::size_t>(-1) / 2); +} +std::string CurlTransport::get(const std::string& url, int timeout_s, std::size_t max_bytes) { return run(url, nullptr, timeout_s, max_bytes); } + +std::string CurlTransport::status_phrase(long s) { + switch (s) { + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 402: return "Payment Required"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 408: return "Request Timeout"; + case 409: return "Conflict"; + case 410: return "Gone"; + case 413: return "Content Too Large"; + case 418: return "I'm a Teapot"; + case 422: return "Unprocessable Content"; + case 429: return "Too Many Requests"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + case 504: return "Gateway Timeout"; + default: return ""; + } +} +} // namespace detz::telegram diff --git a/taicpp/src/telegram/telegramtransport.h b/taicpp/src/telegram/telegramtransport.h new file mode 100644 index 0000000..5ab8faa --- /dev/null +++ b/taicpp/src/telegram/telegramtransport.h @@ -0,0 +1,27 @@ +#pragma once +// HTTP transport behind the Telegram client (the urllib.request calls of tai.py). Replaceable in tests. +#include <stdexcept> +#include <string> +namespace detz::telegram { + +// urllib's OSError family (URLError, HTTPError, timeout): str(exc) is what short_error shows. +class TelegramNetError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; + +class TelegramTransport { + public: + virtual ~TelegramTransport() = default; + // POST application/json, returns the body; non-2xx and network failures throw TelegramNetError + virtual std::string post_json(const std::string& url, const std::string& body, int timeout_s) = 0; + // GET returning at most max_bytes of the body (urlopen(...).read(max_bytes)) + virtual std::string get(const std::string& url, int timeout_s, std::size_t max_bytes) = 0; +}; + +class CurlTransport : public TelegramTransport { + public: + std::string post_json(const std::string& url, const std::string& body, int timeout_s) override; + std::string get(const std::string& url, int timeout_s, std::size_t max_bytes) override; + // http.HTTPStatus phrase used in "HTTP Error 401: Unauthorized" + static std::string status_phrase(long status); +}; + +} // namespace detz::telegram diff --git a/taicpp/tests/gen_misc_data.py b/taicpp/tests/gen_misc_data.py new file mode 100644 index 0000000..6ba3b5a --- /dev/null +++ b/taicpp/tests/gen_misc_data.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Generates the shared inputs of the differential tests (tests/misc_data/*.json) with a fixed seed. + +Both the Python reference driver and the C++ driver read these files, so the two sides see identical +inputs. Re-run: python3 -I tests/gen_misc_data.py (from taicpp/). +""" +import json +import os +import random + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "misc_data") + + +def write(name, data): + os.makedirs(OUT, exist_ok=True) + with open(os.path.join(OUT, name), "w", encoding="utf-8") as handle: + json.dump(data, handle, ensure_ascii=True, indent=0) + handle.write("\n") + + +def json_cases(): + cases = [ + "{}", "[]", "null", "true", "false", "0", "-0", "1", "-1", "123456789012", "1.5", "-0.25", "1e3", "1E-7", "0.1", "1e22", "1e16", "123456789.123456789", + "0.00001", "0.0001", "100.0", "1e-5", "5e-324", "1.7976931348623157e308", "NaN", "Infinity", "-Infinity", + '"plain"', '"quote\\"inside"', '"back\\\\slash"', '"tab\\there"', '"nl\\nhere"', '"cr\\rhere"', '"\\b\\f"', '"\\u00e9"', '"\\u20ac"', '"\\ud83d\\ude00"', + '"\\u007f"', '"\\u0000"', '"\\u001f"', '"/"', '"\\/"', '"café"', '"€100"', '"\U0001f600 smile"', '"\\ud83d"', + '{"a": 1, "b": [1, 2, {"c": null}], "d": "e"}', '{"z": 1, "a": 2, "z": 3}', '[1, [2, [3, [4]]]]', ' { "x" : [ 1 , 2 ] } ', + '{"source": "tai-bundled", "mode": "bundled", "scope": "home", "files": {"SKILL.md": "abc12345"}, "installed_at": "2026-10-25T10:11:12.123456+00:00"}', + '[1,2', '{"a":}', '{"a" 1}', "{'a': 1}", '"unterminated', '[1,]', '{"a":1,}', "tru", "01", "1.", ".5", "+1", '"\\x41"', "", " ", "{} extra", '"a\nb"', + '[true, false, null, 1, 2.0, "s"]', '{"k": "\\u00e9\\u20ac\\ud83d\\ude00"}', "12345678901234567890", "-9223372036854775808", "9223372036854775807", + ] + return cases + + +def datetime_cases(): + return [ + "2026-10-08T09:00", "2026-10-08 09:00", "2026-10-08", "2026-10-08T09:00:30", "2026-10-08T09:00:30.5", "2026-10-08T09:00:30.123456", + "2026-10-08T09:00:30.1234567", "2026-10-08T09:00Z", "2026-10-08T09:00:00z", "2026-10-08T09:00+02:00", "2026-10-08T09:00-05:30", "2026-10-08T09:00:00+0200", + "2026-10-08T09:00:00+02", "2026-10-08T09:00:00.25+02:00", "20261008T0900", "20261008", "20261008T090030", "2026-10-08T09", "2026-10-08T0900", "2026-10-08x09:00", + "2026-13-01", "2026-02-30", "2024-02-29T12:00", "2023-02-29T12:00", "2026-10-08T25:00", "2026-10-08T09:61", "2026-10-08T09:00:61", "now", "", " ", "tomorrow", + "2026-10-08T", "2026-10-08T09:", "2026-10-08T09:0", "2026-1-8", "26-10-08", "2026-10-08T09:00+25:00", "2026-10-08T09:00+02:00:30", " 2026-10-08T09:00 ", + "2026-10-08T09:00:00.123+01:00", "0001-01-01T00:00", "9999-12-31T23:59:59.999999", "2026-10-08T09:00 +02:00", "2026-10-08T09:00:00Z\n", "2026-03-29T02:30", + "2026-10-25T02:30", "2026-10-25T12:00:00+00:00", "2026-10-25T12:00:00.000001+00:00", + "2026-10-08T09:00x+02:00", "2026-10-08T09:00\u00a0+02:00", "2026-10-08T09:00\n+02:00", "2026-10-08T09:00:00x-02:00", "2026-10-08T09:00:00xZ", "2026-10-08T09:00:00 Z", + "2026-10-08T09 Z", "2026-10-08T0900 Z", "2026-10-08T090000 +01", "2026-10-08T09:00:00.123456789 +01", "2026-10-08T09:00:00,5 +01", "2026-10-08T09:00:00.5x+01", + "2026-10-08T09:00:00+01 ", "2026-10-08T09:00:00+0100", "2026-10-08T09:00:00+01:0", "2026-10-08T09:00:00+1", "2026-10-08T09:00:00+01:00:", "2026-10-08T09:00:00++01", + "2026-10-08T09:00:00+01Z", "2026-10-08T09:00:00 +", "2026-10-08T09:00:00-", "2026-10-08 T09:00", "2026-10-08TT09:00", "2026-10-08\u00e9", "2026-10-08\u00e909:00", + "2026-10-08\u20ac09:00", "2026-10-08T09:00:00.000001+01:00:00.5", "1999-12-31T24:00:00", "2026-10-08T24:00", "2026-10-08T24:00:00.0", "9999-12-31T24:00", + "2026-10-08T09:00:00+0100:00", "2026-10-08T09:00:00+01:00:0000", "2026-10-08T09:00:00-00:00", "2026-W01-1", "2026-W53-7", "2026-W00-1", "2026-W41-8", "2026W414", + "2026W41", "2026-W41-4T10:00", "2026-W41T10:00", "2026-10-08t09:00", "2026-10-08T09:+02:00", "2026-10-08T09:00xx+02:00", "2026-10-08T09:00:00.12345x+02:00", + "2026-10-08T09:00:00.1234567x+02:00", "2026-10-08T09:00:00.123456xy+02:00", "2026-10-08T09:0+02:00", "2026-10-08T09:00:00:00", "2026-10-08T09:00:00.5.5", "2026-10-08T0900:00", + "2026-10-08T09:0000", "2026-10-08T09:00:00+01:00:00.1234567", "2026-10-08T09:00:00+01:00:00.", "2026-W53-1", "2020-W53-5", "2015-W53-4", "2026-W41-4 10:00", "2026-W41-4x", + "2026-W41-4-10", "2026-W41-410", "2026W4141", "2026W41410", "2026-10-08T09:00-23:59", "2026-10-08T09:00+23:59:59", "2026-10-08T09:00+24:00", "2026-10-08T09:00-24:00", + "0001-01-01T00:00+00:00", "0001-01-01T00:00+01:00", "0001-01-01T00:00-01:00", "0001-01-01T10:00:00+00:00", "9999-12-31T23:59:59.999999+00:00", "9999-12-31T23:59:59+00:00", + "9999-12-31T23:59:59-01:00", "9999-12-31T00:00:00+01:00", "0001-01-01T00:00:00.000001+00:00", "1970-01-01T00:00:00", "1900-01-01T00:00:00", "1800-06-15T12:00:00", + "2026-03-29T01:59:59", "2026-03-29T02:00:00", "2026-03-29T03:00:00", "2026-10-25T01:59:59", "2026-10-25T02:00:00", "2026-10-25T03:00:00", "2026-10-25T02:59:59.999999", + "2038-01-19T03:14:08", "2100-12-31T23:59:59", "5000-01-01T00:00:00", "2026-10-08T09:00:00\u00e9", "\u2003 2026-10-08T09:00 \u2003", "2026-10-08T09:00:00,123", + "2026-02-29", "2028-02-29", "2100-02-29", "2000-02-29", "20260229", "2026-10-08T09:00:00.5+02:00", "2026-10-08T09:00:00,5+02:00", "2026-10-08T09:00:00.+02:00", + ] + + +def duration_cases(): + return ["30", "30s", "5m", "2h", "1d", "1w", "10 m", " 7D ", "0", "0s", "5x", "", "m", "-5", "5.5", "1e3", "00012h", "9999999999d", "5 M", "5mm", "5٠"] + + +def delay_cases(): + return [-5.5, -1, -0.5, 0, 0.9, 1, 59, 59.99, 60, 61, 3599, 3600, 3601, 86399, 86400, 86401, 90061, 1000000, 1e9] + + +def chunk_cases(): + rng = random.Random(20261025) + words = ["alpha", "beta", "gamma", "delta", "epsilon", "café", "naïve", "€", "\U0001f600", "x", "long" * 20, "", " ", "\t"] + seps = ["\n", "\r\n", "\r", "\x0b", "\x0c", "\x1c", "\x1d", "\x1e", "\x85", "
", "
"] + cases = [] + fixed = [ + "", "one line", "a\nb\nc", "a\n\nb\n\n\nc", "\n\n\n", " \n ", "trail\n", "\nlead", "a\r\nb\rc\x0bd\x0ce\x1cf\x1dg\x1eh\x85i
j
k", + "x" * 5000, "\n".join("line %d" % i for i in range(200)), "\n".join("w" * 50 for _ in range(100)), "é" * 4100 + "\nnext", "\n".join(["same"] * 12), + "a\nb\nb\nb\nc\nc\nd", "p\np\np\nq\nq\nq\nq\nr", "same\nsame\nsame", "\n".join(["same"] * 2 + ["other"] + ["same"] * 3), + ] + for text in fixed: + cases.append({"text": text, "max_lines": 60, "overlap": 3, "max_chars": 4000}) + for text in fixed[:12]: + cases.append({"text": text, "max_lines": 5, "overlap": 2, "max_chars": 40}) + cases.append({"text": text, "max_lines": 3, "overlap": 0, "max_chars": 10}) + cases.append({"text": text, "max_lines": 4, "overlap": 9, "max_chars": 100}) + for index in range(80): + count = rng.randint(0, 150) + parts = [] + for _ in range(count): + parts.append(" ".join(rng.choice(words) for _ in range(rng.randint(0, 12)))) + parts.append(rng.choice(seps) if rng.random() < 0.3 else "\n") + text = "".join(parts) + cases.append({"text": text, "max_lines": rng.choice([60, 7, 13, 1, 2]), "overlap": rng.choice([3, 0, 1, 5]), "max_chars": rng.choice([4000, 120, 300, 17])}) + return cases + + +def fts_cases(): + return ["", "hello world", "Hello, World!", "café naïve", "Kelvin", "İstanbul", "a-b_c d.e", "UPPER lower 123", " ", "!!!", "x", "\"quoted\" 'single'", "tab\tsep\nnl", "éè ü", "snake_case_word", "alpha1beta2"] + + +def main(): + write("json_cases.json", json_cases()) + write("datetime_cases.json", datetime_cases()) + write("duration_cases.json", duration_cases()) + write("delay_cases.json", delay_cases()) + write("chunk_cases.json", chunk_cases()) + write("fts_cases.json", fts_cases()) + print("wrote %s" % OUT) + + +main() diff --git a/taicpp/tests/misc.mk b/taicpp/tests/misc.mk new file mode 100644 index 0000000..e761ef3 --- /dev/null +++ b/taicpp/tests/misc.mk @@ -0,0 +1,34 @@ +# Standalone build of the misc modules (skills, sched, telegram, rag, install) and their tests. +# Usage (from taicpp/): make -f tests/misc.mk test +# The integrator can fold these into the main Makefile; see docs/wiring/misc.md. +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1 +B = build/misc +DIRS = src/skills src/sched src/telegram src/rag src/install +INC = $(foreach d,$(DIRS),-I$d) +SRCS = $(foreach d,$(DIRS),$(wildcard $(d)/*.cpp)) +OBJS = $(patsubst %.cpp,$(B)/%.o,$(SRCS)) +LIBS = -lsqlite3 -lcurl -lz -lpthread + +all: $(B)/misc_test $(B)/misc_driver + +$(B)/%.o: %.cpp + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +$(B)/misc_test: tests/misc_test.cpp $(OBJS) + @mkdir -p $(B) + $(CXX) $(CXXFLAGS) $(INC) tests/misc_test.cpp $(OBJS) $(LIBS) -o $@ + +$(B)/misc_driver: tests/misc_driver.cpp $(OBJS) + @mkdir -p $(B) + $(CXX) $(CXXFLAGS) $(INC) tests/misc_driver.cpp $(OBJS) $(LIBS) -lz -o $@ + +test: $(B)/misc_test + $(B)/misc_test + +objs: $(OBJS) + +.PHONY: all test objs + +-include $(OBJS:.o=.d) diff --git a/taicpp/tests/misc_diff.py b/taicpp/tests/misc_diff.py new file mode 100644 index 0000000..cbfaf89 --- /dev/null +++ b/taicpp/tests/misc_diff.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Differential test of the misc modules: runs each scenario through the real tai.py (tests/misc_py.py) +and through the C++ driver (build/misc/misc_driver) on identical state, then diffs the transcripts and +the resulting file trees. + +Usage (from taicpp/): python3 tests/misc_diff.py [scenario ...] (build first: make -f tests/misc.mk all) +Exit code 0 only when every selected scenario matches byte for byte after path/timestamp normalising. +""" +import difflib +import http.server +import json +import threading +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +TAICPP = os.path.dirname(HERE) +DRIVER = os.path.join(TAICPP, "build", "misc", "misc_driver") +PYDRV = os.path.join(HERE, "misc_py.py") +DATA = os.path.join(HERE, "misc_data") + +ORDER = ["json", "datetime", "chunk", "fts", "skills_data", "skills_files", "skills_ops", "install_hooks", "install_hooks_noshell", "install_ops", + "install_noengine", "vault_line", "sched", "rag", "telegram", "telegram_install", "release", "audit"] +EXTRA_BIN = {"install_hooks": ("zsh", "fish")} # scenario -> extra executables to create in the restricted PATH dir +TZ = "Europe/Amsterdam" + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2})?") +STAMP_RE = re.compile(r"\d{8}T\d{6}Z") +PID_RE = re.compile(r"\.tmp-\d+") + + +TG_UPDATES = [ + {"update_id": 10, "message": {"chat": {"id": 111}, "text": "/start"}}, + {"update_id": 11, "message": {"chat": {"id": 111}, "text": "hello there"}}, + {"update_id": 12, "message": {"chat": {"id": 222}, "voice": {"file_id": "f1"}}}, + {"update_id": 13, "message": {"chat": {"id": 111}, "text": "explode now"}}, + {"update_id": 14}, + {"update_id": 15, "message": {"chat": {"id": 333}, "text": "/new"}}, + {"update_id": 16, "message": {"chat": {"id": 111}, "text": "long"}}, + {"message": {"chat": {"id": 5}, "text": "no update id"}}, +] + + +class TgServer: + """Scripted Bot API on localhost; records every request so both sides can be compared on the wire.""" + + def __init__(self): + self.log = [] + self.polls = {} + outer = self + + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def reply(self, code, body, reason=None): + data = body if isinstance(body, bytes) else body.encode() + self.send_response(code, reason) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def handle_any(self, method): + n = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(n).decode() if n else "" + outer.log.append("%s %s ct=%s body=%s" % (method, self.path, self.headers.get("Content-Type"), body)) + path = self.path + if path.startswith("/file/bot"): + if path.endswith("voice/f1.oga"): + return self.reply(200, b"AUDIO") + if path.endswith("voice/fmodel.oga"): + return self.reply(200, b"OTHER") + return self.reply(404, "{}", "Not Found") + token, _, name = path[len("/bot"):].partition("/") + if token == "bad": + return self.reply(401, json.dumps({"ok": False, "description": "Unauthorized"}), "Unauthorized") + if token == "nojson" and name == "getMe": + return self.reply(200, "not json") + if name == "getMe": + return self.reply(200, json.dumps({"ok": True, "result": {} if token == "noname" else {"username": "tai_test_bot"}})) + if name == "getUpdates": + count = outer.polls[token] = outer.polls.get(token, 0) + 1 + if count == 1: + return self.reply(200, json.dumps({"ok": True, "result": TG_UPDATES})) + return self.reply(502, "{}", "Bad Gateway") + if name == "getFile": + fid = json.loads(body).get("file_id") + if fid == "nofile": + return self.reply(200, json.dumps({"ok": True, "result": {}})) + if fid == "fbad": + return self.reply(500, "{}", "Internal Server Error") + if fid == "fmodel": + return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/fmodel.oga"}})) + if fid == "fmissingfile": + return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/gone.oga"}})) + return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/f1.oga"}})) + return self.reply(200, json.dumps({"ok": True, "result": {}})) + + def do_POST(self): + self.handle_any("POST") + + def do_GET(self): + self.handle_any("GET") + + self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H) + self.port = self.httpd.server_address[1] + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def stop(self): + self.httpd.shutdown() + self.httpd.server_close() + + +SERVER_SCENARIOS = {"telegram", "telegram_install"} + + +def normalize(text, root): + text = text.replace(root, "<ROOT>") + text = TS_RE.sub("<TS>", text) + text = STAMP_RE.sub("<STAMP>", text) + text = PID_RE.sub(".tmp-<PID>", text) + return text + + +def make_root(base, name, extra_bins): + root = os.path.join(base, name) + for sub in ("home/.tai", "proj", "bin"): + os.makedirs(os.path.join(root, sub), exist_ok=True) + os.symlink(sys.executable, os.path.join(root, "bin", "python3")) + for tool in extra_bins: + path = os.path.join(root, "bin", tool) + with open(path, "w") as handle: + handle.write("#!/bin/sh\nexit 0\n") + os.chmod(path, 0o755) + return root + + +def env_for(root): + return { + "HOME": os.path.join(root, "home"), + "TAI_HOME": os.path.join(root, "home", ".tai"), + "PATH": os.path.join(root, "bin"), + "TZ": TZ, + "LANG": "C.UTF-8", + "MISC_DATA_DIR": DATA, + "PYTHONIOENCODING": "utf-8", + "SHELL": "/bin/bash", + "MISC_TAI_PY": os.path.join(os.path.dirname(TAICPP), "tai.py"), + } + + +def tree(root): + found = {} + for dirpath, dirs, files in os.walk(root): + dirs.sort() + rel_dir = os.path.relpath(dirpath, root) + if rel_dir == "bin" or rel_dir.startswith("bin" + os.sep): + continue + if rel_dir.startswith(os.path.join("home", ".tai", "venv")): + continue # venv contents embed the invoking interpreter path; the report lines are compared instead + for d in dirs: + full = os.path.join(dirpath, d) + if os.path.islink(full): + found[os.path.relpath(full, root)] = "symlink -> " + normalize(os.readlink(full), root) + for f in files: + full = os.path.join(dirpath, f) + rel = os.path.relpath(full, root) + if os.path.islink(full): + found[rel] = "symlink -> " + normalize(os.readlink(full), root) + continue + with open(full, "rb") as handle: + raw = handle.read() + mode = oct(os.stat(full).st_mode & 0o777) + try: + body = normalize(raw.decode("utf-8"), root) + digest = hashlib.sha256(body.encode("utf-8")).hexdigest()[:16] + except UnicodeDecodeError: + digest = "bin-" + hashlib.sha256(raw).hexdigest()[:16] + if rel.endswith((".db", ".db-wal", ".db-shm")): + digest = "db" + found[normalize(rel, root)] = "%s %s" % (mode, digest) + return found + + +def run(cmd, root, extra_env=None): + env = env_for(root) + if extra_env: + env.update(extra_env) + done = subprocess.run(cmd, cwd=os.path.join(root, "proj"), env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True, timeout=600) + return done.returncode, done.stdout.decode("utf-8", "replace"), done.stderr.decode("utf-8", "replace") + + +def check(name, base, verbose): + root_py = make_root(base, name + "-py", EXTRA_BIN.get(name, ())) + code, _out, err = run([sys.executable, "-I", PYDRV, name, root_py, "prepare"], root_py) + if code != 0: + return False, "python prepare failed (%d): %s" % (code, err[-2000:]) + root_cpp = os.path.join(base, name + "-cpp") + shutil.copytree(root_py, root_cpp, symlinks=True) + # copytree keeps absolute symlink targets; point the bin symlink at the same interpreter (already absolute) + def one(cmd, root): + if name not in SERVER_SCENARIOS: + return run(cmd, root) + srv = TgServer() + try: + extra = {"MISC_TG_API": "http://127.0.0.1:%d/bot" % srv.port, "MISC_TG_FILE": "http://127.0.0.1:%d/file/bot" % srv.port} + code, o, e = run(cmd, root, extra) + finally: + srv.stop() + o = o + "---- wire log\n" + "\n".join(l.replace(str(srv.port), "PORT") for l in srv.log) + if name == "telegram": + o += "\n---- stderr\n" + e + return code, o, "" + + code_p, out_p, err_p = one([sys.executable, "-I", PYDRV, name, root_py], root_py) + code_c, out_c, err_c = one([DRIVER, name, root_cpp], root_cpp) + out_p = normalize(out_p, root_py) + out_c = normalize(out_c, root_cpp) + problems = [] + if code_p != 0: + problems.append("python exit %d: %s" % (code_p, err_p[-2000:])) + if code_c != 0: + problems.append("cpp exit %d: %s" % (code_c, err_c[-2000:])) + if out_p != out_c: + diff = list(difflib.unified_diff(out_p.splitlines(), out_c.splitlines(), "python", "cpp", lineterm="", n=1)) + problems.append("transcript differs (%d diff lines):\n%s" % (len(diff), "\n".join(diff[:60]))) + tp, tc = tree(root_py), tree(root_cpp) + if tp != tc: + lines = [] + for key in sorted(set(tp) | set(tc)): + if tp.get(key) != tc.get(key): + lines.append(" %s: py=%s cpp=%s" % (key, tp.get(key), tc.get(key))) + problems.append("file tree differs:\n%s" % "\n".join(lines[:60])) + if verbose: + sys.stdout.write("---- python transcript (%d lines)\n%s\n" % (len(out_p.splitlines()), out_p)) + return not problems, "\n".join(problems), len(out_p.splitlines()), len(tp) + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("-")] + verbose = "-v" in sys.argv + if not os.path.isdir(DATA): + subprocess.run([sys.executable, "-I", os.path.join(HERE, "gen_misc_data.py")], check=True) + names = args or ORDER + base = tempfile.mkdtemp(prefix="misc-diff-") + failed = 0 + try: + for name in names: + result = check(name, base, verbose) + ok, why = result[0], result[1] + if ok: + print("PASS %-14s transcript=%d lines, tree=%d files" % (name, result[2], result[3])) + else: + failed += 1 + print("FAIL %s\n%s" % (name, why)) + finally: + if "--keep" not in sys.argv: + shutil.rmtree(base, ignore_errors=True) + else: + print("kept %s" % base) + print("%d scenario(s), %d failed" % (len(names), failed)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tests/misc_driver.cpp b/taicpp/tests/misc_driver.cpp new file mode 100644 index 0000000..d99554d --- /dev/null +++ b/taicpp/tests/misc_driver.cpp @@ -0,0 +1,1666 @@ +// C++ side of the differential tests (tests/misc_diff.py): prints the same transcript as tests/misc_py.py. +#include <fcntl.h> +#include <sys/stat.h> +#include <unistd.h> +#include <algorithm> +#include <cstdio> +#include <cstdlib> +#include <fstream> +#include <functional> +#include <chrono> +#include <condition_variable> +#include <iostream> +#include <mutex> +#include <thread> +#include <map> +#include <sstream> +#include "bundled.h" +#include "bundled_data.h" +#include "audittool.h" +#include "release.h" +#include "chunker.h" +#include "telegram.h" +#include "ragtool.h" +#include "retrievaleval.h" +#include "install.h" +#include "installconsts.h" +#include "ftssearch.h" +#include "miscdatetime.h" +#include "miscjson.h" +#include "miscpy.h" +#include "miscsha256.h" +#include "schedtime.h" +#include "schedtool.h" +#include "schedvault.h" +#include "sqlitedb.h" +#include "scheduler.h" +#include "schedtool.h" +#include "skillfile.h" +#include "skilltool.h" +using detz::misc::Json; +using detz::misc::PyFs; +using detz::misc::PyStr; +namespace sk = detz::skills; + +static std::string DATA_DIR; + +static Json data(const std::string& name) { + std::ifstream in(DATA_DIR + "/" + name, std::ios::binary); + std::stringstream ss; + ss << in.rdbuf(); + return Json::loads(ss.str()); +} +static std::string j(const Json& v) { return v.dumps(); } +static std::string js(const std::string& s) { return Json(s).dumps(); } +static Json arr_of(const std::vector<std::string>& v) { + Json a = Json::array(); + for (const auto& s : v) a.push(s); + return a; +} +static void out(const std::string& line = "") { std::cout << line << "\n"; } +static std::string tai_home() { return PyFs::env("TAI_HOME"); } +static std::string py_bool(bool b) { return b ? "True" : "False"; } +static void write(const std::string& path, const std::string& text) { + PyFs::makedirs(PyFs::dirname(path)); + PyFs::write_bytes(path, text); +} +static std::string read(const std::string& path) { return PyFs::read_bytes(path); } + +// ------------------------------------------------------------------ scenarios without state +static void sc_json(const std::string&) { + for (const auto& c : data("json_cases.json").items()) { + try { + Json v = Json::loads(c.as_string()); + out(js(c.as_string()) + " => " + v.dumps()); + } catch (const detz::misc::JsonError&) { + out(js(c.as_string()) + " => ERR"); + } + } +} + +static void sc_datetime(const std::string&) { + for (const auto& c : data("datetime_cases.json").items()) { + std::string iso, parsed; + try { + iso = detz::misc::DateTime::fromisoformat(c.as_string()).isoformat(); + } catch (const std::exception&) { + iso = "ERR"; + } + try { + parsed = detz::sched::ScheduleTime::parse_schedule_at(c.as_string()); + } catch (const std::exception& e) { + parsed = std::string("ERR ") + e.what(); + } + bool err = parsed.rfind("ERR", 0) == 0; + std::string shown = "-"; + if (!err) { + try { + shown = js(detz::sched::ScheduleTime::local_display(parsed)); + } catch (const std::exception& e) { + shown = std::string("ERR ") + e.what(); + } + } + out(js(c.as_string()) + " => " + iso + " | " + parsed + " | " + shown); + } + for (const auto& c : data("duration_cases.json").items()) { + auto d = detz::sched::ScheduleTime::parse_duration(c.as_string()); + out("duration " + js(c.as_string()) + " => " + (d ? std::to_string(*d) : std::string("null"))); + } + for (const auto& c : data("delay_cases.json").items()) { + out("delay " + j(c) + " => " + js(detz::sched::ScheduleTime::format_delay(c.as_double()))); + } +} + +static void sc_chunk(const std::string&) { + for (const auto& c : data("chunk_cases.json").items()) { + std::string text = c.get("text")->as_string(); + auto chunks = detz::rag::Chunker::chunk_lines(text, static_cast<int>(c.get("max_lines")->as_int()), static_cast<int>(c.get("overlap")->as_int()), + static_cast<int>(c.get("max_chars")->as_int())); + out("chunks " + std::to_string(chunks.size()) + " " + j(arr_of(chunks))); + out("collapse " + js(detz::rag::Chunker::collapse_repeated_lines(text))); + out("lines " + j(arr_of(PyStr::splitlines(text)))); + out("strip " + js(PyStr::strip(text))); + } +} + +static void sc_fts(const std::string&) { + using detz::rag::FtsQuery; + for (const auto& c : data("fts_cases.json").items()) { + auto terms = FtsQuery::fts_escape(c.as_string()); + out(js(c.as_string()) + " => " + (terms ? j(arr_of(*terms)) : std::string("null"))); + if (terms) { + out(" match " + js(FtsQuery::fts_match(*terms))); + out(" or " + js(FtsQuery::fts_match_or(*terms))); + out(" prefix " + js(FtsQuery::fts_match_prefix(*terms))); + out(" variants " + j(arr_of(FtsQuery::fts_match_variants(*terms)))); + } + } + out("rank " + js(FtsQuery::fts_rank("fts_docs"))); + for (const std::string& raw : std::vector<std::string>{"Release Notes!", " a b ", "---", std::string(40, 'x'), "\xc3\xa9t\xc3\xa9", "a,b;c", "Hello World 2", "..", "UPPER", "a-b-c"}) { + out("tag " + js(raw) + " => " + js(detz::misc::TagNames::normalize_tag(raw))); + } + std::vector<std::string> items = {"A b", "a-b", "", "C!", std::string(40, 'd'), std::string(39, 'd') + "e"}; + for (int i = 0; i < 30; ++i) items.push_back("t" + std::to_string(i)); + out("tags " + j(arr_of(detz::misc::TagNames::normalize_tags(items)))); +} + +static void sc_skills_data(const std::string&) { + for (const auto& kv : sk::BundledData::files()) { + out("skill " + kv.first + " upstream " + js(sk::BundledData::upstream().at(kv.first))); + for (const auto& f : kv.second) { + out(" file " + f.first + " len " + std::to_string(f.second.size()) + " sha8 " + sk::BundledSkills::bundled_skill_digest(f.second) + + " sha256 " + detz::misc::Sha256::hex(f.second)); + } + } + for (const auto& kv : sk::BundledData::blueprints()) { + out("blueprint " + kv.first + " hint " + js(kv.second.hint) + " scope " + kv.second.scope + " brief " + js(kv.second.brief)); + } + out("catalog empty " + js(sk::SkillFiles::skill_catalog(sk::SkillSet()))); + sk::SkillSet two; + sk::Skill a; + a.name = "deep-research"; + a.description = std::string(400, 'x'); + sk::Skill b; + b.name = "zeta"; + b.description = "short"; + two.put(a); + two.put(b); + out("catalog one " + js(sk::SkillFiles::skill_catalog(two))); + out("prompt " + js(sk::SkillFiles::create_skill_prompt("my-skill", "do the thing", "/tmp/x/my-skill"))); + for (const std::string& raw : std::vector<std::string>{"a", "ab", "a-b", "-a", "a-", "A", "a_b", std::string(64, 'a'), std::string(65, 'a'), "", "a b", "0x", "a\n"}) { + out("name " + js(raw) + " => " + py_bool(sk::SkillFiles::skill_name_valid(raw))); + } + std::vector<std::vector<std::string>> front_cases = { + {"name: x", "description: y"}, + {"name: 'x'", "description: \"y z\""}, + {"name: x", "description: >", " line one", " line two", "other: v"}, + {"name: x", "description: |", " line one", " line two", "tail: 1"}, + {"name: x", "description: >-", " folded"}, + {" indented: no", "key: va:lue", "noval", "k2:", "k3: ''quoted''"}, + {"a: >", "b: 1"}, + {"a: |", "\tt1", " t2", "z: 9"}, + }; + for (const auto& lines : front_cases) { + auto front = sk::SkillFiles::parse_skill_front(lines); + // json.dumps of a dict keeps the parse (insertion) order; the C++ map is sorted, so compare as sorted key objects + Json o = Json::object(); + for (const auto& kv : front) o.set(kv.first, kv.second); + out("front " + j(arr_of(lines)) + " => " + j(o)); + } +} + +// ------------------------------------------------------------------ filesystem scenarios +static Json skill_json(const std::optional<sk::Skill>& s) { + if (!s) return Json(); + Json o = Json::object(); + o.set("name", s->name); + o.set("description", s->description); + o.set("body", s->body); + return o; +} + +static void sc_skills_files(const std::string& root) { + std::string home = tai_home(); + out("== front-matter parse"); + std::vector<std::pair<std::string, std::string>> cases = { + {"ok", "---\nname: ok\ndescription: fine one\n---\n\n\nbody here\n"}, + {"crlf", "---\r\nname: crlf\r\ndescription: win\r\n---\r\nbody\r\n"}, + {"nofront", "just text\n"}, + {"noend", "---\nname: noend\ndescription: d\n"}, + {"badname", "---\nname: Bad_Name\ndescription: d\n---\nbody\n"}, + {"nodesc", "---\nname: nodesc\n---\nbody\n"}, + {"quoted", "---\nname: 'quoted'\ndescription: \"has: colon\"\n---\nbody\n"}, + {"folded", "---\nname: folded\ndescription: >\n one\n two\n---\nbody\n"}, + {"emptybody", "---\nname: emptybody\ndescription: d\n---"}, + }; + for (const auto& kv : cases) { + std::string path = root + "/parse/" + kv.first + "/SKILL.md"; + write(path, kv.second); + out("parse " + kv.first + " => " + j(skill_json(sk::SkillFiles::parse_skill_file(path)))); + } + out("== discover with external roots and collisions"); + std::string proj = root + "/proj"; + std::string uh = PyFs::home(); + write(uh + "/.claude/skills/ext-home/SKILL.md", "---\nname: ext-home\ndescription: from claude home\n---\nbody ext-home\n"); + write(uh + "/.claude/skills/shared/SKILL.md", "---\nname: shared\ndescription: external copy\n---\nbody external\n"); + write(proj + "/.claude/skills/proj-claude/SKILL.md", "---\nname: proj-claude\ndescription: project claude\n---\nbody pc\n"); + write(proj + "/.agents/skills/proj-agents/SKILL.md", "---\nname: proj-agents\ndescription: project agents\n---\nbody pa\n"); + write(home + "/skills/shared/SKILL.md", "---\nname: shared\ndescription: tai copy\n---\nbody tai\n"); + write(home + "/skills/native/SKILL.md", "---\nname: native\ndescription: native home skill\n---\nbody native\n"); + write(home + "/skills/native/scripts/run.sh", "echo hi\n"); + write(home + "/skills/native/references/ref.md", "ref\n"); + write(home + "/skills/native/assets/a.txt", "a\n"); + write(proj + "/.tai/skills/proj-tai/SKILL.md", "---\nname: proj-tai\ndescription: project tai skill\n---\nbody ptai\n"); + write(home + "/skills/broken/SKILL.md", "no front matter\n"); + write(home + "/skills/stray-file.txt", "x\n"); + { + Json meta = Json::object(); + meta.set("source", "/some/path"); + meta.set("mode", "symlink"); + meta.set("scope", "home"); + meta.set("n", 3); + Json nested = Json::object(); + Json arr = Json::array(); + arr.push(1).push(Json()).push(true); + nested.set("a", arr); + meta.set("nested", nested); + sk::SkillFiles::save_skill_install_meta(home + "/skills", "native", meta); + } + auto skills = sk::SkillFiles::discover_skills(home, proj); + for (const auto& s : skills.items()) { + out("skill " + s.name + " root=" + s.root + " managed=" + py_bool(s.managed) + " install=" + (s.install ? j(*s.install) : std::string("null")) + + " note=" + js(sk::SkillFiles::skill_provenance_note(s)) + " extras=" + j(arr_of(sk::SkillFiles::skill_extra_files(s.root)))); + out(" desc=" + js(s.description) + " body=" + js(s.body)); + } + std::vector<std::string> order; + for (const auto& s : skills.items()) order.push_back(s.name); + out("order " + j(arr_of(order))); + out("catalog " + js(sk::SkillFiles::skill_catalog(skills))); + out("== scope helpers"); + for (const std::string scope : {"project", "home"}) { + out("scope_dir " + scope + " " + sk::SkillFiles::skill_scope_dir(scope, home, proj, "nm")); + out("scope_dir " + scope + " " + sk::SkillFiles::skill_scope_dir(scope, home, proj)); + } + std::string empty = root + "/emptyproj"; + PyFs::makedirs(empty); + out("default scope empty " + sk::SkillFiles::default_skill_scope(empty)); + for (const std::string marker : {".git", ".tai", ".claude/skills", ".agents/skills"}) { + std::string tag = PyStr::replace_all(marker, "/", "_"); + std::string d = root + "/mk-" + tag; + PyFs::makedirs(d + "/" + marker); + out("default scope " + marker + " " + sk::SkillFiles::default_skill_scope(d)); + } + for (const std::string path : {"/a/b/.claude/skills/x", "/a/b/.agents/skills/x/", "/a/b/other/x", "/a/.claude/skills/../x", "/a/.claude/skills//x"}) { + out("external " + path + " " + py_bool(sk::SkillFiles::is_external_skill_root(path))); + } + auto none = sk::SkillFiles::load_skill_install_meta(home + "/skills", "nothing"); + out("meta none " + (none ? j(*none) : std::string("null"))); + write(home + "/skills/.installs/bad.json", "{not json"); + auto bad = sk::SkillFiles::load_skill_install_meta(home + "/skills", "bad"); + out("meta bad " + (bad ? j(*bad) : std::string("null"))); + auto ok = sk::SkillFiles::load_skill_install_meta(home + "/skills", "native"); + out("meta ok " + (ok ? j(*ok) : std::string("null"))); +} + +static int count_files(const std::string& base) { return static_cast<int>(PyFs::walk_files(base).size()); } + +static void sc_skills_install_tool(const std::string& root); + +static void sc_skills_ops(const std::string& root) { + std::string home = tai_home(); + sk::BundledSkills bundled; + auto step = [&](const std::string& label, const std::string& mode) { + out(label + " => " + bundled.op_skills(mode, home)); + out(" status: " + sk::BundledSkills::status_line(home)); + out(" files: " + std::to_string(count_files(home + "/skills"))); + }; + out("-- fresh"); + step("install", "install"); + step("install again", "install"); + step("upgrade clean", "upgrade"); + { + std::string s = "["; + bool first = true; + for (const auto& n : sk::BundledSkills::names()) { + s += std::string(first ? "" : ", ") + (sk::BundledSkills::bundled_skill_pristine(home, n) ? "true" : "false"); + first = false; + } + out("pristine all " + s + "]"); + } + std::string taste = home + "/skills/taste/SKILL.md"; + write(taste, "user rewrite"); + out("-- modified taste"); + step("install", "install"); + step("upgrade", "upgrade"); + std::string extra = home + "/skills/taste/mynotes.md"; + write(extra, "mine"); + step("reinstall", "reinstall"); + out("extra kept " + py_bool(PyFs::isfile(extra))); + out("-- crlf edit counts as equal after universal newlines"); + std::string path = home + "/skills/plan/SKILL.md"; + write(path, PyStr::replace_all(read(path), "\n", "\r\n")); + step("upgrade crlf", "upgrade"); + out("-- uninstall with user content"); + std::string gitp = home + "/skills/git/SKILL.md"; + write(gitp, "user rewrite git"); + std::string gextra = home + "/skills/git/mynotes.md"; + write(gextra, "mine"); + write(home + "/skills/git/sub/deep.txt", "deep"); + step("uninstall", "uninstall"); + step("uninstall again", "uninstall"); + PyFs::remove(gitp); + PyFs::remove(gextra); + PyFs::remove(home + "/skills/git/sub/deep.txt"); + step("uninstall final", "uninstall"); + step("uninstall empty", "uninstall"); + out("-- sidecar-only uninstall"); + { + Json meta = Json::object(); + meta.set("files", Json::object()); + sk::SkillFiles::save_skill_install_meta(home + "/skills", "taste", meta); + } + step("uninstall sidecar", "uninstall"); + out("-- tracked-digest removal of an edited file"); + step("install", "install"); + auto meta = sk::SkillFiles::load_skill_install_meta(home + "/skills", "taste"); + std::vector<std::string> keys; + for (const auto& m : meta->members()) keys.push_back(m.first); + std::sort(keys.begin(), keys.end()); + out("meta " + j(arr_of(keys))); + out("meta files " + j(*meta->get("files"))); + { + Json fixed = Json::object(); + for (const char* k : {"source", "mode", "scope", "upstream", "bundled_version"}) fixed.set(k, *meta->get(k)); + out("meta fixed " + j(fixed)); + } + chmod((home + "/skills/taste").c_str(), 0500); + step("install readonly", "install"); + chmod((home + "/skills/taste").c_str(), 0700); + out("-- error path: skills dir is a file"); + std::string other = root + "/home2"; + PyFs::makedirs(other); + write(other + "/skills", "file not dir"); + out("blocked => " + bundled.op_skills("install", other)); + out("blocked uninstall => " + bundled.op_skills("uninstall", other)); + sc_skills_install_tool(root); +} + +class SkillApp : public sk::SkillHost { + public: + std::string home_, cwd_; + sk::SkillSet skills_; + int depth_{0}; + std::vector<std::string> approvals; + int refreshed{0}; + bool approve{true}; + std::function<std::string(const std::string&, int)> runner; + bool ask_approval(const std::string& what) override { approvals.push_back(what); return approve; } + std::string home() const override { return home_; } + std::string cwd() const override { return cwd_; } + int depth() const override { return depth_; } + const sk::SkillSet& skills() const override { return skills_; } + void refresh_skills() override { ++refreshed; skills_ = sk::SkillFiles::discover_skills(home_, cwd_); } + void audit_skill_install(const std::string&, const std::string&) override {} + std::string run_worker(const std::string& prompt, int timeout, int) override { return runner(prompt, timeout); } +}; + +static std::string dispatch(sk::SkillTool& tool, const std::string& name, const Json& args) { + try { + if (name == "install_skill") return tool.run_install_skill(args); + if (name == "load_skill") return tool.run_load_skill(args); + if (name == "create_skill") return tool.run_create_skill(args); + } catch (const std::exception& e) { + return "error: " + PyStr::short_error(e.what()); + } + return "error: unknown tool " + name; +} + +static Json obj(std::initializer_list<std::pair<const char*, Json>> kv) { + Json o = Json::object(); + for (const auto& p : kv) o.set(p.first, p.second); + return o; +} + +static std::string replace_root(std::string s, const std::string& root) { return PyStr::replace_all(s, root, "<ROOT>"); } + +static void sc_skills_install_tool(const std::string& root) { + std::string home = tai_home(); + std::string proj = root + "/proj"; + if (chdir(proj.c_str()) != 0) return; + SkillApp app; + app.home_ = home; + app.cwd_ = proj; + app.skills_ = sk::SkillFiles::discover_skills(home, proj); + sk::SkillTool tool(app); + out("== install_skill / load_skill / create_skill"); + std::string src = root + "/srcskill"; + write(src + "/SKILL.md", "---\nname: from-src\ndescription: installed from a dir\n---\nbody src\n"); + write(src + "/scripts/x.sh", "echo x\n"); + write(src + "/references/r.md", "r\n"); + std::string nested = root + "/nested"; + write(nested + "/inner/SKILL.md", "---\nname: inner\ndescription: nested one\n---\nbody inner\n"); + std::string two = root + "/two"; + write(two + "/a/SKILL.md", "---\nname: aa\ndescription: d\n---\nb\n"); + write(two + "/b/SKILL.md", "---\nname: bb\ndescription: d\n---\nb\n"); + std::string barefile = root + "/bare.md"; + write(barefile, "---\nname: bare-one\ndescription: bare md file\n---\nbody bare\n"); + std::string badfile = root + "/bad.md"; + write(badfile, "no front\n"); + std::string ext = proj + "/.claude/skills/extskill"; + write(ext + "/SKILL.md", "---\nname: extskill\ndescription: external one\n---\nbody ext\n"); + + auto call = [&](const std::string& label, const Json& args) { out(label + " => " + js(dispatch(tool, "install_skill", args))); }; + call("empty", obj({{"source", ""}})); + call("bad scope", obj({{"source", src}, {"scope", "mars"}})); + call("bad mode", obj({{"source", src}, {"mode", "link"}})); + call("missing", obj({{"source", root + "/nope"}})); + call("dir home", obj({{"source", src}, {"scope", "home"}})); + call("dir home again denied?", obj({{"source", src}, {"scope", "home"}})); + app.approve = false; + call("dir home denied", obj({{"source", src}, {"scope", "home"}})); + app.approve = true; + call("dir home force", obj({{"source", src}, {"scope", "home"}, {"force", true}})); + call("dir project", obj({{"source", src}, {"scope", "project"}, {"name", "renamed"}})); + call("bad name", obj({{"source", src}, {"name", "Bad Name"}})); + call("nested", obj({{"source", nested}, {"scope", "home"}})); + call("two children", obj({{"source", two}, {"scope", "home"}})); + call("bare file", obj({{"source", barefile}, {"scope", "home"}})); + call("bad file", obj({{"source", badfile}})); + call("symlink ext default", obj({{"source", ext}, {"scope", "home"}})); + call("symlink explicit", obj({{"source", src}, {"scope", "home"}, {"mode", "symlink"}, {"name", "linked"}, {"force", true}})); + call("symlink bare", obj({{"source", barefile}, {"mode", "symlink"}})); + call("not a skill source", obj({{"source", "/dev/null"}})); + out("approvals " + j(arr_of(app.approvals))); + out("refreshed " + std::to_string(app.refreshed)); + out("skills " + j(arr_of(app.skills_.sorted_names()))); + out("== load_skill"); + app.skills_ = sk::SkillFiles::discover_skills(home, proj); + for (const std::string name : {"from-src", "deep-research", "extskill", "nonexistent", ""}) { + out("load " + js(name) + " => " + js(dispatch(tool, "load_skill", obj({{"name", name}})))); + } + out("== create_skill"); + app.runner = [](const std::string& prompt, int timeout) { return "worker said: " + PyStr::prefix(prompt, 40) + " t=" + std::to_string(timeout); }; + auto create = [&](const Json& args) { out("create " + j(args) + " => " + js(dispatch(tool, "create_skill", args))); }; + create(obj({{"name", ""}, {"brief", "b"}})); + create(obj({{"name", "ok-name"}, {"brief", ""}})); + create(obj({{"name", "ok-name"}, {"brief", "b"}, {"scope", "mars"}})); + create(obj({{"name", "made"}, {"brief", "make it"}, {"scope", "home"}, {"timeout", 10}})); + create(obj({{"name", "made2"}, {"brief", "make it"}, {"timeout", "abc"}})); + create(obj({{"name", "made3"}, {"brief", "x"}, {"timeout", 99999}})); + app.depth_ = 5; + out("create depth => " + js(dispatch(tool, "create_skill", obj({{"name", "dd"}, {"brief", "b"}})))); + app.depth_ = 0; + app.runner = [&](const std::string& prompt, int) { + std::string marker = "Create a new agent skill named 'wrote' in "; + std::string path = prompt.substr(marker.size()); + path = path.substr(0, path.find(".\n")); + write(path + "/SKILL.md", "---\nname: wrote\ndescription: written by worker\n---\nbody wrote\n"); + return replace_root("done writing " + path, root); + }; + out("create written => " + js(replace_root(dispatch(tool, "create_skill", obj({{"name", "wrote"}, {"brief", "w"}, {"scope", "home"}})), root))); + out("create replaced => " + js(replace_root(dispatch(tool, "create_skill", obj({{"name", "wrote"}, {"brief", "w"}, {"scope", "home"}})), root))); + out("refreshed " + std::to_string(app.refreshed)); +} + +// ------------------------------------------------------------------ install scenarios +namespace ins = detz::install; + +class FakeBox : public ins::ContainerPort { + public: + std::optional<std::string> engine; + std::string state = "running", venv = "venv ready", ensure_error; + std::optional<std::string> container_engine() override { return engine; } + std::string box_state(const std::string&) override { return state; } + std::string box_venv_state(const std::string&) override { return venv; } + std::string ensure_box() override { + if (!ensure_error.empty()) throw detz::misc::BackendError(ensure_error); + return engine ? *engine : std::string(); + } +}; + +static ins::InstallContext install_ctx() { + ins::InstallContext ctx; + const char* py = std::getenv("MISC_TAI_PY"); + ctx.script_path = py ? py : "tai.py"; + return ctx; +} + +static void sc_install_hooks(const std::string& root) { + FakeBox box; + ins::StdIo io; + ins::Installer inst(install_ctx(), box, io); + std::string home = tai_home(); + std::string uh = PyFs::home(); + std::string rc = uh + "/.bashrc", zrc = uh + "/.zshrc"; + write(rc, "# my bashrc\nexport A=1"); + write(zrc, ""); + auto status = [&] { return PyStr::join(inst.status_lines(home), "\n"); }; + std::vector<std::pair<std::string, std::function<std::string()>>> steps = { + {"bash install", [&] { return inst.op_hook("install", home); }}, + {"bash install again", [&] { return inst.op_hook("install", home); }}, + {"bash upgrade", [&] { return inst.op_hook("upgrade", home); }}, + {"plexer install", [&] { return inst.op_plexer_hook("install", home); }}, + {"plexer again", [&] { return inst.op_plexer_hook("install", home); }}, + {"zsh install", [&] { return inst.op_zsh_hook("install", home); }}, + {"zsh again", [&] { return inst.op_zsh_hook("install", home); }}, + {"fish install", [&] { return inst.op_fish_hook("install", home); }}, + {"zsh plexer install", [&] { return inst.op_zsh_plexer_hook("install", home); }}, + {"fish plexer install", [&] { return inst.op_fish_plexer_hook("install", home); }}, + {"status", status}, + {"bash reinstall", [&] { return inst.op_hook("reinstall", home); }}, + {"plexer reinstall", [&] { return inst.op_plexer_hook("reinstall", home); }}, + {"zsh uninstall", [&] { return inst.op_zsh_hook("uninstall", home); }}, + {"fish uninstall", [&] { return inst.op_fish_hook("uninstall", home); }}, + {"zsh uninstall again", [&] { return inst.op_zsh_hook("uninstall", home); }}, + {"bash uninstall", [&] { return inst.op_hook("uninstall", home); }}, + {"plexer uninstall", [&] { return inst.op_plexer_hook("uninstall", home); }}, + {"zsh plexer uninstall", [&] { return inst.op_zsh_plexer_hook("uninstall", home); }}, + {"fish plexer uninstall", [&] { return inst.op_fish_plexer_hook("uninstall", home); }}, + {"status after", status}, + }; + for (const auto& st : steps) out(st.first + " => " + js(st.second())); + out("rc " + js(read(rc))); + out("zrc " + js(read(zrc))); + out("== stale and duplicated blocks"); + const std::string& begin = ins::InstallConsts::BASHRC_MARK_BEGIN(); + const std::string& end = ins::InstallConsts::BASHRC_MARK_END(); + write(rc, "head\n" + begin + "\nstale content\n" + end + "\nmid\n" + begin + "\nsecond stale\n" + end + "\ntail\n"); + out("install over stale => " + js(inst.op_hook("install", home))); + out("rc " + js(read(rc))); + write(rc, "x\n" + begin + "\nunterminated\n"); + out("install over unterminated => " + js(inst.op_hook("install", home))); + out("rc " + js(read(rc))); + out("uninstall unterminated => " + js(inst.op_hook("uninstall", home))); + out("rc " + js(read(rc))); + write(rc, "crlf\r\nline\r\n"); + out("install crlf => " + js(inst.op_hook("install", home))); + out("rc " + js(read(rc))); + chmod(rc.c_str(), 0400); + out("readonly install => " + js(inst.op_plexer_hook("install", home))); + chmod(rc.c_str(), 0600); + auto present = [&](const std::string& path) { + Json a = Json::array(); + for (bool b : {ins::Installer::hook_present(path), ins::Installer::zsh_hook_present(path), ins::Installer::fish_hook_present(path), + ins::Installer::plexer_hook_present(path), ins::Installer::zsh_plexer_hook_present(path), ins::Installer::fish_plexer_hook_present(path)}) + a.push(b); + return a; + }; + out("present " + j(present(rc))); + out("present missing " + j(present(root + "/nope"))); + for (const std::string shell : {"bash", "zsh", "fish", "other"}) out("rc path " + shell + " " + ins::Install::shell_rc_path(shell)); + setenv("XDG_CONFIG_HOME", (root + "/xdg").c_str(), 1); + out("rc path fish xdg " + ins::Install::shell_rc_path("fish")); + unsetenv("XDG_CONFIG_HOME"); + Json sp = Json::array(); + for (const char* n : {"zsh", "fish", "bash", "nosuchshell"}) sp.push(ins::Install::shell_present(n)); + out("shell_present " + j(sp)); +} + +class InstallApp : public sk::SkillHost { + public: + std::string home_, cwd_; + sk::SkillSet skills_; + std::vector<std::string> approvals; + bool approve{true}; + bool ask_approval(const std::string& what) override { approvals.push_back(what); return approve; } + std::string home() const override { return home_; } + std::string cwd() const override { return cwd_; } + const sk::SkillSet& skills() const override { return skills_; } + void refresh_skills() override { skills_ = sk::SkillFiles::discover_skills(home_, cwd_); } + void audit_skill_install(const std::string&, const std::string&) override {} + std::string run_worker(const std::string&, int, int) override { return ""; } +}; + +static void set_box(FakeBox& box, std::optional<std::string> engine, const std::string& state = "running", const std::string& venv = "venv ready", const std::string& err = "") { + box.engine = engine; + box.state = state; + box.venv = venv; + box.ensure_error = err; +} + +static void sc_install_ops(const std::string& root) { + FakeBox box; + set_box(box, std::string("podman")); + ins::StdIo io; + ins::Installer inst(install_ctx(), box, io); + std::string home = tai_home(); + std::string uh = PyFs::home(); + out("-- binary"); + for (const std::string mode : {"install", "install", "upgrade", "reinstall", "uninstall", "uninstall", "install"}) out("binary " + mode + " => " + js(inst.op_binary(mode, home))); + std::string target = uh + "/.local/bin/tai.py"; + { + struct stat st; + stat(target.c_str(), &st); + struct stat st2; + stat(install_ctx().script_path.c_str(), &st2); + char buf[16]; + std::snprintf(buf, sizeof buf, "0o%o", static_cast<unsigned>(st.st_mode & 0777)); + out(std::string("binary mode ") + buf + " size_equal " + py_bool(st.st_size == st2.st_size)); + } + write(target, "VERSION = \"0.0.1\"\n"); + out("binary refresh => " + js(inst.op_binary("install", home))); + out("binary refresh upgrade => " + js(inst.op_binary("upgrade", home))); + auto opt_js = [](const std::optional<std::string>& v) { return v ? js(*v) : std::string("null"); }; + out("script_version " + opt_js(inst.script_version(target)) + " " + opt_js(inst.script_version(root + "/nope")) + " " + opt_js(inst.script_version(install_ctx().script_path))); + write(root + "/v1", "x\nVERSION = \"1.2.3\"\ny\n"); + write(root + "/v2", "VERSION = \"1.2\"\n VERSION = \"1.2.3\"\nVERSION = \"a.b.c\"\n"); + write(root + "/v3", "VERSION = \"4.5.6\""); + write(root + "/v4", "VERSION = \"4.5.6\" \n"); + { + std::string s = "["; + bool first = true; + for (const char* n : {"v1", "v2", "v3", "v4"}) { + s += std::string(first ? "" : ", ") + (inst.script_version(root + "/" + n) ? js(*inst.script_version(root + "/" + n)) : std::string("null")); + first = false; + } + out("script_version v1..v4 " + s + "]"); + } + out("-- venv"); + for (const std::string mode : {"install", "install", "reinstall", "uninstall", "uninstall", "install"}) out("venv " + mode + " => " + js(inst.op_venv(mode, home))); + { + auto vp = ins::Install::venv_python(home); + out("venv helpers " + js(ins::Install::venv_dir(home)) + " " + js(vp ? *vp : std::string()) + " " + py_bool(ins::Install::venv_available())); + } + out("-- services (no systemctl)"); + for (const std::string mode : {"install", "install", "reinstall", "upgrade", "uninstall", "uninstall"}) out("scheduler " + mode + " => " + js(inst.op_scheduler_service(mode, home))); + out("telegram without token => " + js(inst.op_telegram_service("install", home))); + out("service_exec " + js(inst.service_exec(home, "/x/tai.py", {"--a", "b"}))); + out("unit_dir " + js(ins::Systemd::unit_dir(home))); + { + auto st = ins::Systemd::service_state("tai-scheduler.service"); + Json a = Json::array(); + a.push(st.first).push(st.second); + out("service_state " + j(a)); + } + out("activate " + js(ins::Systemd::activate_service("tai-scheduler.service"))); + { + auto r = ins::Systemd::remove_service_unit(home, "tai-scheduler.service"); + Json a = Json::array(); + a.push(r.first).push(r.second); + out("remove " + j(a)); + } + out("-- container"); + for (const std::string mode : {"install", "upgrade", "reinstall", "uninstall"}) out("container " + mode + " => " + js(inst.op_container(mode, home))); + set_box(box, std::string("podman"), "stopped", "no venv (reinstall container to rebuild)"); + for (const std::string mode : {"install", "reinstall"}) out("container stopped " + mode + " => " + js(inst.op_container(mode, home))); + set_box(box, std::string("podman"), "running", "venv ready", "engine exploded"); + out("container failing => " + js(inst.op_container("install", home))); + set_box(box, std::nullopt); + for (const std::string mode : {"install", "uninstall"}) out("container noengine " + mode + " => " + js(inst.op_container(mode, home))); + set_box(box, std::string("podman")); + out("-- install_report"); + struct Rep { std::string action; std::vector<std::string> targets; }; + std::vector<Rep> reps = {{"install", {"skills", "bash-hook", "venv"}}, {"status", {"skills"}}, {"upgrade", ins::InstallConsts::INSTALL_TARGETS()}, + {"uninstall", {"skills", "bash-hook", "plexer-hook", "binary"}}}; + for (const auto& r : reps) out("report " + r.action + " " + PyStr::replace_all(PyStr::replace_all(j(arr_of(r.targets)), "\"", "'"), ",", ",") + " =>\n" + inst.install_report(r.action, r.targets, home)); + out("-- install_self"); + for (const std::string shell : {"/bin/bash", "/usr/bin/zsh", "/usr/bin/fish", "", "/bin/sh"}) { + setenv("SHELL", shell.c_str(), 1); + out("shell " + js(shell)); + std::cout.flush(); + inst.install_self(); + } + out("-- install tool"); + InstallApp app; + app.home_ = home; + app.cwd_ = root; + auto run_tool = [&](const Json& args) { + return inst.run_install(args, home, nullptr, [&](const std::string& w) { return app.ask_approval(w); }, [&] { app.refresh_skills(); }); + }; + std::vector<Json> tool_args = { + obj({{"action", "bogus"}}), obj({{"action", "status"}}), obj({{"action", "STATUS "}, {"targets", arr_of({"skills"})}}), obj({{"action", "install"}, {"targets", arr_of({"nope"})}}), + obj({{"action", "install"}, {"targets", Json::array()}}), obj({{"action", "install"}, {"targets", "skills"}}), obj({{"action", "install"}, {"targets", arr_of({"skills"})}}), + obj({{"action", "uninstall"}, {"targets", arr_of({"skills", "bash-hook"})}}), Json::object()}; + for (const auto& args : tool_args) { + app.skills_ = sk::SkillSet(); + std::string res = run_tool(args); + out("tool " + j(args) + " => " + js(res) + " | skills=" + j(arr_of(app.skills_.sorted_names()))); + } + app.approve = false; + out("tool denied => " + js(run_tool(obj({{"action", "install"}, {"targets", arr_of({"skills"})}})))); + out("approvals " + j(arr_of(app.approvals))); +} + +static void sc_install_noengine(const std::string&) { + FakeBox box; + set_box(box, std::nullopt); + ins::StdIo io; + ins::Installer inst(install_ctx(), box, io); + std::string home = tai_home(); + auto lines = inst.status_lines(home); + Json a = Json::array(); + for (std::size_t i = lines.size() - 3; i < lines.size(); ++i) a.push(lines[i]); + out(j(a)); + for (const std::string mode : {"install", "uninstall"}) out("container " + mode + " => " + js(inst.op_container(mode, home))); +} + +static void sc_vault_line(const std::string&) { + std::string home = tai_home(); + out("empty " + js(ins::Installer::vault_line(home))); + write(home + "/memory.db", ""); + out("kept " + js(ins::Installer::vault_line(home))); +} + +// ------------------------------------------------------------------ scheduler scenario +namespace sc = detz::sched; +using detz::misc::DbValue; +using detz::misc::SqliteDb; + +static Json db_json(const DbValue& v) { + if (v.is_null()) return Json(); + if (v.is_int()) return Json(v.as_int()); + return Json(v.as_text()); +} +static Json row_json(const detz::misc::DbRow& r) { + Json a = Json::array(); + for (const auto& v : r) a.push(db_json(v)); + return a; +} + +// Seal with passphrase disabled: lock is the identity, unlock follows Seal.unlock's error paths. +class DisabledSeal : public sc::SealPort { + public: + bool enabled() const override { return false; } + std::string lock(const std::string& plain) const override { return plain; } + std::string unlock(const std::string& sealed) const override { + if (!is_sealed(sealed)) throw sc::SealError("not sealed"); + std::size_t d1 = sealed.find('$'), d2 = sealed.find('$', d1 + 1), d3 = sealed.find('$', d2 + 1); + if (d2 == std::string::npos || d3 == std::string::npos) throw sc::SealError("sealed value is corrupt"); + throw sc::SealError("seal check failed"); + } + std::string safe_unlock(const std::string& sealed) const override { + try { return unlock(sealed); } catch (const sc::SealError&) { return "[sealed: decrypt failed]"; } + } +}; + +// The tag side of Store as the schedule code uses it (no plural folding needed for the test vocabulary) +class SqlTags : public sc::TagPort { + public: + SqlTags(SqliteDb& db) : db_(db) {} + void tag_item(const std::string& item, const std::vector<std::string>& tags, const std::string& profile) override { + for (const auto& t : detz::misc::TagNames::normalize_tags(tags)) db_.execute("INSERT OR IGNORE INTO tags (item, tag, profile) VALUES (?, ?, ?)", {item, t, profile}); + } + void set_item_tags(const std::string& item, const std::vector<std::string>& tags, const std::string& profile) override { + db_.execute("DELETE FROM tags WHERE item = ? AND profile = ?", {item, profile}); + tag_item(item, tags, profile); + } + std::vector<std::string> item_tags(const std::string& item, const std::string& profile) override { + std::vector<std::string> out_tags; + for (const auto& r : db_.query("SELECT tag FROM tags WHERE item = ? AND profile = ? ORDER BY tag", {item, profile})) out_tags.push_back(r[0].as_text()); + return out_tags; + } + private: + SqliteDb& db_; +}; + +// Registry with spawn_agent/poll_agent/running_agents semantics of tai.py and the Gate runner of the Python twin. +class FakeAgents : public sc::AgentPort { + public: + struct Rec { + std::string task, status = "running", result; + std::thread th; + }; + std::mutex m; + std::map<int, std::unique_ptr<Rec>> recs; + int next = 1; + bool report_zero = false; + std::mutex gm; + std::condition_variable gcv; + bool released = false; + + ~FakeAgents() override { + release(); + for (auto& kv : recs) { + if (kv.second->th.joinable()) kv.second->th.join(); + } + } + void release() { + std::lock_guard<std::mutex> lock(gm); + released = true; + gcv.notify_all(); + } + int running_agents() override { + if (report_zero) return 0; + std::lock_guard<std::mutex> lock(m); + int n = 0; + for (auto& kv : recs) n += kv.second->status == "running"; + return n; + } + int spawn_agent(const std::string& task, const std::string&, int64_t) override { + std::lock_guard<std::mutex> lock(m); + int live = 0; + for (auto& kv : recs) live += kv.second->status == "running"; + if (live >= 8) throw sc::AgentsFull("too many live agents (" + std::to_string(live) + " of 8)"); + int id = next++; + auto rec = std::make_unique<Rec>(); + rec->task = task; + Rec* raw = rec.get(); + recs[id] = std::move(rec); + raw->th = std::thread([this, raw, task] { + std::string result, status; + try { + if (task.rfind("slow:", 0) == 0) { + std::unique_lock<std::mutex> lock(gm); + gcv.wait_for(lock, std::chrono::seconds(30), [this] { return released; }); + result = "slow done " + task; + } else if (task.find("boom") != std::string::npos) { + throw std::runtime_error("boom"); + } else { + result = "ok:" + task; + } + status = result.find("[time limit reached]") != std::string::npos ? "timeout" : "done"; + } catch (const std::exception& e) { + result = std::string("worker failed: ") + PyStr::short_error(e.what()); + status = "error"; + } + std::lock_guard<std::mutex> lock(m); + raw->result = result; + raw->status = status; + }); + return id; + } + std::pair<std::string, std::string> poll_agent(int id, double wait) override { + Rec* rec; + { + std::lock_guard<std::mutex> lock(m); + auto it = recs.find(id); + if (it == recs.end()) return {"missing", "no agent " + std::to_string(id)}; + rec = it->second.get(); + } + auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(wait); + while (std::chrono::steady_clock::now() < deadline) { + { + std::lock_guard<std::mutex> lock(m); + if (rec->status != "running") break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + std::lock_guard<std::mutex> lock(m); + if (rec->status == "running") return {rec->status, "agent " + std::to_string(id) + " still running (0s elapsed)"}; + if (rec->status == "error") return {rec->status, rec->result}; + std::string text = rec->result; + std::size_t n = PyStr::len(text); + if (n > 6000 + 2000 + 50) text = PyStr::prefix(text, 6000) + "\n...[" + std::to_string(n - 6000 - 2000) + " chars truncated]...\n" + PyStr::suffix(text, 2000); + if (text.empty()) text = "(empty result)"; + return {rec->status, text}; + } +}; + +class SchedApp : public sc::ScheduleHost { + public: + std::vector<std::string> approvals; + bool approve = true; + sc::ScheduleVault* vault_ = nullptr; + bool ask_approval(const std::string& what) override { approvals.push_back(what); return approve; } + std::string active_profile() const override { return "t"; } + sc::ScheduleVault& vault() override { return *vault_; } +}; + +static void sched_rows(SqliteDb& db, const std::string& label) { + out("-- " + label); + for (const auto& r : db.query("SELECT id, name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, updated FROM schedules ORDER BY id")) out("row " + j(row_json(r))); + Json tags = Json::array(); + for (const auto& r : db.query("SELECT item, tag, profile FROM tags WHERE item LIKE 'sched:%' ORDER BY item, tag")) tags.push(row_json(r)); + out("tags " + j(tags)); +} + +static void sc_sched(const std::string&) { + std::string db_path = tai_home() + "/memory.db"; + SqliteDb db(db_path, 30, true); + DisabledSeal seal; + SqlTags tags(db); + FakeAgents agents; + sc::Scheduler scheduler(db_path, seal, agents); + sc::ScheduleVault vault(db, seal, tags, "t"); + auto quiesce = [&] { scheduler.join_watchers(); }; + auto item_json = [](const sc::ScheduleItem& it) { + Json o = Json::object(); + o.set("id", it.id).set("name", it.name).set("prompt", it.prompt).set("profile", it.profile).set("every", it.every).set("next_run", it.next_run).set("timeout", it.timeout); + o.set("status", it.status).set("last", it.last).set("result", it.result).set("created", it.created).set("tags", arr_of(it.tags)); + return o; + }; + const std::string NOW = "2026-10-25T12:00:00+00:00"; + out("== list and format"); + auto items = vault.list_schedules(); + for (const auto& it : items) out("item " + j(item_json(it))); + out("format:\n" + sc::Scheduler::format_schedule_lines(items, detz::misc::DateTime::fromisoformat(NOW))); + out("format empty: " + sc::Scheduler::format_schedule_lines({}, detz::misc::DateTime::fromisoformat(NOW))); + { + Json a = Json::array(); + for (const auto& it : vault.list_schedules("other")) a.push(item_json(it)); + out("other profile: " + j(a)); + } + out("== running leases"); + std::string nowiso = detz::misc::DateTime::now_utc().isoformat(); + struct Lease { std::string status, updated; int64_t timeout; }; + for (const auto& l : std::vector<Lease>{{"running", nowiso, 600}, {"running", "2000-01-01T00:00:00+00:00", 600}, {"done", "", 0}, {"running", "garbage", 10}, {"running", nowiso, 0}}) { + out("running " + l.status + " " + (l.updated.rfind("2026", 0) == 0 ? std::string("now") : l.updated) + " " + std::to_string(l.timeout) + " => " + + py_bool(sc::Scheduler::schedule_running(l.status, l.updated, l.timeout))); + } + out("== sealed rows (added now: Store refuses to open with them present and no passphrase)"); + vault.add_schedule("sealed", "tai1$AAAA$BBBB$CCCC", "t", 0, "2026-05-01T00:00:00+00:00", 300); + vault.add_schedule("sealed-corrupt", "tai1$bad", "t", 0, "2026-05-02T00:00:00+00:00", 300); + out("== tick 1"); + std::cout.flush(); + int fired = scheduler.scheduler_tick(); + out("fired " + std::to_string(fired)); + quiesce(); + sched_rows(db, "after tick 1"); + out("== tick 2"); + out("fired " + std::to_string(scheduler.scheduler_tick())); + quiesce(); + out("== cap phase"); + for (int k = 0; k < 9; ++k) vault.add_schedule("slow" + std::to_string(k), "slow:" + std::to_string(k), "t", 315360000, "2025-01-0" + std::to_string(k + 1) + "T00:00:00+00:00", 600); + out("fired " + std::to_string(scheduler.scheduler_tick())); + out("fired " + std::to_string(scheduler.scheduler_tick())); + out("running agents " + std::to_string(agents.running_agents())); + out("fired " + std::to_string(scheduler.scheduler_tick())); + sched_rows(db, "after cap ticks"); + out("== AgentsFull restore branch"); + vault.add_schedule("full-once", "slow:once", "t", 0, "2025-02-01T00:00:00+00:00", 600); + agents.report_zero = true; + out("fired " + std::to_string(scheduler.scheduler_tick())); + agents.report_zero = false; + sched_rows(db, "after AgentsFull"); + agents.release(); + quiesce(); + sched_rows(db, "after release"); + out("== schedule tool"); + SchedApp app; + app.vault_ = &vault; + sc::ScheduleTool tool(app); + auto dispatch_tool = [&](const std::string& name, const Json& args) { + try { + if (name == "schedule") return tool.run_schedule(args); + if (name == "unschedule") return tool.run_unschedule(args); + return tool.run_schedules(args); + } catch (const std::exception& e) { + return "error: " + PyStr::short_error(e.what()); + } + }; + std::vector<Json> calls = { + obj({{"prompt", ""}}), obj({{"prompt", "p"}}), obj({{"prompt", "p"}, {"at", "2099-01-01T10:00"}, {"every", 120}}), obj({{"prompt", "p"}, {"at", "nonsense"}}), + obj({{"prompt", "p"}, {"every", "abc"}}), obj({{"prompt", "p"}, {"every", 30}}), + obj({{"prompt", "tool one"}, {"at", "2099-01-01T10:00"}, {"name", std::string(100, 'N')}, {"timeout", 5}, {"tags", arr_of({"Alpha", "beta!", "", std::string(40, 'x')})}}), + obj({{"prompt", "tool two"}, {"every", 3600}, {"timeout", "abc"}, {"profile", "t"}}), obj({{"prompt", "tool three"}, {"every", "7200"}, {"timeout", 99999}}), + obj({{"prompt", "p"}, {"every", 60}, {"profile", "other"}}), obj({{"prompt", "p"}, {"every", 60}, {"profile", "bad profile!"}}), + obj({{"prompt", "p"}, {"every", 60}, {"tags", "notalist"}}), obj({{"prompt", "p"}, {"every", 60}, {"tags", [] { Json a = Json::array(); a.push(1).push(2); return a; }()}}), + obj({{"prompt", "tool four"}, {"every", 86400}, {"tags", Json::array()}}), obj({{"prompt", " padded "}, {"at", "2099-02-02T02:02:02Z"}, {"timeout", 0}}), + obj({{"prompt", "p"}, {"at", "9999-12-31T24:00"}}), obj({{"prompt", "p"}, {"at", "0001-01-01T00:00"}}), obj({{"prompt", "p"}, {"every", 60.5}}), obj({{"prompt", "p"}, {"every", true}}), + }; + for (const auto& args : calls) out("schedule " + j(args) + " => " + js(dispatch_tool("schedule", args))); + app.approve = false; + out("schedule denied => " + js(dispatch_tool("schedule", obj({{"prompt", "denied"}, {"every", 60}})))); + app.approve = true; + out("schedules => " + js(dispatch_tool("schedules", Json::object()))); + for (const auto& args : std::vector<Json>{obj({{"id", 999}}), obj({{"id", "abc"}}), obj({{"id", 0}}), Json::object(), obj({{"id", "13"}}), obj({{"id", 14.0}})}) { + out("unschedule " + j(args) + " => " + js(dispatch_tool("unschedule", args))); + } + app.approve = false; + out("unschedule denied => " + js(dispatch_tool("unschedule", obj({{"id", 15}})))); + app.approve = true; + out("approvals " + j(arr_of(app.approvals))); + sched_rows(db, "final"); + out("remove nonexistent " + py_bool(vault.remove_schedule(99999))); +} + +// ------------------------------------------------------------------ rag scenario +namespace rg = detz::rag; + +static Json hit_json(const rg::FtsHit& h) { + Json o = Json::object(); + o.set("item", db_json(h.item)).set("kind", h.kind).set("sub", h.sub).set("title", h.title).set("snippet", h.snippet).set("rank", h.rank); + return o; +} +static Json ids_json(const std::vector<DbValue>& v) { + Json a = Json::array(); + for (const auto& x : v) a.push(db_json(x)); + return a; +} +static Json mean_json(const rg::EvalMean& m) { + Json o = Json::object(); + o.set("hit", m.hit).set("mrr", m.mrr).set("ndcg", m.ndcg).set("cp", m.cp); + return o; +} +static Json result_json(const rg::EvalResult& r) { + Json o = Json::object(); + o.set("k", r.k).set("goldens", r.goldens).set("overall", mean_json(r.overall)); + Json cats = Json::object(); + for (const auto& c : r.by_category) { + Json e = Json::object(); + e.set("goldens", c.second.goldens).set("mean", mean_json(c.second.mean)); + cats.set(c.first, e); + } + o.set("by_category", cats); + Json misses = Json::array(); + for (const auto& m : r.misses) { + Json e = Json::object(); + e.set("query", m.query).set("category", m.category); + misses.push(e); + } + o.set("misses", misses); + return o; +} +static Json pairs_json(const rg::ImplicitPairs& p) { + Json o = Json::object(); + o.set("searches", p.searches).set("read_follow", p.read_follow); + Json pairs = Json::array(); + for (const auto& x : p.pairs) { + Json e = Json::object(); + e.set("query", x.query).set("read", x.read).set("rank", x.rank ? Json(*x.rank) : Json()).set("shown", x.shown); + pairs.push(e); + } + o.set("pairs", pairs).set("zero_hit", arr_of(p.zero_hit)).set("zero_hit_count", p.zero_hit_count); + return o; +} + +class RagApp : public rg::RagHost { + public: + RagApp(SqliteDb& db, rg::FtsSearch& fts, rg::ChunkStore& chunks) : db_(db), fts_(fts), chunks_(chunks) {} + std::string active_profile() const override { return "t"; } + bool sealed() const override { return false; } + std::vector<rg::FtsHit> fts_search(const std::string& q, const std::vector<std::string>& kinds, const std::string& profile, int limit) override { return fts_.fts_search(q, kinds, profile, limit); } + std::vector<rg::EventRow> events_like(const std::string&, const std::string&, int) override { return {}; } + void log_event(const std::string& profile, const std::string& role, const std::string& kind, const std::string& text) override { + std::string value = PyStr::prefix(text, 2000); + db_.execute("INSERT INTO events (profile, ts, role, kind, text, tags) VALUES (?, ?, ?, ?, ?, ?)", {profile, detz::sched::ScheduleTime::now_iso(), role, kind, value, " " + role + " " + kind + " "}); + } + std::map<int64_t, rg::ChunkRow> get_chunks_by_ids(const std::vector<int64_t>& ids, const std::string& profile) override { return chunks_.get_chunks_by_ids(ids, profile); } + std::vector<rg::GraphEdge> edges_for(const std::string& item, const std::string& profile) override { + std::vector<rg::GraphEdge> out_edges; + for (const auto& r : db_.query("SELECT src, dst, relation FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {profile, item, item})) { + out_edges.push_back({r[0].as_text() == item ? r[1].as_text() : r[0].as_text(), r[2].as_text()}); + } + return out_edges; + } + std::string node_title(const std::string& node, const std::string& profile) override { + std::size_t colon = node.find(':'); + if (colon == std::string::npos) return node; + std::string kind = node.substr(0, colon), key = node.substr(colon + 1); + if (kind == "mem") { + auto rows = db_.query("SELECT title FROM records WHERE profile = ? AND id = ?", {profile, node}); + if (rows.empty()) return node; + std::string t = rows[0][0].as_text(); + return t.empty() ? node : t; + } + if (kind == "secret") return "secret:" + key; + auto rows = db_.query("SELECT name, prompt FROM schedules WHERE profile = ? AND id = ?", {profile, std::stoll(key)}); + if (rows.empty()) return node; + std::string name = rows[0][0].as_text(); + return name.empty() ? "schedule #" + key : name; + } + private: + SqliteDb& db_; + rg::FtsSearch& fts_; + rg::ChunkStore& chunks_; +}; + +static void sc_rag(const std::string&) { + SqliteDb db(tai_home() + "/memory.db", 30, true); + rg::FtsSearch fts(db, "t"); + rg::ChunkStore chunks(db, "t"); + out("== fts_search"); + std::vector<std::string> queries = {"alpha", "alpha bravo", "zebra", "caf\xc3\xa9", "r\xc3\xa9sum\xc3\xa9 na\xc3\xafve", "snake_case", "numbers123", "nothingmatchesthis", "", "!!!", "retrieval index", "wh", + "ALPHA", std::string(50, 'x'), "unique zebra jumps", "message", "file", "ro", "v2beta kernel"}; + std::vector<std::vector<std::string>> kind_sets = {{}, {"record"}, {"chunk"}, {"audit"}, {"event"}, {"record", "chunk"}}; + for (const auto& q : queries) { + for (const auto& kinds : kind_sets) { + for (int limit : {10, 3}) { + Json a = Json::array(); + for (const auto& h : fts.fts_search(q, kinds, "t", limit)) a.push(hit_json(h)); + out("search " + js(q) + " " + j(arr_of(kinds)) + " " + std::to_string(limit) + " => " + j(a)); + } + } + } + { + Json a = Json::array(); + for (const auto& h : fts.fts_search("alpha", {"record"}, "other", 10)) a.push(hit_json(h)); + out("other profile " + j(a)); + Json l = Json::array(); + for (int n : {0, -5, 1, 100, 500}) l.push(static_cast<int64_t>(fts.fts_search("alpha", {}, "t", n).size())); + out("limits " + j(l)); + } + out("== eval"); + out("tokens " + j(arr_of(rg::RetrievalEval::tokens("The quick Brown fox jumped_over 123456 abcdefg ABCDEFG caf\xc3\xa9s r\xc3\xa9sum\xc3\xa9s")))); + out("tokens4 " + j(arr_of(rg::RetrievalEval::tokens("The quick Brown fox", 4)))); + auto goldens = rg::RetrievalEval::goldens_from_vault(db, "t"); + out("goldens " + std::to_string(goldens.size())); + for (const auto& g : goldens) { + Json o = Json::object(); + o.set("query", g.query).set("expected", ids_json(g.expected)).set("category", g.category).set("source", db_json(g.source)); + out("golden " + j(o)); + } + auto result = rg::RetrievalEval::score(fts, goldens, "t"); + out("score " + j(result_json(result))); + auto pairs = rg::RetrievalEval::implicit_pairs(db, "t", false); + out("pairs " + j(pairs_json(pairs))); + out("report:\n" + rg::RetrievalEval::format_report(result, pairs)); + out("empty report:\n" + rg::RetrievalEval::format_report(rg::RetrievalEval::score(fts, {}, "t"), rg::RetrievalEval::implicit_pairs(db, "nobody", false))); + out("small k " + j(result_json(rg::RetrievalEval::score(fts, std::vector<rg::Golden>(goldens.begin(), goldens.begin() + 12), "t", {"record"}, 1, 2)))); + { + auto ro = [](std::optional<int> r) { return r ? Json(*r) : Json(); }; + out("rank_of " + j(ro(rg::RetrievalEval::rank_of(DbValue("b"), {DbValue("a"), DbValue("b")}))) + " " + j(ro(rg::RetrievalEval::rank_of(DbValue("z"), {DbValue("a")}))) + " " + + j(ro(rg::RetrievalEval::best_rank({DbValue("z"), DbValue("a")}, {DbValue("q"), DbValue("a"), DbValue("z")})))); + Json m = Json::array(); + m.push(rg::RetrievalEval::hit_at_k(3)).push(rg::RetrievalEval::hit_at_k(6)).push(rg::RetrievalEval::hit_at_k(std::nullopt)).push(rg::RetrievalEval::reciprocal_rank(4)); + m.push(rg::RetrievalEval::ndcg_at_k(3)).push(rg::RetrievalEval::ndcg_at_k(9)); + m.push(rg::RetrievalEval::context_precision({DbValue("a"), DbValue("c")}, {DbValue("a"), DbValue("b"), DbValue("c"), DbValue("d")})); + m.push(rg::RetrievalEval::context_precision({}, {DbValue("a")})).push(rg::RetrievalEval::context_precision({DbValue("x")}, {DbValue("a")})); + out("metrics " + j(m)); + } + for (const std::string& text : std::vector<std::string>{"q=alpha\thits=record:m1,m2,,chunk:4", "nothing", "q=\thits=a", "a=b=c\tq= spaced \thits= x ,y ", "q=a\tq=b\thits=record:z"}) { + auto parsed = rg::RetrievalEval::parse_search_event(text); + Json v; + if (parsed) { + v = Json::array(); + v.push(parsed->first).push(arr_of(parsed->second)); + } + out("parse " + js(text) + " => " + j(v)); + } + out("== chunks table"); + std::string rid = db.query("SELECT id FROM records WHERE profile = 't' ORDER BY id LIMIT 1")[0][0].as_text(); + chunks.add_chunks("record", rid, {"one", "two", "three"}, "t"); + Json rows_j = Json::array(); + std::vector<int64_t> ids; + for (const auto& r : db.query("SELECT id, parent_kind, parent_id, idx, text, profile FROM chunks WHERE parent_id = ? ORDER BY idx", {rid})) { + rows_j.push(row_json(r)); + ids.push_back(r[0].as_int()); + } + out("chunks " + j(rows_j)); + auto chunk_obj = [](const std::map<int64_t, rg::ChunkRow>& m) { + Json o = Json::object(); + for (const auto& kv : m) { + Json c = Json::object(); + c.set("parent_kind", kv.second.parent_kind).set("parent_id", kv.second.parent_id).set("idx", kv.second.idx).set("text", kv.second.text); + o.set(std::to_string(kv.first), c); + } + return o; + }; + std::vector<int64_t> ask = ids; + ask.push_back(999999); + out("by ids " + j(chunk_obj(chunks.get_chunks_by_ids(ask, "t")))); + out("by ids empty " + j(chunk_obj(chunks.get_chunks_by_ids({}, "t"))) + " other " + j(chunk_obj(chunks.get_chunks_by_ids(ids, "other")))); + chunks.add_chunks("record", rid, {}, "t"); + out("after clear " + std::to_string(db.query("SELECT COUNT(*) FROM chunks WHERE parent_id = ?", {rid})[0][0].as_int())); + out("== search tool"); + RagApp host(db, fts, chunks); + rg::RagTool tool(host); + std::vector<Json> calls = { + obj({{"query", ""}}), obj({{"query", "alpha"}}), obj({{"query", "alpha"}, {"limit", 3}}), obj({{"query", "alpha"}, {"limit", "2"}}), obj({{"query", "alpha"}, {"limit", "x"}}), + obj({{"query", "alpha"}, {"limit", 0}}), obj({{"query", "alpha"}, {"limit", -2}}), obj({{"query", "retrieval"}, {"kinds", arr_of({"chunk"})}}), + obj({{"query", "alpha"}, {"kinds", arr_of({"record"})}, {"expand", true}, {"limit", 20}}), obj({{"query", "alpha"}, {"kinds", arr_of({"bogus"})}}), obj({{"query", "alpha"}, {"kinds", Json::array()}}), + obj({{"query", "alpha"}, {"kinds", "record"}}), obj({{"query", "file"}, {"kinds", arr_of({"audit"})}}), obj({{"query", "message"}, {"kinds", arr_of({"event"})}}), + obj({{"query", "zebra"}, {"kinds", arr_of({"record", "chunk", "audit"})}}), obj({{"query", "nothingmatchesthis"}}), obj({{"query", std::string(200, 'x')}}), obj({{"query", " spaced "}}), + obj({{"query", "caf\xc3\xa9 r\xc3\xa9sum\xc3\xa9"}, {"limit", 50}}), + }; + for (const auto& args : calls) { + std::string res; + try { + res = tool.run_search(args); + } catch (const std::exception& e) { + res = "error: " + PyStr::short_error(e.what()); + } + out("search " + j(args) + " => " + js(res)); + } + Json logged = Json::array(); + for (const auto& r : db.query("SELECT profile, role, kind, text FROM events WHERE kind = 'search-query' ORDER BY id DESC LIMIT 25")) logged.push(row_json(r)); + out("logged " + j(logged)); +} + +// ------------------------------------------------------------------ telegram scenarios +namespace tg = detz::telegram; + +class LogRec : public detz::misc::LogSink { + public: + void event(const std::string& level, const std::string& component, const std::string& event, const std::string& message, const std::string& profile, + const std::string& bot, const std::string& chat, const Json& meta) override { + Json a = Json::array(); + a.push(level).push(component).push(event).push(message).push(profile).push(bot).push(chat).push(meta); + out("LOG " + j(a)); + } + void set_context(const std::string& key, const std::string& value) override { out("CTX set " + key + "=" + value); } + void clear_context(const std::string& key) override { out("CTX pop " + key); } +}; + +class TgAgent : public tg::TelegramAgent { + public: + std::string home_; + std::string profile() const override { return "t"; } + std::string bot() const override { return "main"; } + std::string home() const override { return home_; } + void reset_history() override { out("AGENT reset_history"); } + std::string run_turn(const std::string& text) override { + out("AGENT run_turn " + js(text) + " capture=True"); + if (text.find("explode") != std::string::npos) throw std::runtime_error("agent exploded"); + if (text == "long") return std::string(9000, 'L'); + if (text == "empty reply") return ""; + return "echo:" + text; + } + void start_scheduler() override { out("start_scheduler called"); } +}; + +class TgBox : public tg::BoxPort { + public: + std::string ensure_box() override { + out("BOX ensure_box"); + if (std::getenv("MISC_BOX_FAIL")) throw detz::misc::BackendError("box down"); + return "eng"; + } + std::string box_transcribe(const std::string& engine, const std::string& audio) override { + // Python prints the audio decoded as latin-1: each byte becomes one code point + std::string latin; + for (unsigned char c : audio) { + if (c < 0x80) latin += static_cast<char>(c); + else { latin += static_cast<char>(0xC0 | (c >> 6)); latin += static_cast<char>(0x80 | (c & 0x3F)); } + } + out("BOX transcribe " + engine + " " + js(latin)); + if (audio == "AUDIO") return "voice says hi"; + return "error: model failed"; + } +}; + +struct StopLoop {}; + +static std::string tg_api() { return std::getenv("MISC_TG_API"); } +static std::string tg_file() { return std::getenv("MISC_TG_FILE"); } + +static void sc_telegram(const std::string& root) { + LogRec log; + TgAgent agent; + TgBox box; + std::string home = tai_home(); + agent.home_ = home; + tg::TelegramBot bot(agent, nullptr, &box, log); + bot.set_api(tg_api(), tg_file()); + out("== token"); + std::string env_path = tg::TelegramBot::telegram_env_path(home); + out("env path " + js(env_path)); + unsetenv("TELEGRAM_BOT_TOKEN"); + out("no file => " + js(tg::TelegramBot::load_telegram_token(home))); + std::vector<std::pair<std::string, std::string>> bodies = { + {"plain", "TELEGRAM_BOT_TOKEN=abc123\n"}, {"quoted", "TELEGRAM_BOT_TOKEN='abc'\n"}, {"dquoted", "TELEGRAM_BOT_TOKEN=\"x y\"\n"}, {"spaces", "TELEGRAM_BOT_TOKEN= spaced \n"}, + {"other first", "OTHER=1\nTELEGRAM_BOT_TOKEN=second\nTELEGRAM_BOT_TOKEN=third\n"}, {"none", "OTHER=1\n"}, {"empty", ""}, {"crlf", "TELEGRAM_BOT_TOKEN=win\r\n"}, + {"noeq", "TELEGRAM_BOT_TOKEN\n"}, {"eq in value", "TELEGRAM_BOT_TOKEN=a=b=c\n"}, {"quotes only", "TELEGRAM_BOT_TOKEN=''\n"}, {"indented", " TELEGRAM_BOT_TOKEN=nope\n"}}; + for (const auto& b : bodies) { + write(env_path, b.second); + out("file " + b.first + " => " + js(tg::TelegramBot::load_telegram_token(home))); + } + setenv("TELEGRAM_BOT_TOKEN", "fromenv", 1); + out("env wins => " + js(tg::TelegramBot::load_telegram_token(home))); + setenv("TELEGRAM_BOT_TOKEN", "", 1); + out("empty env falls back => " + js(tg::TelegramBot::load_telegram_token(home))); + unsetenv("TELEGRAM_BOT_TOKEN"); + PyFs::remove(env_path); + out("== call / send"); + tg::Telegram good("good", nullptr, tg_api(), tg_file()); + out("getMe " + j(good.call("getMe", Json::object()))); + for (const std::pair<std::string, std::string>& c : std::vector<std::pair<std::string, std::string>>{{"bad token", "bad"}, {"not json", "nojson"}}) { + try { + tg::Telegram t(c.second, nullptr, tg_api(), tg_file()); + out(c.first + " => " + j(t.call("getMe", Json::object()))); + } catch (const std::exception& e) { + out(c.first + " => raised " + js(PyStr::short_error(e.what()))); + } + } + std::string e1(4001, 'x'); + std::string em; + for (int i = 0; i < 4001; ++i) em += "\xc3\xa9"; + std::string smile; + for (int i = 0; i < 4100; ++i) smile += "\xf0\x9f\x98\x80"; + std::string lines; + for (int i = 0; i < 2000; ++i) lines += "line\n"; + for (const std::string& text : std::vector<std::string>{"", "short", std::string(4000, 'S'), std::string(4001, 'S'), std::string(8000, 'S'), std::string(8001, 'S'), em, smile, lines}) { + tg::TelegramBot::telegram_send(good, Json(42), text); + } + tg::TelegramBot::telegram_send(good, Json("chat-as-string"), "hi"); + out("== updates"); + auto chat = [](const Json& id) { return obj({{"id", id}}); }; + auto msg = [&](const Json& id, std::initializer_list<std::pair<const char*, Json>> extra) { + Json m = Json::object(); + m.set("chat", chat(id)); + for (const auto& p : extra) m.set(p.first, p.second); + return m; + }; + std::vector<Json> updates; + auto upd = [&](int id, const Json& message) { updates.push_back(obj({{"update_id", id}, {"message", message}})); }; + upd(1, msg(111, {{"text", "/start"}})); + upd(2, msg(111, {{"text", "/new"}})); + upd(3, msg(111, {{"text", "hello there"}})); + upd(4, msg(111, {{"text", ""}})); + upd(5, msg(111, {{"text", " "}})); + upd(6, msg(0, {{"text", "zero chat"}})); + upd(7, obj({{"text", "no chat"}})); + updates.push_back(obj({{"update_id", 8}})); + upd(9, msg(111, {})); + upd(10, msg(111, {{"text", "explode please"}})); + upd(11, msg(111, {{"text", "long"}})); + upd(12, msg(111, {{"text", "empty reply"}})); + upd(13, msg(222, {{"voice", obj({{"file_id", "f1"}})}})); + upd(14, msg(222, {{"audio", obj({{"file_id", "f1"}})}})); + upd(15, msg(222, {{"voice", obj({{"file_id", "nofile"}})}})); + upd(16, msg(222, {{"voice", obj({{"file_id", "fbad"}})}})); + upd(17, msg(222, {{"voice", obj({{"file_id", "fmodel"}})}})); + upd(18, msg(222, {{"voice", Json::object()}, {"text", "text with empty voice"}})); + upd(19, msg(222, {{"voice", obj({{"file_id", "fmissingfile"}})}})); + upd(20, msg(Json("str-chat"), {{"text", "string chat id"}})); + upd(21, msg(111, {{"text", "unicode caf\xc3\xa9 \xf0\x9f\x98\x80"}})); + for (const auto& update : updates) { + out("-- update " + j(update)); + try { + bot.handle_telegram_update(good, update); + } catch (const std::exception& e) { + out("raised " + js(PyStr::short_error(e.what()))); + } + } + setenv("MISC_BOX_FAIL", "1", 1); + out("-- box failing"); + try { + bot.handle_telegram_update(good, obj({{"update_id", 30}, {"message", msg(5, {{"voice", obj({{"file_id", "f1"}})}})}})); + } catch (const std::exception& e) { + out("raised " + js(PyStr::short_error(e.what()))); + } + unsetenv("MISC_BOX_FAIL"); + out("== run_telegram_bot"); + struct Run { const char* label; const char* token; }; + for (const Run& r : {Run{"missing", nullptr}, Run{"bad auth", "bad"}, Run{"not json", "nojson"}, Run{"no username", "noname"}, Run{"good", "good"}}) { + unsetenv("TELEGRAM_BOT_TOKEN"); + if (r.token) setenv("TELEGRAM_BOT_TOKEN", r.token, 1); + out(std::string("-- run ") + r.label); + std::cout.flush(); + try { + int res = bot.run_telegram_bot(nullptr, [](int seconds) { + out("sleep " + std::to_string(seconds)); + throw StopLoop(); + }); + out("returned " + std::to_string(res)); + } catch (const StopLoop&) { + out("stopped by sleep"); + } + std::cout.flush(); + } + (void)root; +} + +static void sc_telegram_install(const std::string& root) { + LogRec log; + TgAgent agent; + TgBox box; + FakeBox fbox; + set_box(fbox, std::string("podman")); + ins::StdIo io; + ins::InstallContext ctx = install_ctx(); + ctx.telegram_api = tg_api(); + ins::Installer inst(ctx, fbox, io); + std::string home = tai_home(); + struct T { const char* label; const char* token; }; + for (const T& t : {T{"good", "good"}, T{"bad", "bad"}, T{"nojson", "nojson"}, T{"noname", "noname"}}) { + setenv("TELEGRAM_BOT_TOKEN", t.token, 1); + out(std::string("-- install_telegram ") + t.label); + std::cout.flush(); + int res = inst.install_telegram(); + out("returned " + std::to_string(res)); + out("unit exists " + py_bool(PyFs::exists(PyFs::home() + "/.config/systemd/user/tai-telegram.service")) + " env exists " + + py_bool(PyFs::exists(tg::TelegramBot::telegram_env_path(home)))); + } + unsetenv("TELEGRAM_BOT_TOKEN"); + out("-- install_telegram without token (stdin closed)"); + std::cout.flush(); + out("returned " + std::to_string(inst.install_telegram())); + out("-- install_telegram box failing"); + set_box(fbox, std::string("podman"), "running", "venv ready", "no engine here"); + setenv("TELEGRAM_BOT_TOKEN", "good", 1); + out("returned " + std::to_string(inst.install_telegram())); + set_box(fbox, std::string("podman")); + out("-- op_telegram_service"); + for (const char* token : {"good", "bad", "nojson", "noname"}) { + setenv("TELEGRAM_BOT_TOKEN", token, 1); + for (const std::string mode : {"install", "install", "reinstall", "upgrade", "uninstall", "uninstall"}) { + out(std::string("service ") + token + " " + mode + " => " + js(inst.op_telegram_service(mode, home))); + } + } + unsetenv("TELEGRAM_BOT_TOKEN"); + write(tg::TelegramBot::telegram_env_path(home), "TELEGRAM_BOT_TOKEN=good\n"); + out("service from env file => " + js(inst.op_telegram_service("install", home))); + out("-- uninstall_telegram"); + out("returned " + std::to_string(inst.uninstall_telegram())); + set_box(fbox, std::nullopt); + out("returned noengine " + std::to_string(inst.uninstall_telegram())); + (void)root; +} + +// ------------------------------------------------------------------ release / audit scenarios +using detz::install::SelfRelease; + +class NoRecords : public ins::FileRecordPort { + public: + void upsert_file_record(const std::string&, const std::string&, const std::optional<int64_t>&) override {} // records table: store module +}; + +class RelHost : public ins::ReleaseHost { + public: + std::vector<std::string> approvals; + bool approve = true; + std::string script, home_; + ins::AuditVault* vault_ = nullptr; + bool ask_approval(const std::string& w) override { approvals.push_back(w); return approve; } + std::string home() const override { return home_; } + std::string audit_actor() const override { return "main"; } + ins::AuditPort& store() override { return *vault_; } + std::string script_file() const override { return script; } +}; + +static std::string copy_file_bytes(const std::string& from, const std::string& to) { + write(to, PyFs::read_bytes(from)); + return to; +} + +static void sc_release(const std::string& root) { + SqliteDb db(tai_home() + "/memory.db", 30, true); + ins::AuditVault vault(db, "t"); + NoRecords records; + std::string script = copy_file_bytes(install_ctx().script_path, root + "/script.py"); + std::string folder = root + "/backups"; + out("== versions"); + struct V { const char* cur; const char* part; }; + for (const V& v : {V{"1.2.3", "patch"}, V{"1.2.3", "minor"}, V{"1.2.3", "major"}, V{"9.9.9", "patch"}, V{"1.2.3", "bogus"}, V{"1.2", "patch"}, V{"a.b.c", "patch"}, V{"1.2.3.4", "patch"}}) { + try { + out(std::string("next ") + v.cur + " " + v.part + " => " + SelfRelease::next_version(v.cur, v.part)); + } catch (const std::exception& e) { + out(std::string("next ") + v.cur + " " + v.part + " => raised " + js(e.what())); + } + } + out("== bump"); + for (const char* part : {"patch", "minor", "major", "patch"}) { + auto r = SelfRelease::release_bump(script, part); + Json a = Json::array(); + a.push(r.first).push(r.second); + out(std::string("bump ") + part + " => " + j(a)); + } + { + std::string text = PyFs::read_text(script); + std::vector<std::string> vl; + for (const auto& l : PyStr::splitlines(text)) { + if (l.rfind("VERSION = ", 0) == 0) vl.push_back(l); + } + out("version line " + j(arr_of(vl))); + } + write(root + "/nov.py", "print('x')\n"); + for (const std::string& f : {root + "/nov.py", root + "/missing.py"}) { + try { + SelfRelease::release_bump(f, "patch"); + } catch (const std::exception& e) { + out("bump error " + js(replace_root(e.what(), root))); + } + } + write(root + "/crlf.py", "a\r\nVERSION = \"1.0.0\"\r\nb\r\n"); + { + auto r = SelfRelease::release_bump(root + "/crlf.py", "patch"); + Json a = Json::array(); + a.push(r.first).push(r.second); + out("crlf bump " + j(a) + " " + js(PyFs::read_bytes(root + "/crlf.py"))); + } + out("== snapshots and pruning"); + auto snap = SelfRelease::snapshot_self(script, folder, "9.9.9"); + out("snapshot " + js(PyFs::basename(snap.first)) + " " + snap.second); + out("has digest " + py_bool(SelfRelease::backup_has_digest(folder, snap.second)) + " " + py_bool(SelfRelease::backup_has_digest(folder, "deadbeef"))); + out("has digest nofolder " + py_bool(SelfRelease::backup_has_digest(root + "/nofolder", snap.second))); + for (int i = 0; i < 14; ++i) { + char name[128]; + std::snprintf(name, sizeof name, "/tai-0.0.%d-20200101T000000Z-%08d.py", i, i); + write(folder + name, "x"); + struct timespec ts[2] = {{1000 + i, 0}, {1000 + i, 0}}; + utimensat(AT_FDCWD, (folder + name).c_str(), ts, 0); + } + write(folder + "/notes.txt", "keep"); + out("pruned " + std::to_string(SelfRelease::prune_backups(folder))); + out("left " + PyStr::replace_all(j(arr_of(PyFs::listdir_sorted(folder))), PyFs::basename(snap.first), "<SNAP>")); + out("prune missing " + std::to_string(SelfRelease::prune_backups(root + "/nofolder"))); + out("== ensure_self_backup"); + PyFs::rmtree(folder); + auto first = SelfRelease::ensure_self_backup(tai_home(), &vault, &records, script, "1.44.2"); + out("ensure first " + (first ? js(PyFs::basename(*first)) : std::string("null"))); + auto second = SelfRelease::ensure_self_backup(tai_home(), &vault, &records, script, "1.44.2"); + out("ensure second " + (second ? js(PyFs::basename(*second)) : std::string("null"))); + { + Json rows = Json::array(); + for (const auto& r : db.query("SELECT actor, action, path, message, old_size, new_size, tags FROM audit ORDER BY id")) rows.push(row_json(r)); + out("audit " + j(rows)); + } + out("== do_release"); + try { + SelfRelease::do_release(vault, script, folder, "patch", " "); + } catch (const std::exception& e) { + out("blank message " + js(e.what())); + } + { + auto r = SelfRelease::do_release(vault, script, folder, "minor", " ship it ", "tester"); + Json a = Json::array(); + a.push(r.old_version).push(r.new_version).push(PyFs::basename(r.dest)).push(r.row_id); + out("do_release " + j(a)); + } + out("== release tool"); + RelHost host; + host.script = script; + host.home_ = tai_home(); + host.vault_ = &vault; + ins::ReleaseTool tool(host); + auto rel = [&](const Json& args) { + try { + return tool.run_release(args); + } catch (const std::exception& e) { + return "error: " + PyStr::short_error(e.what()); + } + }; + for (const Json& args : std::vector<Json>{obj({{"part", ""}, {"message", "m"}}), obj({{"part", "patch"}, {"message", ""}}), obj({{"part", "huge"}, {"message", "m"}}), + obj({{"part", "Patch"}, {"message", "tool release"}}), obj({{"part", "major"}, {"message", std::string(300, 'm')}})}) { + out("release " + j(args) + " => " + js(replace_root(rel(args), root))); + } + host.approve = false; + out("release denied => " + js(rel(obj({{"part", "patch"}, {"message", "no"}})))); + host.approve = true; + write(script, "nothing here\n"); + out("release no version => " + js(rel(obj({{"part", "patch"}, {"message", "m"}})))); + out("approvals " + j(arr_of(host.approvals))); + out("== capped helpers"); + { + auto c0 = SelfRelease::capped_text(std::nullopt); + auto c1 = SelfRelease::capped_text(std::string("abc")); + std::string e; + for (int i = 0; i < 20001; ++i) e += "\xc3\xa9"; + auto c2 = SelfRelease::capped_text(e); + auto c3 = SelfRelease::capped_text(std::string(20001, 'x')); + auto pair_json = [](const std::pair<std::optional<std::string>, std::optional<int64_t>>& p) { + Json a = Json::array(); + a.push(p.first ? Json(*p.first) : Json()).push(p.second ? Json(*p.second) : Json()); + return a; + }; + Json all = Json::array(); + all.push(pair_json(c0)).push(pair_json(c1)).push(Json(*c2.second)); + all.push(Json(PyStr::suffix(*c3.first, 60))).push(SelfRelease::trunc_marker(5)); + out("capped " + j(all)); + } + std::string big = "line\r\nsecond\rthird\n"; + for (int i = 0; i < 30000; ++i) big += "\xc3\xa9"; + write(root + "/big.txt", big); + for (int cap : {20000, 50, 100000}) { + auto r = SelfRelease::read_capped(root + "/big.txt", static_cast<std::size_t>(cap)); + out("read_capped " + std::to_string(cap) + " => " + (r.text ? js(PyStr::prefix(*r.text, 40)) : std::string("null")) + " " + (r.size ? std::to_string(*r.size) : std::string("null"))); + } + write(root + "/bad.bin", std::string("ok \xff\xfe bad \xc3 end")); + { + auto r = SelfRelease::read_capped(root + "/bad.bin"); + Json a = Json::array(); + a.push(r.text ? Json(*r.text) : Json()).push(r.size ? Json(*r.size) : Json()); + out("read_capped invalid " + j(a)); + } + { + auto r = SelfRelease::read_capped(root + "/missing"); + Json a = Json::array(); + a.push(Json()).push(Json()); + out("read_capped missing " + j(a)); + (void)r; + } +} + +class AuditApp : public ins::AuditHost { + public: + std::vector<std::string> approvals; + bool approve = true; + ins::AuditVault* vault_ = nullptr; + rg::FtsSearch* fts_ = nullptr; + bool ask_approval(const std::string& w) override { approvals.push_back(w); return approve; } + std::string active_profile() const override { return "t"; } + ins::AuditVault& vault() override { return *vault_; } + rg::FtsSearch& fts() override { return *fts_; } + void read_files_add(const std::string&) override {} + void audit_file(const std::string& action, const std::string& path, const std::optional<std::string>& o, const std::optional<std::string>& n, const std::string& message) override { + vault_->audit_event_full("main", action, path, message, o, n, {"file", action}, std::nullopt, std::nullopt, "t"); + } + void upsert_file_record(const std::string&, const std::string&, const std::string&) override {} // records table: store module + void note_diff(const std::string&, const std::optional<std::string>&, const std::optional<std::string>&) override {} + std::string box_restore(const std::string&, const std::string&, const std::string&) override { return "box restore unsupported in this test"; } +}; + +static void sc_audit(const std::string& root) { + SqliteDb db(tai_home() + "/memory.db", 30, true); + ins::AuditVault vault(db, "t"); + rg::FtsSearch fts(db, "t"); + AuditApp app; + app.vault_ = &vault; + app.fts_ = &fts; + ins::AuditTool tool(app); + auto call = [&](const std::string& name, const Json& args) { + std::string res; + try { + res = name == "audit" ? tool.run_audit(args) : name == "tags" ? tool.run_tags(args) : tool.run_restore(args); + } catch (const std::exception& e) { + res = "error: " + PyStr::short_error(e.what()); + } + out(name + " " + j(args) + " => " + js(replace_root(res, root))); + }; + out("== audit"); + for (const Json& a : std::vector<Json>{Json::object(), obj({{"limit", 3}}), obj({{"path", "area/a.txt"}}), obj({{"tag", "file"}}), obj({{"tag", "nomatch"}}), obj({{"query", "alpha"}}), + obj({{"query", "write"}}), obj({{"query", "zzzz"}}), obj({{"limit", "x"}}), obj({{"limit", 500}}), obj({{"query", "unicode"}})}) + call("audit", a); + out("== tags"); + for (const Json& a : std::vector<Json>{Json::object(), obj({{"prefix", "al"}}), obj({{"prefix", "A_"}}), obj({{"prefix", "a%"}}), obj({{"prefix", "zzz"}}), obj({{"limit", 1}}), + obj({{"limit", "x"}}), obj({{"prefix", " Al "}})}) + call("tags", a); + out("== restore"); + for (const Json& a : std::vector<Json>{Json::object(), obj({{"id", "x"}}), obj({{"id", 9999}}), obj({{"id", 6}}), obj({{"id", 7}}), obj({{"id", 1}}), obj({{"id", 3}}), obj({{"id", 4}}), + obj({{"id", 5}}), obj({{"id", 12}}), obj({{"id", 10}}), obj({{"id", 11}})}) + call("restore", a); + for (const std::string rel : {"a.txt", "sub/deep/r.txt", "u.txt"}) { + std::string path = "area/" + rel; + out("file " + rel + " " + (PyFs::exists(path) ? js(read(path)) : std::string("null"))); + } + app.approve = false; + call("restore", obj({{"id", 2}})); + app.approve = true; + Json ap = Json::array(); + for (const auto& a : app.approvals) ap.push(replace_root(a, root)); + out("approvals " + j(ap)); +} + +int main(int argc, char** argv) { + if (argc < 3) return 2; + std::string name = argv[1], root = argv[2]; + bool prepare = argc > 3 && std::string(argv[3]) == "prepare"; + const char* data_dir = std::getenv("MISC_DATA_DIR"); + DATA_DIR = data_dir ? data_dir : "tests/misc_data"; + if (chdir((root + "/proj").c_str()) != 0) return 2; + if (prepare) return 0; // C++ never prepares: the Python side builds fixtures with the real tai.py + static const std::map<std::string, std::function<void(const std::string&)>> scenarios = { + {"json", sc_json}, {"datetime", sc_datetime}, {"chunk", sc_chunk}, {"fts", sc_fts}, + {"skills_data", sc_skills_data}, {"skills_files", sc_skills_files}, {"skills_ops", sc_skills_ops}, + {"install_hooks", sc_install_hooks}, {"install_hooks_noshell", sc_install_hooks}, {"install_ops", sc_install_ops}, + {"install_noengine", sc_install_noengine}, {"vault_line", sc_vault_line}, {"sched", sc_sched}, {"rag", sc_rag}, {"release", sc_release}, {"audit", sc_audit}, {"telegram", sc_telegram}, {"telegram_install", sc_telegram_install}, + }; + auto it = scenarios.find(name); + if (it == scenarios.end()) { + std::cerr << "unknown scenario " << name << "\n"; + return 2; + } + it->second(root); + return 0; +} diff --git a/taicpp/tests/misc_py.py b/taicpp/tests/misc_py.py new file mode 100644 index 0000000..35b2562 --- /dev/null +++ b/taicpp/tests/misc_py.py @@ -0,0 +1,1167 @@ +#!/usr/bin/env python3 +"""Python reference driver for the differential tests of the misc modules (tests/misc_diff.py). + +Runs one scenario against the real tai.py and prints a line-oriented transcript. The C++ side +(tests/misc_driver.cpp) prints the same transcript for the same scenario; the orchestrator diffs them. +Usage: misc_py.py SCENARIO ROOT [prepare] +""" +import importlib.util +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +TAI_PY = os.path.join(os.path.dirname(os.path.dirname(HERE)), "tai.py") +DATA = os.path.join(HERE, "misc_data") + + +def load_tai(): + spec = importlib.util.spec_from_file_location("tai", TAI_PY) + mod = importlib.util.module_from_spec(spec) + sys.modules["tai"] = mod + spec.loader.exec_module(mod) + return mod + + +tai = load_tai() + + +def data(name): + with open(os.path.join(DATA, name), encoding="utf-8") as handle: + return json.load(handle) + + +def j(value): + return json.dumps(value) + + +def out(line=""): + sys.stdout.write(line + "\n") + + +def show(label, text): + out("%s: %s" % (label, j(text))) + + +# ------------------------------------------------------------------ scenarios without state +def sc_json(root): + for case in data("json_cases.json"): + try: + value = json.loads(case) + out("%s => %s" % (j(case), json.dumps(value))) + except ValueError: + out("%s => ERR" % j(case)) + + +def sc_datetime(root): + for case in data("datetime_cases.json"): + try: + moment = tai._fromisoformat(case) + iso = moment.isoformat() + except Exception: + iso = "ERR" + try: + parsed = tai.parse_schedule_at(case) + except Exception as exc: + parsed = "ERR " + str(exc) + shown = "-" + if not parsed.startswith("ERR"): + try: + shown = j(tai.local_display(parsed)) + except Exception as exc: + shown = "ERR " + str(exc) + out("%s => %s | %s | %s" % (j(case), iso, parsed, shown)) + for case in data("duration_cases.json"): + out("duration %s => %s" % (j(case), j(tai.parse_duration(case)))) + for case in data("delay_cases.json"): + out("delay %s => %s" % (j(case), j(tai.format_delay(case)))) + + +def sc_chunk(root): + for case in data("chunk_cases.json"): + chunks = tai.chunk_lines(case["text"], case["max_lines"], case["overlap"], case["max_chars"]) + out("chunks %d %s" % (len(chunks), j(chunks))) + out("collapse %s" % j(tai.collapse_repeated_lines(case["text"]))) + out("lines %s" % j(case["text"].splitlines())) + out("strip %s" % j(case["text"].strip())) + + +def sc_fts(root): + for case in data("fts_cases.json"): + terms = tai.fts_escape(case) + out("%s => %s" % (j(case), j(terms))) + if terms: + out(" match %s" % j(tai.fts_match(terms))) + out(" or %s" % j(tai.fts_match_or(terms))) + out(" prefix %s" % j(tai.fts_match_prefix(terms))) + out(" variants %s" % j(tai.fts_match_variants(terms))) + out("rank %s" % j(tai.fts_rank("fts_docs"))) + for raw in ["Release Notes!", " a b ", "---", "x" * 40, "été", "a,b;c", "Hello World 2", "..", "UPPER", "a-b-c"]: + out("tag %s => %s" % (j(raw), j(tai.normalize_tag(raw)))) + out("tags %s" % j(tai.normalize_tags(["A b", "a-b", "", "C!", "d" * 40, "d" * 39 + "e"] + ["t%d" % i for i in range(30)]))) + + +def sc_skills_data(root): + for name in sorted(tai.BUNDLED_SKILLS): + out("skill %s upstream %s" % (name, j(tai.BUNDLED_SKILL_UPSTREAM[name]))) + for rel in sorted(tai.BUNDLED_SKILLS[name]): + text = tai.BUNDLED_SKILLS[name][rel] + out(" file %s len %d sha8 %s sha256 %s" % (rel, len(text.encode("utf-8")), tai.bundled_skill_digest(text), __import__("hashlib").sha256(text.encode("utf-8")).hexdigest())) + for name in sorted(tai.SKILL_BLUEPRINTS): + bp = tai.SKILL_BLUEPRINTS[name] + out("blueprint %s hint %s scope %s brief %s" % (name, j(bp["hint"]), bp["scope"], j(bp["brief"]))) + out("catalog empty %s" % j(tai.skill_catalog({}))) + out("catalog one %s" % j(tai.skill_catalog({"deep-research": {"description": "x" * 400}, "zeta": {"description": "short"}}))) + out("prompt %s" % j(tai.create_skill_prompt("my-skill", "do the thing", "/tmp/x/my-skill"))) + for raw in ["a", "ab", "a-b", "-a", "a-", "A", "a_b", "a" * 64, "a" * 65, "", "a b", "0x", "a\n"]: + out("name %s => %s" % (j(raw), bool(tai.SKILL_NAME_RE.match(raw)))) + front_cases = [ + ["name: x", "description: y"], + ["name: 'x'", 'description: "y z"'], + ["name: x", "description: >", " line one", " line two", "other: v"], + ["name: x", "description: |", " line one", " line two", "tail: 1"], + ["name: x", "description: >-", " folded"], + [" indented: no", "key: va:lue", "noval", "k2:", "k3: ''quoted''"], + ["a: >", "b: 1"], + ["a: |", "\tt1", " t2", "z: 9"], + ] + for lines in front_cases: + out("front %s => %s" % (j(lines), j(tai.parse_skill_front(lines)))) + + +# ------------------------------------------------------------------ helpers for filesystem scenarios +def write(path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="") as handle: + handle.write(text) + + +def read(path): + with open(path, encoding="utf-8", newline="") as handle: + return handle.read() + + +def tai_home(): + return os.environ["TAI_HOME"] + + +def sc_skills_files(root): + home = tai_home() + out("== front-matter parse") + cases = { + "ok": "---\nname: ok\ndescription: fine one\n---\n\n\nbody here\n", + "crlf": "---\r\nname: crlf\r\ndescription: win\r\n---\r\nbody\r\n", + "nofront": "just text\n", + "noend": "---\nname: noend\ndescription: d\n", + "badname": "---\nname: Bad_Name\ndescription: d\n---\nbody\n", + "nodesc": "---\nname: nodesc\n---\nbody\n", + "quoted": "---\nname: 'quoted'\ndescription: \"has: colon\"\n---\nbody\n", + "folded": "---\nname: folded\ndescription: >\n one\n two\n---\nbody\n", + "emptybody": "---\nname: emptybody\ndescription: d\n---", + } + for key, text in cases.items(): + path = os.path.join(root, "parse", key, "SKILL.md") + write(path, text) + skill = tai.parse_skill_file(path) + out("parse %s => %s" % (key, j(skill))) + out("== discover with external roots and collisions") + proj = os.path.join(root, "proj") + write(os.path.join(os.path.expanduser("~"), ".claude", "skills", "ext-home", "SKILL.md"), "---\nname: ext-home\ndescription: from claude home\n---\nbody ext-home\n") + write(os.path.join(os.path.expanduser("~"), ".claude", "skills", "shared", "SKILL.md"), "---\nname: shared\ndescription: external copy\n---\nbody external\n") + write(os.path.join(proj, ".claude", "skills", "proj-claude", "SKILL.md"), "---\nname: proj-claude\ndescription: project claude\n---\nbody pc\n") + write(os.path.join(proj, ".agents", "skills", "proj-agents", "SKILL.md"), "---\nname: proj-agents\ndescription: project agents\n---\nbody pa\n") + write(os.path.join(home, "skills", "shared", "SKILL.md"), "---\nname: shared\ndescription: tai copy\n---\nbody tai\n") + write(os.path.join(home, "skills", "native", "SKILL.md"), "---\nname: native\ndescription: native home skill\n---\nbody native\n") + write(os.path.join(home, "skills", "native", "scripts", "run.sh"), "echo hi\n") + write(os.path.join(home, "skills", "native", "references", "ref.md"), "ref\n") + write(os.path.join(home, "skills", "native", "assets", "a.txt"), "a\n") + write(os.path.join(proj, ".tai", "skills", "proj-tai", "SKILL.md"), "---\nname: proj-tai\ndescription: project tai skill\n---\nbody ptai\n") + write(os.path.join(home, "skills", "broken", "SKILL.md"), "no front matter\n") + write(os.path.join(home, "skills", "stray-file.txt"), "x\n") + tai.save_skill_install_meta(os.path.join(home, "skills"), "native", {"source": "/some/path", "mode": "symlink", "scope": "home", "n": 3, "nested": {"a": [1, None, True]}}) + skills = tai.discover_skills(home, proj) + for name in skills: + s = skills[name] + out("skill %s root=%s managed=%s install=%s note=%s extras=%s" % (name, s["root"], s["managed"], j(s["install"]), j(tai.skill_provenance_note(s)), j(tai.skill_extra_files(s["root"])))) + out(" desc=%s body=%s" % (j(s["description"]), j(s["body"]))) + out("order %s" % j(list(skills))) + out("catalog %s" % j(tai.skill_catalog(skills))) + out("== scope helpers") + for scope in ("project", "home"): + out("scope_dir %s %s" % (scope, tai.skill_scope_dir(scope, home, proj, "nm"))) + out("scope_dir %s %s" % (scope, tai.skill_scope_dir(scope, home, proj))) + empty = os.path.join(root, "emptyproj") + os.makedirs(empty, exist_ok=True) + out("default scope empty %s" % tai.default_skill_scope(empty)) + for marker in (".git", ".tai", ".claude/skills", ".agents/skills"): + d = os.path.join(root, "mk-" + marker.replace("/", "_")) + os.makedirs(os.path.join(d, marker), exist_ok=True) + out("default scope %s %s" % (marker, tai.default_skill_scope(d))) + for path in ("/a/b/.claude/skills/x", "/a/b/.agents/skills/x/", "/a/b/other/x", "/a/.claude/skills/../x", "/a/.claude/skills//x"): + out("external %s %s" % (path, tai.is_external_skill_root(path))) + out("meta none %s" % j(tai.load_skill_install_meta(os.path.join(home, "skills"), "nothing"))) + write(os.path.join(home, "skills", ".installs", "bad.json"), "{not json") + out("meta bad %s" % j(tai.load_skill_install_meta(os.path.join(home, "skills"), "bad"))) + out("meta ok %s" % j(tai.load_skill_install_meta(os.path.join(home, "skills"), "native"))) + + +def sc_skills_ops(root): + home = tai_home() + + def tree(): + found = [] + base = os.path.join(home, "skills") + for dirpath, _dirs, files in os.walk(base): + for f in files: + found.append(os.path.relpath(os.path.join(dirpath, f), base)) + return sorted(found) + + def step(label, mode): + out("%s => %s" % (label, tai.op_skills(mode, home))) + out(" status: %s" % tai.status_lines(home)[-2]) + out(" files: %d" % len(tree())) + + out("-- fresh") + step("install", "install") + step("install again", "install") + step("upgrade clean", "upgrade") + out("pristine all %s" % j([tai.bundled_skill_pristine(home, n) for n in sorted(tai.BUNDLED_SKILLS)])) + taste = os.path.join(home, "skills", "taste", "SKILL.md") + write(taste, "user rewrite") + out("-- modified taste") + step("install", "install") + step("upgrade", "upgrade") + extra = os.path.join(home, "skills", "taste", "mynotes.md") + write(extra, "mine") + step("reinstall", "reinstall") + out("extra kept %s" % os.path.isfile(extra)) + out("-- crlf edit counts as equal after universal newlines") + path = os.path.join(home, "skills", "plan", "SKILL.md") + write(path, read(path).replace("\n", "\r\n")) + step("upgrade crlf", "upgrade") + out("-- uninstall with user content") + gitp = os.path.join(home, "skills", "git", "SKILL.md") + write(gitp, "user rewrite git") + gextra = os.path.join(home, "skills", "git", "mynotes.md") + write(gextra, "mine") + write(os.path.join(home, "skills", "git", "sub", "deep.txt"), "deep") + step("uninstall", "uninstall") + step("uninstall again", "uninstall") + os.remove(gitp) + os.remove(gextra) + os.remove(os.path.join(home, "skills", "git", "sub", "deep.txt")) + step("uninstall final", "uninstall") + step("uninstall empty", "uninstall") + out("-- sidecar-only uninstall") + tai.save_skill_install_meta(os.path.join(home, "skills"), "taste", {"files": {}}) + step("uninstall sidecar", "uninstall") + out("-- tracked-digest removal of an edited file") + step("install", "install") + meta = tai.load_skill_install_meta(os.path.join(home, "skills"), "taste") + out("meta %s" % j(sorted(meta))) + out("meta files %s" % j(meta["files"])) + out("meta fixed %s" % j({k: meta[k] for k in ("source", "mode", "scope", "upstream", "bundled_version")})) + os.chmod(os.path.join(home, "skills", "taste"), 0o500) + step("install readonly", "install") + os.chmod(os.path.join(home, "skills", "taste"), 0o700) + out("-- error path: skills dir is a file") + other = os.path.join(root, "home2") + os.makedirs(other, exist_ok=True) + write(os.path.join(other, "skills"), "file not dir") + out("blocked => %s" % tai.op_skills("install", other)) + out("blocked uninstall => %s" % tai.op_skills("uninstall", other)) + sc_skills_install_tool(root) + + +class SkillApp: + def __init__(self, home, cwd): + self.config = type("C", (), {"home": home})() + self.profile = "default" + self.skills = {} + self.store = None + self.depth = 0 + self.runner_override = None + self.approvals = [] + self.refreshed = 0 + self.approve = True + + def ask_approval(self, text): + self.approvals.append(text) + return self.approve + + def apply_system(self): + self.refreshed += 1 + + +def sc_skills_install_tool(root): + home = tai_home() + proj = os.path.join(root, "proj") + os.chdir(proj) + app = SkillApp(home, proj) + app.skills = tai.discover_skills(home, proj) + tools = tai.Tools(app) + out("== install_skill / load_skill / create_skill") + src = os.path.join(root, "srcskill") + write(os.path.join(src, "SKILL.md"), "---\nname: from-src\ndescription: installed from a dir\n---\nbody src\n") + write(os.path.join(src, "scripts", "x.sh"), "echo x\n") + write(os.path.join(src, "references", "r.md"), "r\n") + nested = os.path.join(root, "nested") + write(os.path.join(nested, "inner", "SKILL.md"), "---\nname: inner\ndescription: nested one\n---\nbody inner\n") + two = os.path.join(root, "two") + write(os.path.join(two, "a", "SKILL.md"), "---\nname: aa\ndescription: d\n---\nb\n") + write(os.path.join(two, "b", "SKILL.md"), "---\nname: bb\ndescription: d\n---\nb\n") + barefile = os.path.join(root, "bare.md") + write(barefile, "---\nname: bare-one\ndescription: bare md file\n---\nbody bare\n") + badfile = os.path.join(root, "bad.md") + write(badfile, "no front\n") + ext = os.path.join(proj, ".claude", "skills", "extskill") + write(os.path.join(ext, "SKILL.md"), "---\nname: extskill\ndescription: external one\n---\nbody ext\n") + notes = [] + + def call(label, args): + res = tools.dispatch("install_skill", json.dumps(args)) + out("%s => %s" % (label, j(res))) + + call("empty", {"source": ""}) + call("bad scope", {"source": src, "scope": "mars"}) + call("bad mode", {"source": src, "mode": "link"}) + call("missing", {"source": os.path.join(root, "nope")}) + call("dir home", {"source": src, "scope": "home"}) + call("dir home again denied?", {"source": src, "scope": "home"}) + app.approve = False + call("dir home denied", {"source": src, "scope": "home"}) + app.approve = True + call("dir home force", {"source": src, "scope": "home", "force": True}) + call("dir project", {"source": src, "scope": "project", "name": "renamed"}) + call("bad name", {"source": src, "name": "Bad Name"}) + call("nested", {"source": nested, "scope": "home"}) + call("two children", {"source": two, "scope": "home"}) + call("bare file", {"source": barefile, "scope": "home"}) + call("bad file", {"source": badfile}) + call("symlink ext default", {"source": ext, "scope": "home"}) + call("symlink explicit", {"source": src, "scope": "home", "mode": "symlink", "name": "linked", "force": True}) + call("symlink bare", {"source": barefile, "mode": "symlink"}) + call("not a skill source", {"source": "/dev/null"}) + out("approvals %s" % j(app.approvals)) + out("refreshed %d" % app.refreshed) + out("skills %s" % j(sorted(app.skills))) + out("== load_skill") + app.skills = tai.discover_skills(home, proj) + for name in ("from-src", "deep-research", "extskill", "bot-creator", "nonexistent", ""): + if name == "bot-creator": + continue + out("load %s => %s" % (j(name), j(tools.dispatch("load_skill", json.dumps({"name": name}))))) + out("== create_skill") + app.runner_override = lambda prompt, profile, timeout: "worker said: " + prompt[:40] + " t=%s" % timeout + for args in ({"name": "", "brief": "b"}, {"name": "ok-name", "brief": ""}, {"name": "ok-name", "brief": "b", "scope": "mars"}, + {"name": "made", "brief": "make it", "scope": "home", "timeout": 10}, {"name": "made2", "brief": "make it", "timeout": "abc"}, + {"name": "made3", "brief": "x", "timeout": 99999}): + out("create %s => %s" % (j(args), j(tools.dispatch("create_skill", json.dumps(args))))) + app.depth = 5 + out("create depth => %s" % j(tools.dispatch("create_skill", json.dumps({"name": "dd", "brief": "b"})))) + app.depth = 0 + + def worker(prompt, profile, timeout): + marker = "Create a new agent skill named 'wrote' in " + path = prompt[len(marker):].split(".\n", 1)[0] + write(os.path.join(path, "SKILL.md"), "---\nname: wrote\ndescription: written by worker\n---\nbody wrote\n") + return "done writing " + path.replace(root, "<ROOT>") + + app.runner_override = worker + out("create written => %s" % j(tools.dispatch("create_skill", json.dumps({"name": "wrote", "brief": "w", "scope": "home"})).replace(root, "<ROOT>"))) + out("create replaced => %s" % j(tools.dispatch("create_skill", json.dumps({"name": "wrote", "brief": "w", "scope": "home"})).replace(root, "<ROOT>"))) + out("refreshed %d" % app.refreshed) + + +# ------------------------------------------------------------------ install scenarios +class FakeBox: + """Deterministic stand-in for the container probes (patched into tai.py; the C++ twin is FakeBox in the driver).""" + + def __init__(self, engine, state, venv, ensure_error): + self.engine, self.state, self.venv, self.ensure_error = engine, state, venv, ensure_error + + def install(self): + tai.container_engine = lambda: self.engine + tai.box_state = lambda engine: self.state + tai.box_venv_state = lambda engine: self.venv + + def ensure(): + if self.ensure_error: + raise tai.BackendError(self.ensure_error) + return self.engine + + tai.ensure_box = ensure + + +def install_env(root, engine=None, state="running", venv="venv ready", ensure_error=""): + FakeBox(engine, state, venv, ensure_error).install() + + +def sc_install_hooks(root): + install_env(root) + home = tai_home() + rc = os.path.join(os.path.expanduser("~"), ".bashrc") + zrc = os.path.join(os.path.expanduser("~"), ".zshrc") + write(rc, "# my bashrc\nexport A=1") + write(zrc, "") + steps = [ + ("bash install", lambda: tai.op_hook("install", home)), + ("bash install again", lambda: tai.op_hook("install", home)), + ("bash upgrade", lambda: tai.op_hook("upgrade", home)), + ("plexer install", lambda: tai.op_plexer_hook("install", home)), + ("plexer again", lambda: tai.op_plexer_hook("install", home)), + ("zsh install", lambda: tai.op_zsh_hook("install", home)), + ("zsh again", lambda: tai.op_zsh_hook("install", home)), + ("fish install", lambda: tai.op_fish_hook("install", home)), + ("zsh plexer install", lambda: tai.op_zsh_plexer_hook("install", home)), + ("fish plexer install", lambda: tai.op_fish_plexer_hook("install", home)), + ("status", lambda: "\n".join(tai.status_lines(home))), + ("bash reinstall", lambda: tai.op_hook("reinstall", home)), + ("plexer reinstall", lambda: tai.op_plexer_hook("reinstall", home)), + ("zsh uninstall", lambda: tai.op_zsh_hook("uninstall", home)), + ("fish uninstall", lambda: tai.op_fish_hook("uninstall", home)), + ("zsh uninstall again", lambda: tai.op_zsh_hook("uninstall", home)), + ("bash uninstall", lambda: tai.op_hook("uninstall", home)), + ("plexer uninstall", lambda: tai.op_plexer_hook("uninstall", home)), + ("zsh plexer uninstall", lambda: tai.op_zsh_plexer_hook("uninstall", home)), + ("fish plexer uninstall", lambda: tai.op_fish_plexer_hook("uninstall", home)), + ("status after", lambda: "\n".join(tai.status_lines(home))), + ] + for label, fn in steps: + out("%s => %s" % (label, j(fn()))) + out("rc %s" % j(read(rc))) + out("zrc %s" % j(read(zrc))) + out("== stale and duplicated blocks") + write(rc, "head\n" + tai.BASHRC_MARK_BEGIN + "\nstale content\n" + tai.BASHRC_MARK_END + "\nmid\n" + tai.BASHRC_MARK_BEGIN + "\nsecond stale\n" + tai.BASHRC_MARK_END + "\ntail\n") + out("install over stale => %s" % j(tai.op_hook("install", home))) + out("rc %s" % j(read(rc))) + write(rc, "x\n" + tai.BASHRC_MARK_BEGIN + "\nunterminated\n") + out("install over unterminated => %s" % j(tai.op_hook("install", home))) + out("rc %s" % j(read(rc))) + out("uninstall unterminated => %s" % j(tai.op_hook("uninstall", home))) + out("rc %s" % j(read(rc))) + write(rc, "crlf\r\nline\r\n") + out("install crlf => %s" % j(tai.op_hook("install", home))) + out("rc %s" % j(read(rc))) + os.chmod(rc, 0o400) + out("readonly install => %s" % j(tai.op_plexer_hook("install", home))) + os.chmod(rc, 0o600) + blocks = [tai.hook_present, tai.zsh_hook_present, tai.fish_hook_present, tai.plexer_hook_present, tai.zsh_plexer_hook_present, tai.fish_plexer_hook_present] + out("present %s" % j([fn(rc) for fn in blocks])) + out("present missing %s" % j([fn(os.path.join(root, "nope")) for fn in blocks])) + for shell in ("bash", "zsh", "fish", "other"): + out("rc path %s %s" % (shell, tai.shell_rc_path(shell))) + os.environ["XDG_CONFIG_HOME"] = os.path.join(root, "xdg") + out("rc path fish xdg %s" % tai.shell_rc_path("fish")) + del os.environ["XDG_CONFIG_HOME"] + out("shell_present %s" % j([tai.shell_present(n) for n in ("zsh", "fish", "bash", "nosuchshell")])) + + +def sc_install_ops(root): + install_env(root, engine="podman", state="running", venv="venv ready") + home = tai_home() + out("-- binary") + for mode in ("install", "install", "upgrade", "reinstall", "uninstall", "uninstall", "install"): + out("binary %s => %s" % (mode, j(tai.op_binary(mode, home)))) + target = os.path.join(os.path.expanduser("~"), ".local", "bin", "tai.py") + out("binary mode %s size_equal %s" % (oct(os.stat(target).st_mode & 0o777), os.path.getsize(target) == os.path.getsize(tai.__file__))) + write(target, "VERSION = \"0.0.1\"\n") + out("binary refresh => %s" % j(tai.op_binary("install", home))) + out("binary refresh upgrade => %s" % j(tai.op_binary("upgrade", home))) + out("script_version %s %s %s" % (j(tai.script_version(target)), j(tai.script_version(os.path.join(root, "nope"))), j(tai.script_version(tai.__file__)))) + write(os.path.join(root, "v1"), "x\nVERSION = \"1.2.3\"\ny\n") + write(os.path.join(root, "v2"), "VERSION = \"1.2\"\n VERSION = \"1.2.3\"\nVERSION = \"a.b.c\"\n") + write(os.path.join(root, "v3"), "VERSION = \"4.5.6\"") + write(os.path.join(root, "v4"), "VERSION = \"4.5.6\" \n") + out("script_version v1..v4 %s" % j([tai.script_version(os.path.join(root, n)) for n in ("v1", "v2", "v3", "v4")])) + out("-- venv") + for mode in ("install", "install", "reinstall", "uninstall", "uninstall", "install"): + out("venv %s => %s" % (mode, j(tai.op_venv(mode, home).replace(root, "<ROOT>")))) + out("venv helpers %s %s %s" % (j(tai.venv_dir(home).replace(root, "<ROOT>")), j((tai.venv_python(home) or "").replace(root, "<ROOT>")), tai.venv_available())) + out("-- services (no systemctl)") + for mode in ("install", "install", "reinstall", "upgrade", "uninstall", "uninstall"): + out("scheduler %s => %s" % (mode, j(tai.op_scheduler_service(mode, home).replace(root, "<ROOT>")))) + out("telegram without token => %s" % j(tai.op_telegram_service("install", home))) + out("service_exec %s" % j(tai.service_exec(home, "/x/tai.py", "--a", "b").replace(root, "<ROOT>"))) + out("unit_dir %s" % j(tai.unit_dir(home).replace(root, "<ROOT>"))) + out("service_state %s" % j(tai.service_state("tai-scheduler.service"))) + out("activate %s" % j(tai.activate_service("tai-scheduler.service"))) + out("remove %s" % j(tai.remove_service_unit(home, "tai-scheduler.service"))) + out("-- container") + for mode in ("install", "upgrade", "reinstall", "uninstall"): + out("container %s => %s" % (mode, j(tai.op_container(mode, home)))) + install_env(root, engine="podman", state="stopped", venv="no venv (reinstall container to rebuild)") + for mode in ("install", "reinstall"): + out("container stopped %s => %s" % (mode, j(tai.op_container(mode, home)))) + install_env(root, engine="podman", state="running", venv="venv ready", ensure_error="engine exploded") + out("container failing => %s" % j(tai.op_container("install", home))) + install_env(root, engine=None) + for mode in ("install", "uninstall"): + out("container noengine %s => %s" % (mode, j(tai.op_container(mode, home)))) + install_env(root, engine="podman", state="running", venv="venv ready") + out("-- install_report") + for action, targets in (("install", ["skills", "bash-hook", "venv"]), ("status", ["skills"]), ("upgrade", list(tai.INSTALL_TARGETS)), ("uninstall", ["skills", "bash-hook", "plexer-hook", "binary"])): + out("report %s %s =>\n%s" % (action, targets, tai.install_report(action, targets, home).replace(root, "<ROOT>"))) + out("-- install_self") + for shell in ("/bin/bash", "/usr/bin/zsh", "/usr/bin/fish", "", "/bin/sh"): + os.environ["SHELL"] = shell + out("shell %s" % j(shell)) + sys.stdout.flush() + tai.install_self() + out("-- install tool") + app = SkillApp(home, root) + app.config.home = home + tools = tai.Tools(app) + for args in ({"action": "bogus"}, {"action": "status"}, {"action": "STATUS ", "targets": ["skills"]}, {"action": "install", "targets": ["nope"]}, {"action": "install", "targets": []}, + {"action": "install", "targets": "skills"}, {"action": "install", "targets": ["skills"]}, {"action": "uninstall", "targets": ["skills", "bash-hook"]}, {}): + app.skills = {} + res = tools.dispatch("install", json.dumps(args)).replace(root, "<ROOT>") + out("tool %s => %s | skills=%s" % (j(args), j(res), j(sorted(app.skills)))) + app.approve = False + out("tool denied => %s" % j(tools.dispatch("install", json.dumps({"action": "install", "targets": ["skills"]})))) + out("approvals %s" % j(app.approvals)) + + +def sc_install_noengine(root): + install_env(root, engine=None) + home = tai_home() + out("%s" % j(tai.status_lines(home)[-3:])) + for mode in ("install", "uninstall"): + out("container %s => %s" % (mode, j(tai.op_container(mode, home)))) + + +def sc_vault_line(root): + home = tai_home() + out("empty %s" % j(tai.vault_line(home))) + write(os.path.join(home, "memory.db"), "") + out("kept %s" % j(tai.vault_line(home).replace(root, "<ROOT>"))) + + +# ------------------------------------------------------------------ scheduler scenarios +def open_store(profile="t"): + class FakeArgs: + yes = True + + FakeArgs.profile = profile + config = tai.Config(FakeArgs) + seal = tai.Seal(config.home, "") + store = tai.Store(config, seal) + return config, seal, store + + +NOW_FIXED = "2026-10-25T12:00:00+00:00" +LONG_EVERY = 315360000 # ten years: recurring claims land far in the future so reruns are stable + + +def prep_sched(root): + config, seal, store = open_store() + past = "2026-01-01T00:00:00+00:00" + store.add_schedule("once-due", "do:once", "t", 0, past, 300) + store.add_schedule("rec-due", "do:rec", "t", LONG_EVERY, past, 600, ["alpha", "beta"]) + store.add_schedule("future", "do:future", "t", 0, "2099-01-01T00:00:00+00:00", 600, []) + store.add_schedule("leased", "do:leased", "t", LONG_EVERY, "2026-02-01T00:00:00+00:00", 600) + store.add_schedule("stale-lease", "do:stale", "t", LONG_EVERY, "2026-03-01T00:00:00+00:00", 600) + store.add_schedule("exploder", "boom: fail", "t", 0, "2026-04-01T00:00:00+00:00", 300) + store.add_schedule("other-profile", "do:other", "other", 0, "2026-06-01T00:00:00+00:00", 300) + store.add_schedule("bad-time", "do:badtime", "t", LONG_EVERY, "!!bad", 300) + store.add_schedule("", "x" * 100, "t", 0, "2099-06-01T00:00:00+00:00", 300, ["one"]) + store.add_schedule("with-result", "do:result", "t", 3600, "2099-07-01T00:00:00+00:00", 300) + store.db.execute("UPDATE schedules SET last_status = 'running', updated = ? WHERE name = 'leased'", (tai.datetime.now(tai.timezone.utc).isoformat(),)) + store.db.execute("UPDATE schedules SET last_status = 'running', updated = '2000-01-01T00:00:00+00:00' WHERE name = 'stale-lease'") + store.db.execute("UPDATE schedules SET last_status = 'done', last_result = ? WHERE name = 'with-result'", ("r" * 200,)) + store.db.commit() + store.close() + + +def sched_rows(store, label): + rows = store.db.execute("SELECT id, name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, updated FROM schedules ORDER BY id").fetchall() + out("-- %s" % label) + for row in rows: + out("row %s" % j(list(row))) + tags = store.db.execute("SELECT item, tag, profile FROM tags WHERE item LIKE 'sched:%' ORDER BY item, tag").fetchall() + out("tags %s" % j([list(t) for t in tags])) + + +class Gate: + """Blocks 'slow:' tasks until released, like a long-running worker.""" + + def __init__(self): + self.event = tai.threading.Event() + + def runner(self, task, profile, timeout): + if task.startswith("slow:"): + self.event.wait(30) + return "slow done " + task + if "boom" in task: + raise RuntimeError("boom") + return "ok:" + task + + +def quiesce(): + for _ in range(200): + if tai.threading.active_count() <= 1: + return + tai.time.sleep(0.05) + + +def sc_sched(root): + config, seal, store = open_store() + gate = Gate() + out("== list and format") + items = store.list_schedules() + for it in items: + out("item %s" % j(it)) + out("format:\n%s" % tai.format_schedule_lines(items, tai._fromisoformat(NOW_FIXED))) + out("format empty: %s" % tai.format_schedule_lines([], tai._fromisoformat(NOW_FIXED))) + out("other profile: %s" % j(store.list_schedules("other"))) + out("== running leases") + for status, updated, timeout in (("running", tai.datetime.now(tai.timezone.utc).isoformat(), 600), ("running", "2000-01-01T00:00:00+00:00", 600), ("done", "", 0), ("running", "garbage", 10), + ("running", tai.datetime.now(tai.timezone.utc).isoformat(), 0)): + out("running %s %s %s => %s" % (status, "now" if updated.startswith("2026") else updated, timeout, tai.schedule_running(status, updated, timeout))) + out("== sealed rows (added now: Store refuses to open with them present and no passphrase)") + store.add_schedule("sealed", "tai1$AAAA$BBBB$CCCC", "t", 0, "2026-05-01T00:00:00+00:00", 300) + store.add_schedule("sealed-corrupt", "tai1$bad", "t", 0, "2026-05-02T00:00:00+00:00", 300) + out("== tick 1") + sys.stdout.flush() + fired = tai.scheduler_tick(config, seal, gate.runner) + sys.stdout.flush() + out("fired %d" % fired) + quiesce() + sched_rows(store, "after tick 1") + out("== tick 2") + out("fired %d" % tai.scheduler_tick(config, seal, gate.runner)) + quiesce() + out("== cap phase") + for k in range(9): + store.add_schedule("slow%d" % k, "slow:%d" % k, "t", LONG_EVERY, "2025-01-0%dT00:00:00+00:00" % (k + 1), 600) + out("fired %d" % tai.scheduler_tick(config, seal, gate.runner)) + out("fired %d" % tai.scheduler_tick(config, seal, gate.runner)) + out("running agents %d" % tai.running_agents()) + out("fired %d" % tai.scheduler_tick(config, seal, gate.runner)) + sys.stdout.flush() + sched_rows(store, "after cap ticks") + out("== AgentsFull restore branch") + store.add_schedule("full-once", "slow:once", "t", 0, "2025-02-01T00:00:00+00:00", 600) + real = tai.running_agents + tai.running_agents = lambda: 0 + out("fired %d" % tai.scheduler_tick(config, seal, gate.runner)) + tai.running_agents = real + sched_rows(store, "after AgentsFull") + gate.event.set() + quiesce() + sched_rows(store, "after release") + out("== schedule tool") + app = SkillApp(tai_home(), root) + app.store = store + app.profile = "t" + tools = tai.Tools(app) + calls = [ + {"prompt": ""}, {"prompt": "p"}, {"prompt": "p", "at": "2099-01-01T10:00", "every": 120}, {"prompt": "p", "at": "nonsense"}, {"prompt": "p", "every": "abc"}, + {"prompt": "p", "every": 30}, {"prompt": "tool one", "at": "2099-01-01T10:00", "name": "N" * 100, "timeout": 5, "tags": ["Alpha", "beta!", "", "x" * 40]}, + {"prompt": "tool two", "every": 3600, "timeout": "abc", "profile": "t"}, {"prompt": "tool three", "every": "7200", "timeout": 99999}, + {"prompt": "p", "every": 60, "profile": "other"}, {"prompt": "p", "every": 60, "profile": "bad profile!"}, {"prompt": "p", "every": 60, "tags": "notalist"}, + {"prompt": "p", "every": 60, "tags": [1, 2]}, {"prompt": "tool four", "every": 86400, "tags": []}, {"prompt": " padded ", "at": "2099-02-02T02:02:02Z", "timeout": 0}, + {"prompt": "p", "at": "9999-12-31T24:00"}, {"prompt": "p", "at": "0001-01-01T00:00"}, {"prompt": "p", "every": 60.5}, {"prompt": "p", "every": True}, + ] + for args in calls: + out("schedule %s => %s" % (j(args), j(tools.dispatch("schedule", json.dumps(args))))) + app.approve = False + out("schedule denied => %s" % j(tools.dispatch("schedule", json.dumps({"prompt": "denied", "every": 60})))) + app.approve = True + out("schedules => %s" % j(tools.dispatch("schedules", "{}"))) + for args in ({"id": 999}, {"id": "abc"}, {"id": 0}, {}, {"id": "13"}, {"id": 14.0}): + out("unschedule %s => %s" % (j(args), j(tools.dispatch("unschedule", json.dumps(args))))) + app.approve = False + out("unschedule denied => %s" % j(tools.dispatch("unschedule", json.dumps({"id": 15})))) + app.approve = True + out("approvals %s" % j(app.approvals)) + sched_rows(store, "final") + out("remove nonexistent %s" % store.remove_schedule(99999)) + store.close() + + +# ------------------------------------------------------------------ rag scenarios +WORDS = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa quebec romeo sierra tango uniform victor whiskey xray yankee zulu " + "database retrieval indexing scheduler terminal pipeline compiler optimizer allocator tokenizer scanner buffer kernel module package release rollback deployment " + "café naïve résumé über 中文 snake_case kebab-case CamelCase ALLCAPS numbers123 v2beta").split() + + +def prep_rag(root): + import random + rng = random.Random(7) + config, seal, store = open_store() + ids = [] + for index in range(30): + title = " ".join(rng.choice(WORDS) for _ in range(rng.randint(1, 4))) + lines = [] + for _ in range(rng.randint(1, 12)): + lines.append(" ".join(rng.choice(WORDS) for _ in range(rng.randint(3, 14)))) + content = "\n".join(lines) + kind = rng.choice(["note", "doc", "termlog"]) + rid = store.add_record(kind, title, content, [rng.choice(["x", "y", "zed"])] if index % 3 == 0 else [], "t") + store.add_chunks("record", rid, tai.chunk_lines(content, 3, 1, 200), "t") + ids.append(rid) + store.add_record("note", "unique zebra", "the quick zebra jumps over the lazy dog " * 30, [], "t") + store.add_record("note", "other profile secret", "alpha bravo hidden", [], "other") + store.add_edge(ids[0], ids[1], "relates to", "t") + store.add_edge(ids[2], ids[0], "see also", "t") + store.add_edge(ids[0], ids[3], "x", "t") + store.add_edge(ids[0], ids[4], "y", "t") + store.add_edge(ids[0], ids[5], "z", "t") + for index in range(6): + store.audit_event("main", "write", "/tmp/file%d.txt" % index, "wrote alpha file %d" % index, None, "content %d" % index, ["t%d" % index], None, None, "t") + store.log_event("t", "user", "message", "hello alpha retrieval message") + store.log_event("t", "assistant", "message", "bravo reply about indexing") + # implicit pairs: search events followed by reads + store.log_event("t", "tool", "search-query", "q=alpha\tlimit=5\thits=record:%s,record:%s,chunk:3" % (ids[0], ids[1])) + store.log_event("t", "tool", "record-read", ids[1]) + store.log_event("t", "tool", "search-query", "q=nothing here\tlimit=5\thits=") + store.log_event("t", "tool", "search-query", "q=bravo\tlimit=5\thits=%s,record:%s" % (ids[2], ids[3])) + store.log_event("t", "tool", "record-read", " %s " % ids[9]) + store.log_event("t", "tool", "record-read", ids[2]) + store.log_event("t", "tool", "search-query", "no equals sign") + store.log_event("t", "tool", "search-query", "q=\thits=x") + store.log_event("t", "tool", "search-query", "q=a=b\tq=delta\thits=,,record:%s," % ids[6]) + store.log_event("t", "tool", "record-read", "") + store.close() + + +def hit_json(h): + return {"item": h["item"], "kind": h["kind"], "sub": h["sub"], "title": h["title"], "snippet": h["snippet"], "rank": h["rank"]} + + +def sc_rag(root): + config, seal, store = open_store() + out("== fts_search") + queries = ["alpha", "alpha bravo", "zebra", "café", "résumé naïve", "snake_case", "numbers123", "nothingmatchesthis", "", "!!!", "retrieval index", "wh", "ALPHA", "x" * 50, "unique zebra jumps", "message", + "file", "ro", "v2beta kernel"] + for q in queries: + for kinds in ((), ("record",), ("chunk",), ("audit",), ("event",), ("record", "chunk")): + for limit in (10, 3): + hits = store.fts_search(q, kinds, "t", limit) + out("search %s %s %d => %s" % (j(q), j(list(kinds)), limit, j([hit_json(h) for h in hits]))) + out("other profile %s" % j([hit_json(h) for h in store.fts_search("alpha", ("record",), "other", 10)])) + out("limits %s" % j([len(store.fts_search("alpha", (), "t", n)) for n in (0, -5, 1, 100, 500)])) + out("== eval") + out("tokens %s" % j(tai.RetrievalEval.tokens("The quick Brown fox jumped_over 123456 abcdefg ABCDEFG cafés résumés"))) + out("tokens4 %s" % j(tai.RetrievalEval.tokens("The quick Brown fox", 4))) + goldens = tai.RetrievalEval.goldens_from_vault(store, "t") + out("goldens %d" % len(goldens)) + for g in goldens: + out("golden %s" % j(g)) + result = tai.RetrievalEval.score(store, goldens, "t") + out("score %s" % j(result)) + pairs = tai.RetrievalEval.implicit_pairs(store, "t") + out("pairs %s" % j(pairs)) + out("report:\n%s" % tai.RetrievalEval.format_report(result, pairs)) + out("empty report:\n%s" % tai.RetrievalEval.format_report(tai.RetrievalEval.score(store, [], "t"), tai.RetrievalEval.implicit_pairs(store, "nobody"))) + out("small k %s" % j(tai.RetrievalEval.score(store, goldens[:12], "t", ("record",), 1, 2))) + out("rank_of %s %s %s" % (j(tai.RetrievalEval.rank_of("b", ["a", "b"])), j(tai.RetrievalEval.rank_of("z", ["a"])), j(tai.RetrievalEval.best_rank(["z", "a"], ["q", "a", "z"])))) + out("metrics %s" % j([tai.RetrievalEval.hit_at_k(3), tai.RetrievalEval.hit_at_k(6), tai.RetrievalEval.hit_at_k(None), tai.RetrievalEval.reciprocal_rank(4), tai.RetrievalEval.ndcg_at_k(3), tai.RetrievalEval.ndcg_at_k(9), + tai.RetrievalEval.context_precision(["a", "c"], ["a", "b", "c", "d"]), tai.RetrievalEval.context_precision([], ["a"]), tai.RetrievalEval.context_precision(["x"], ["a"])])) + for text in ("q=alpha\thits=record:m1,m2,,chunk:4", "nothing", "q=\thits=a", "a=b=c\tq= spaced \thits= x ,y ", "q=a\tq=b\thits=record:z"): + out("parse %s => %s" % (j(text), j(tai.RetrievalEval._parse_search_event(text)))) + out("== chunks table") + rid = store.db.execute("SELECT id FROM records WHERE profile = 't' ORDER BY id LIMIT 1").fetchone()[0] + store.add_chunks("record", rid, ["one", "two", "three"], "t") + rows = store.db.execute("SELECT id, parent_kind, parent_id, idx, text, profile FROM chunks WHERE parent_id = ? ORDER BY idx", (rid,)).fetchall() + out("chunks %s" % j([list(r) for r in rows])) + ids = [r[0] for r in rows] + got = store.get_chunks_by_ids(ids + [999999], "t") + out("by ids %s" % j({str(k): v for k, v in sorted(got.items())})) + out("by ids empty %s other %s" % (j(store.get_chunks_by_ids([], "t")), j(store.get_chunks_by_ids(ids, "other")))) + store.add_chunks("record", rid, [], "t") + out("after clear %d" % store.db.execute("SELECT COUNT(*) FROM chunks WHERE parent_id = ?", (rid,)).fetchone()[0]) + out("== search tool") + app = SkillApp(tai_home(), root) + app.store = store + app.profile = "t" + tools = tai.Tools(app) + calls = [{"query": ""}, {"query": "alpha"}, {"query": "alpha", "limit": 3}, {"query": "alpha", "limit": "2"}, {"query": "alpha", "limit": "x"}, {"query": "alpha", "limit": 0}, {"query": "alpha", "limit": -2}, + {"query": "retrieval", "kinds": ["chunk"]}, {"query": "alpha", "kinds": ["record"], "expand": True, "limit": 20}, {"query": "alpha", "kinds": ["bogus"]}, {"query": "alpha", "kinds": []}, + {"query": "alpha", "kinds": "record"}, {"query": "file", "kinds": ["audit"]}, {"query": "message", "kinds": ["event"]}, {"query": "zebra", "kinds": ["record", "chunk", "audit"]}, + {"query": "nothingmatchesthis"}, {"query": "x" * 200}, {"query": " spaced "}, {"query": "café résumé", "limit": 50}] + for args in calls: + out("search %s => %s" % (j(args), j(tools.dispatch("search", json.dumps(args))))) + out("logged %s" % j([list(r) for r in store.db.execute("SELECT profile, role, kind, text FROM events WHERE kind = 'search-query' ORDER BY id DESC LIMIT 25").fetchall()])) + store.close() + + +# ------------------------------------------------------------------ telegram scenarios +class CtxDict(dict): + def __setitem__(self, key, value): + out("CTX set %s=%s" % (key, value)) + dict.__setitem__(self, key, value) + + def pop(self, key, *default): + out("CTX pop %s" % key) + return dict.pop(self, key, *default) + + +class LogStub: + enabled = False + + def __init__(self): + self.context = CtxDict() + + def event(self, level, component, event, message, profile="", bot="", chat="", meta=None): + out("LOG %s" % j([level, component, event, message, profile, bot, chat, meta])) + + +class TgAgent: + profile = "t" + bot = "main" + + def __init__(self, home): + self.config = type("C", (), {"home": home})() + self.store = type("S", (), {"seal": None})() + + def reset_history(self): + out("AGENT reset_history") + + def run_turn(self, text, capture=False): + out("AGENT run_turn %s capture=%s" % (j(text), capture)) + if "explode" in text: + raise RuntimeError("agent exploded") + if text == "long": + return "L" * 9000 + if text == "empty reply": + return "" + return "echo:" + text + + +class StopLoop(Exception): + pass + + +def tg_setup(root): + tai.TELEGRAM_API = os.environ["MISC_TG_API"] + tai.TELEGRAM_FILE_API = os.environ["MISC_TG_FILE"] + tai.LOG = LogStub() + + def ensure(): + out("BOX ensure_box") + if os.environ.get("MISC_BOX_FAIL"): + raise tai.BackendError("box down") + return "eng" + + def transcribe(engine, audio): + out("BOX transcribe %s %s" % (engine, j(audio.decode("latin-1")))) + if audio == b"AUDIO": + return "voice says hi" + return "error: model failed" + + tai.ensure_box = ensure + tai.box_transcribe = transcribe + tai.start_scheduler = lambda config, seal: out("start_scheduler called") + + +def sc_telegram(root): + tg_setup(root) + home = tai_home() + out("== token") + env_path = tai.telegram_env_path(home) + out("env path %s" % j(env_path.replace(root, "<ROOT>"))) + os.environ.pop("TELEGRAM_BOT_TOKEN", None) + out("no file => %s" % j(tai.load_telegram_token(home))) + for label, body in (("plain", "TELEGRAM_BOT_TOKEN=abc123\n"), ("quoted", "TELEGRAM_BOT_TOKEN='abc'\n"), ("dquoted", 'TELEGRAM_BOT_TOKEN="x y"\n'), ("spaces", "TELEGRAM_BOT_TOKEN= spaced \n"), + ("other first", "OTHER=1\nTELEGRAM_BOT_TOKEN=second\nTELEGRAM_BOT_TOKEN=third\n"), ("none", "OTHER=1\n"), ("empty", ""), ("crlf", "TELEGRAM_BOT_TOKEN=win\r\n"), + ("noeq", "TELEGRAM_BOT_TOKEN\n"), ("eq in value", "TELEGRAM_BOT_TOKEN=a=b=c\n"), ("quotes only", "TELEGRAM_BOT_TOKEN=''\n"), ("indented", " TELEGRAM_BOT_TOKEN=nope\n")): + write(env_path, body) + out("file %s => %s" % (label, j(tai.load_telegram_token(home)))) + os.environ["TELEGRAM_BOT_TOKEN"] = "fromenv" + out("env wins => %s" % j(tai.load_telegram_token(home))) + os.environ["TELEGRAM_BOT_TOKEN"] = "" + out("empty env falls back => %s" % j(tai.load_telegram_token(home))) + os.environ.pop("TELEGRAM_BOT_TOKEN", None) + os.remove(env_path) + out("== call / send") + good = tai.Telegram("good") + out("getMe %s" % j(good.call("getMe", {}))) + for label, token, method in (("bad token", "bad", "getMe"), ("not json", "nojson", "getMe")): + try: + res = tai.Telegram(token).call(method, {}) + out("%s => %s" % (label, j(res))) + except (OSError, ValueError) as exc: + out("%s => raised %s" % (label, j(tai.short_error(exc)))) + for text in ("", "short", "S" * 4000, "S" * 4001, "S" * 8000, "S" * 8001, "é" * 4001, "\U0001f600" * 4100, "line\n" * 2000): + tai.telegram_send(good, 42, text) + tai.telegram_send(good, "chat-as-string", "hi") + out("== updates") + agent = TgAgent(home) + updates = [ + {"update_id": 1, "message": {"chat": {"id": 111}, "text": "/start"}}, + {"update_id": 2, "message": {"chat": {"id": 111}, "text": "/new"}}, + {"update_id": 3, "message": {"chat": {"id": 111}, "text": "hello there"}}, + {"update_id": 4, "message": {"chat": {"id": 111}, "text": ""}}, + {"update_id": 5, "message": {"chat": {"id": 111}, "text": " "}}, + {"update_id": 6, "message": {"chat": {"id": 0}, "text": "zero chat"}}, + {"update_id": 7, "message": {"text": "no chat"}}, + {"update_id": 8}, + {"update_id": 9, "message": {"chat": {"id": 111}}}, + {"update_id": 10, "message": {"chat": {"id": 111}, "text": "explode please"}}, + {"update_id": 11, "message": {"chat": {"id": 111}, "text": "long"}}, + {"update_id": 12, "message": {"chat": {"id": 111}, "text": "empty reply"}}, + {"update_id": 13, "message": {"chat": {"id": 222}, "voice": {"file_id": "f1"}}}, + {"update_id": 14, "message": {"chat": {"id": 222}, "audio": {"file_id": "f1"}}}, + {"update_id": 15, "message": {"chat": {"id": 222}, "voice": {"file_id": "nofile"}}}, + {"update_id": 16, "message": {"chat": {"id": 222}, "voice": {"file_id": "fbad"}}}, + {"update_id": 17, "message": {"chat": {"id": 222}, "voice": {"file_id": "fmodel"}}}, + {"update_id": 18, "message": {"chat": {"id": 222}, "voice": {}, "text": "text with empty voice"}}, + {"update_id": 19, "message": {"chat": {"id": 222}, "voice": {"file_id": "fmissingfile"}}}, + {"update_id": 20, "message": {"chat": {"id": "str-chat"}, "text": "string chat id"}}, + {"update_id": 21, "message": {"chat": {"id": 111}, "text": "unicode café \U0001f600"}}, + ] + for update in updates: + out("-- update %s" % j(update)) + try: + tai.handle_telegram_update(agent, good, update) + except Exception as exc: + out("raised %s" % j(tai.short_error(exc))) + os.environ["MISC_BOX_FAIL"] = "1" + out("-- box failing") + try: + tai.handle_telegram_update(agent, good, {"update_id": 30, "message": {"chat": {"id": 5}, "voice": {"file_id": "f1"}}}) + except Exception as exc: + out("raised %s" % j(tai.short_error(exc))) + os.environ.pop("MISC_BOX_FAIL", None) + out("== run_telegram_bot") + for label, token in (("missing", None), ("bad auth", "bad"), ("not json", "nojson"), ("no username", "noname"), ("good", "good")): + os.environ.pop("TELEGRAM_BOT_TOKEN", None) + if token: + os.environ["TELEGRAM_BOT_TOKEN"] = token + out("-- run %s" % label) + sys.stdout.flush() + + def stop_sleep(seconds): + out("sleep %s" % seconds) + raise StopLoop() + + tai.time.sleep = stop_sleep + try: + res = tai.run_telegram_bot(agent) + out("returned %s" % res) + except StopLoop: + out("stopped by sleep") + sys.stdout.flush() + sys.stderr.flush() + + +def sc_telegram_install(root): + tg_setup(root) + home = tai_home() + install_env(root, engine="podman", state="running", venv="venv ready") + for label, token in (("good", "good"), ("bad", "bad"), ("nojson", "nojson"), ("noname", "noname")): + os.environ["TELEGRAM_BOT_TOKEN"] = token + out("-- install_telegram %s" % label) + sys.stdout.flush() + res = tai.install_telegram() + out("returned %s" % res) + out("unit exists %s env exists %s" % (os.path.exists(os.path.join(os.path.expanduser("~"), ".config", "systemd", "user", "tai-telegram.service")), os.path.exists(tai.telegram_env_path(home)))) + os.environ.pop("TELEGRAM_BOT_TOKEN", None) + out("-- install_telegram without token (stdin closed)") + sys.stdout.flush() + out("returned %s" % tai.install_telegram()) + out("-- install_telegram box failing") + install_env(root, engine="podman", ensure_error="no engine here") + os.environ["TELEGRAM_BOT_TOKEN"] = "good" + out("returned %s" % tai.install_telegram()) + install_env(root, engine="podman", state="running", venv="venv ready") + out("-- op_telegram_service") + for token in ("good", "bad", "nojson", "noname"): + os.environ["TELEGRAM_BOT_TOKEN"] = token + for mode in ("install", "install", "reinstall", "upgrade", "uninstall", "uninstall"): + out("service %s %s => %s" % (token, mode, j(tai.op_telegram_service(mode, home).replace(root, "<ROOT>")))) + os.environ.pop("TELEGRAM_BOT_TOKEN", None) + write(tai.telegram_env_path(home), "TELEGRAM_BOT_TOKEN=good\n") + out("service from env file => %s" % j(tai.op_telegram_service("install", home))) + out("-- uninstall_telegram") + out("returned %s" % tai.uninstall_telegram()) + install_env(root, engine=None) + out("returned noengine %s" % tai.uninstall_telegram()) + + +# ------------------------------------------------------------------ release / audit scenarios +def prep_release(root): + config, seal, store = open_store() + store.close() + + +def sc_release(root): + import shutil + config, seal, store = open_store() + script = os.path.join(root, "script.py") + shutil.copy(tai.__file__, script) + tai.__file__ = script + folder = os.path.join(root, "backups") + out("== versions") + for cur, part in (("1.2.3", "patch"), ("1.2.3", "minor"), ("1.2.3", "major"), ("9.9.9", "patch"), ("1.2.3", "bogus"), ("1.2", "patch"), ("a.b.c", "patch"), ("1.2.3.4", "patch")): + try: + out("next %s %s => %s" % (cur, part, tai.next_version(cur, part))) + except Exception as exc: + out("next %s %s => raised %s" % (cur, part, j(str(exc)))) + out("== bump") + for part in ("patch", "minor", "major", "patch"): + out("bump %s => %s" % (part, j(tai.release_bump(script, part)))) + with open(script, encoding="utf-8") as handle: + text = handle.read() + out("version line %s" % j([l for l in text.splitlines() if l.startswith("VERSION = ")])) + novers = os.path.join(root, "nov.py") + write(novers, "print('x')\n") + for f in (novers, os.path.join(root, "missing.py")): + try: + tai.release_bump(f, "patch") + except Exception as exc: + out("bump error %s" % j(str(exc).replace(root, "<ROOT>"))) + write(os.path.join(root, "crlf.py"), 'a\r\nVERSION = "1.0.0"\r\nb\r\n') + out("crlf bump %s %s" % (j(tai.release_bump(os.path.join(root, "crlf.py"), "patch")), j(open(os.path.join(root, "crlf.py"), "rb").read().decode()))) + out("== snapshots and pruning") + first, digest = tai.snapshot_self(script, folder, "9.9.9") + out("snapshot %s %s" % (j(os.path.basename(first)), digest)) + out("has digest %s %s" % (tai.backup_has_digest(folder, digest), tai.backup_has_digest(folder, "deadbeef"))) + out("has digest nofolder %s" % tai.backup_has_digest(os.path.join(root, "nofolder"), digest)) + for index in range(14): + name = os.path.join(folder, "tai-0.0.%d-20200101T000000Z-%08d.py" % (index, index)) + write(name, "x") + os.utime(name, (1000 + index, 1000 + index)) + write(os.path.join(folder, "notes.txt"), "keep") + out("pruned %d" % tai.prune_backups(folder)) + out("left %s" % j(sorted(os.listdir(folder))).replace(os.path.basename(first), "<SNAP>")) + out("prune missing %d" % tai.prune_backups(os.path.join(root, "nofolder"))) + out("== ensure_self_backup") + shutil.rmtree(folder) + home = tai_home() + first = tai.ensure_self_backup(config, store, script) + out("ensure first %s" % j(os.path.basename(first))) + out("ensure second %s" % j(tai.ensure_self_backup(config, store, script))) + rows = store.db.execute("SELECT actor, action, path, message, old_size, new_size, tags FROM audit ORDER BY id").fetchall() + out("audit %s" % j([list(r) for r in rows])) + out("== do_release") + try: + tai.do_release(store, script, folder, "patch", " ") + except Exception as exc: + out("blank message %s" % j(str(exc))) + res = tai.do_release(store, script, folder, "minor", " ship it ", "tester") + out("do_release %s" % j([res[0], res[1], os.path.basename(res[2]), res[3]])) + out("== release tool") + app = SkillApp(home, root) + app.store = store + app.profile = "t" + tools = tai.Tools(app) + for args in ({"part": "", "message": "m"}, {"part": "patch", "message": ""}, {"part": "huge", "message": "m"}, {"part": "Patch", "message": "tool release"}, {"part": "major", "message": "m" * 300}): + out("release %s => %s" % (j(args), j(tools.dispatch("release", json.dumps(args)).replace(root, "<ROOT>")))) + app.approve = False + out("release denied => %s" % j(tools.dispatch("release", json.dumps({"part": "patch", "message": "no"})))) + app.approve = True + write(script, "nothing here\n") + out("release no version => %s" % j(tools.dispatch("release", json.dumps({"part": "patch", "message": "m"})))) + out("approvals %s" % j(app.approvals)) + out("== capped helpers") + out("capped %s" % j([tai.capped_text(None), tai.capped_text("abc"), tai.capped_text("é" * 20001)[1], tai.capped_text("x" * 20001)[0][-60:], tai.trunc_marker(5)])) + big = os.path.join(root, "big.txt") + write(big, "line\r\nsecond\rthird\n" + "é" * 30000) + for cap in (20000, 50, 100000): + r = tai.read_capped(big, cap) + out("read_capped %d => %s %s" % (cap, j(r[0][:40] if r[0] else r[0]), j(r[1]))) + write(os.path.join(root, "bad.bin"), "") + with open(os.path.join(root, "bad.bin"), "wb") as handle: + handle.write(b"ok \xff\xfe bad \xc3 end") + out("read_capped invalid %s" % j(tai.read_capped(os.path.join(root, "bad.bin")))) + out("read_capped missing %s" % j(tai.read_capped(os.path.join(root, "missing")))) + store.close() + + +def prep_audit(root): + config, seal, store = open_store() + store.audit_event("main", "write", os.path.join("area", "a.txt"), "first write", None, "hello\n", ["file", "Doc"], None, None, "t") + store.audit_event("main", "edit", os.path.join("area", "a.txt"), "edit it", "hello\n", "hello world\n", ["file"], None, None, "t") + store.audit_event("main", "delete", os.path.join("area", "a.txt"), "", "hello world\n", None, [], None, None, "t") + store.audit_event("main", "write", os.path.join("area", "big.txt"), "big", None, "B" * 25000, ["file"], None, None, "t") + store.audit_event("main", "write", os.path.join("area", "sized.txt"), "oversize", None, "small", [], None, 99999, "t") + store.audit_event("main", "snapshot", "/x/tai.py", "backed up", None, None, ["snapshot"], None, None, "t") + store.audit_event("main", "release", "/x/tai.py", "released", "1.0.0", "1.0.1", ["release", "v1.0.1"], None, None, "t") + store.audit_event("main", "write", "sandbox:/box/f.txt", "in box", None, "boxed", [], None, None, "t") + store.audit_event("other", "write", "/o/f.txt", "other profile", None, "o", [], None, None, "other") + store.audit_event("main", "write", os.path.join("area", "sub", "deep", "r.txt"), "restore deep", None, "deep content", ["file"], None, None, "t") + store.audit_event("main", "write", os.path.join("area", "u.txt"), "unicode é alpha", None, "café \U0001f600", ["file"], None, None, "t") + store.audit_event("main", "noop", "/n", "no content", None, None, [], None, None, "t") + store.tag_item("mem:t1", ["alpha", "beta"], "t") + store.tag_item("mem:t2", ["alpha"], "t") + store.tag_item("mem:t3", ["a_b", "a%c", "alphabet"], "t") + store.close() + + +class AuditApp(SkillApp): + def __init__(self, home, root, store): + SkillApp.__init__(self, home, root) + self.store = store + self.profile = "t" + self.env = "home" + + +def sc_audit(root): + config, seal, store = open_store() + app = AuditApp(tai_home(), root, store) + tools = tai.Tools(app) + + def call(name, args): + res = tools.dispatch(name, json.dumps(args)) + out("%s %s => %s" % (name, j(args), j(res.replace(root, "<ROOT>")))) + + out("== audit") + for args in ({}, {"limit": 3}, {"path": os.path.join("area", "a.txt")}, {"tag": "file"}, {"tag": "nomatch"}, {"query": "alpha"}, {"query": "write"}, {"query": "zzzz"}, {"limit": "x"}, {"limit": 500}, {"query": "unicode"}): + call("audit", args) + out("== tags") + for args in ({}, {"prefix": "al"}, {"prefix": "A_"}, {"prefix": "a%"}, {"prefix": "zzz"}, {"limit": 1}, {"limit": "x"}, {"prefix": " Al "}): + call("tags", args) + out("== restore") + for args in ({}, {"id": "x"}, {"id": 9999}, {"id": 6}, {"id": 7}, {"id": 1}, {"id": 3}, {"id": 4}, {"id": 5}, {"id": 12}, {"id": 10}, {"id": 11}): + call("restore", args) + for rel in ("a.txt", "sub/deep/r.txt", "u.txt"): + path = os.path.join("area", rel) + out("file %s %s" % (rel, j(open(path, encoding="utf-8").read() if os.path.exists(path) else None))) + app.approve = False + call("restore", {"id": 2}) + app.approve = True + out("approvals %s" % j([a.replace(root, "<ROOT>") for a in app.approvals])) + store.close() + + +SCENARIOS = { + "release": sc_release, + "audit": sc_audit, + "telegram": sc_telegram, + "telegram_install": sc_telegram_install, + "rag": sc_rag, + "sched": sc_sched, + "install_hooks": sc_install_hooks, + "install_hooks_noshell": sc_install_hooks, + "install_ops": sc_install_ops, + "install_noengine": sc_install_noengine, + "vault_line": sc_vault_line, + "json": sc_json, + "datetime": sc_datetime, + "chunk": sc_chunk, + "fts": sc_fts, + "skills_data": sc_skills_data, + "skills_files": sc_skills_files, + "skills_ops": sc_skills_ops, +} +PREPARE = {"sched": prep_sched, "rag": prep_rag, "release": prep_release, "audit": prep_audit} + + +def main(): + name, root = sys.argv[1], sys.argv[2] + prepare = len(sys.argv) > 3 and sys.argv[3] == "prepare" + os.chdir(os.path.join(root, "proj")) + if prepare: + fn = PREPARE.get(name) + if fn: + fn(root) + return 0 + SCENARIOS[name](root) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tests/misc_test.cpp b/taicpp/tests/misc_test.cpp new file mode 100644 index 0000000..68d3875 --- /dev/null +++ b/taicpp/tests/misc_test.cpp @@ -0,0 +1,29 @@ +// Fast unit checks for the misc modules. The behavioral proof against tai.py is tests/misc_diff.py. +#include <cassert> +#include <cstdio> +#include "bundled.h" +#include "bundled_data.h" +#include "chunker.h" +#include "miscdatetime.h" +#include "miscjson.h" +#include "miscpy.h" +#include "miscsha256.h" +#include "schedtime.h" +#include "skillfile.h" +using namespace detz; + +int main() { + assert(misc::Sha256::hex("abc") == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert(misc::Sha256::hex("") == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assert(misc::Json::loads("{\"a\": [1, 2.5, \"\\u00e9\"]}").dumps() == "{\"a\": [1, 2.5, \"\\u00e9\"]}"); + assert(misc::Json::float_repr(1e16) == "1e+16" && misc::Json::float_repr(0.0001) == "0.0001" && misc::Json::float_repr(1e-5) == "1e-05"); + assert(misc::DateTime::fromisoformat("2026-10-08T09:00Z").isoformat() == "2026-10-08T09:00:00+00:00"); + assert(sched::ScheduleTime::format_delay(3600) == "1h" && sched::ScheduleTime::format_delay(-1) == "overdue"); + assert(*sched::ScheduleTime::parse_duration("2h") == 7200); + assert(rag::Chunker::chunk_lines("a\nb\nc", 2, 0, 100).size() == 2); + assert(skills::BundledData::files().size() == 6); + assert(skills::SkillFiles::skill_name_valid("deep-research") && !skills::SkillFiles::skill_name_valid("Bad")); + assert(skills::BundledSkills::names().size() == 6); + std::puts("misc_test ok"); + return 0; +}