diff --git a/taicpp/src/agent/agent.cpp b/taicpp/src/agent/agent.cpp index 1512b4e..7aadc34 100644 --- a/taicpp/src/agent/agent.cpp +++ b/taicpp/src/agent/agent.cpp @@ -1,21 +1,1149 @@ #include "agent.h" +#include "../llm/consts.h" +#include "../llm/helpers.h" +#include "../llm/py.h" +#include "../llm/term.h" +#include "consts.h" +#include "data.h" +#include "text.h" +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; namespace detz::agent { -void ToolRegistry::add(const ToolSpec& s, Fn fn) { tools_[s.name] = {s, std::move(fn)}; } -std::optional ToolRegistry::call(const std::string& n, const std::map& a) { - auto it = tools_.find(n); if (it == tools_.end()) { return std::nullopt; } - return it->second.second(a); + +using llm::Ansi; +using llm::BackendError; +using llm::paint; +using llm::print_out; +using llm::short_error; + +namespace { +UiPort* ui() { return ports().ui; } +std::string sv(const json::Value& v) { return py::str(v); } +json::Value msg(const char* role, const std::string& content) { + json::Value m = json::Value::object(); + m.set("role", role); + m.set("content", content); + return m; } -std::vector ToolRegistry::list() const { - std::vector v; for (auto& [k, p] : tools_) v.push_back(p.first); return v; +} // namespace + +int& total_step_cap() { + static int cap = TOTAL_STEP_CAP; + return cap; } -Approval ApprovalGate::review(const std::string& t, const std::map&) { - if (auto_) { - return {true, "auto"}; + +// ------------------------------------------------------------------ module helpers +const std::vector& tool_schemas() { + static const std::vector v = [] { + std::vector out; + for (const auto& text : data::ToolSchemaJson()) { out.push_back(json::Value::loads(text)); } + return out; + }(); + return v; +} + +bool tool_env_ok(const std::string& name) { + if (name == "export_terminal_log") { return !llm::env_get("TMUX").empty(); } + if (name == "plexer_send" || name == "plexer_read" || name == "plexer_type" || name == "plexer_status" || name == "plexer_show") { + return ports().plexer != nullptr && ports().plexer->detect() != nullptr; } - return {t == "read", t == "read" ? "safe" : "needs-user"}; + if (name == "tui_screen" || name == "tui_keys" || name == "tui_type" || name == "tui_click" || name == "tui_goto" || name == "tui_result") { + return ports().tui != nullptr && ports().tui->control_exists(); + } + return true; } -GoalVerdict verify_goal(const std::string& g, const std::string& e) { return {!g.empty() && !e.empty(), e}; } -RagStats rag_stats(size_t h, size_t c, double m) { return {h, c, m}; } -Agent::Agent(ToolRegistry r) : reg_(std::move(r)) {} -std::string Agent::run(const std::string& g) { return "done: " + g; } + +static bool is_core(const std::string& name) { + const auto& core = data::CoreTools(); + return std::find(core.begin(), core.end(), name) != core.end(); +} + +std::string tool_catalog() { + std::vector lines = {"", "", "## Tool catalog (core loads always; name a lazy tool or topic to load it)"}; + for (const auto& schema : tool_schemas()) { + std::string name = sv(schema.get("function").get("name")); + if (!tool_env_ok(name)) { continue; } + if (is_core(name)) { continue; } + std::string first = py::head(py::split(sv(schema.get("function").get("description")), ".")[0], 80); + lines.push_back("- " + name + ": " + first); + } + return py::join(lines, "\n"); +} + +std::string conversation_text(const json::Value& messages) { + std::vector parts; + for (const auto& item : messages.items()) { + const json::Value& content = item.get("content"); + const json::Value& role = item.get("role"); + if (role.is_string() && (role.str() == "user" || role.str() == "assistant" || role.str() == "tool") && content.is_string() && + !content.str().empty()) { + parts.push_back(content.str()); + } + } + return py::tail(py::join(parts, "\n"), 6000); +} + +static std::vector name_tokens(const std::string& name) { + std::vector out; + const auto& stop = data::LazyNameStop(); + for (const auto& item : py::split(name, "_")) { + if (py::len(item) >= 4 && std::find(stop.begin(), stop.end(), item) == stop.end()) { out.push_back(item); } + } + return out; +} +static std::set long_words(const std::string& description) { + std::set out; + for (const auto& w : alnum_tokens(description)) { if (py::len(w) >= 6) { out.insert(w); } } + return out; +} +static bool intersects(const std::set& a, const std::set& b) { + for (const auto& x : a) { if (b.count(x)) { return true; } } + return false; +} + +json::Value select_tools(const std::string& text, const json::Value& extra) { + auto toks = alnum_tokens(text); + std::set words(toks.begin(), toks.end()); + json::Value picked = json::Value::array(); + const auto& tags = data::ToolTags(); + for (const auto& schema : tool_schemas()) { + std::string name = sv(schema.get("function").get("name")); + if (!tool_env_ok(name)) { continue; } + if (is_core(name)) { picked.push(schema); continue; } + auto tokens = name_tokens(name); + if (!tokens.empty() && intersects(std::set(tokens.begin(), tokens.end()), words)) { picked.push(schema); continue; } + bool tagged = false; + for (const auto& kv : tags) { + if (kv.first == name) { + tagged = intersects(std::set(kv.second.begin(), kv.second.end()), words); + break; + } + } + if (tagged) { picked.push(schema); continue; } + if (intersects(long_words(sv(schema.get("function").get("description"))), words)) { picked.push(schema); } + } + if (extra.truthy()) { + for (const auto& schema : extra.items()) { + std::string name = sv(schema.get("function").get("name")); + auto tokens = name_tokens(name); + if (!tokens.empty() && intersects(std::set(tokens.begin(), tokens.end()), words)) { picked.push(schema); continue; } + if (intersects(long_words(sv(schema.get("function").get("description"))), words)) { picked.push(schema); } + } + } + return picked; +} + +std::pair compact_messages(const json::Value& messages, llm::ChatClient& chat, int keep) { + if (messages.size() <= static_cast(keep) + 1) { return {messages, false}; } + json::Value system = (messages.at(0).get("role").is_string() && messages.at(0).get("role").str() == "system") + ? messages.at(0) + : msg("system", data::DefaultSystem()); + size_t total = messages.size(); + // recent = messages[-keep:] + size_t rstart = total - static_cast(keep); + while (rstart < total && !(messages.at(rstart).get("role").is_string() && messages.at(rstart).get("role").str() == "user")) { ++rstart; } + if (rstart >= total) { rstart = total - static_cast(keep); } + // middle = messages[1:len(messages) - len(recent)] + size_t recent_len = total - rstart; + size_t mid_end = total - recent_len; + if (mid_end <= 1) { return {messages, false}; } + std::vector digest; + for (size_t i = 1; i < mid_end; ++i) { + const json::Value& item = messages.at(i); + const json::Value& role = item.get("role"); + if (role.is_string() && role.str() == "assistant" && item.get("tool_calls").truthy()) { + std::vector names; + for (const auto& call : item.get("tool_calls").items()) { + const json::Value& fn = call.get("function"); + names.push_back(fn.is_object() && fn.has("name") ? sv(fn.get("name")) : "?"); + } + digest.push_back("assistant tool calls: " + py::join(names, ", ")); + } + digest.push_back(sv(role) + ": " + py::head(py::str_or_empty(item.get("content")), 1500)); + } + json::Value request = json::Value::array(); + request.push(msg("system", data::CompactSystem())); + request.push(msg("user", "Compress this history:\n" + py::join(digest, "\n"))); + int64_t before = llm::conversation_tokens(messages); + std::string summary = py::strip(chat.complete(request, json::Value(), nullptr, true).content); + json::Value rebuilt = json::Value::array(); + rebuilt.push(system); + rebuilt.push(msg("user", "Previous session summary:\n" + summary)); + rebuilt.push(msg("assistant", "Understood. Continuing with summarized context.")); + for (size_t i = rstart; i < total; ++i) { rebuilt.push(messages.at(i)); } + int64_t after = llm::conversation_tokens(rebuilt); + json::Value meta = json::Value::object(); + meta.set("before_tokens", before); + meta.set("after_tokens", after); + meta.set("kept_turns", static_cast(recent_len)); + llm::log_event("INFO", "agent", "compact", + "compacted context " + std::to_string(before) + " to " + std::to_string(after) + " tokens", + chat.scope.first, chat.scope.second, meta); + return {rebuilt, true}; +} + +std::string dir_key(const std::string& directory) { + std::error_code ec; + fs::path base = directory.empty() ? fs::current_path(ec) : fs::path(directory); + if (ec) { return ""; } + // os.path.realpath: resolves symlinks, tolerates missing tails + fs::path real = fs::weakly_canonical(base, ec); + if (ec) { return abspath(base.string()); } + std::string out = real.string(); + while (out.size() > 1 && out.back() == '/') { out.pop_back(); } + return out; +} + +std::string orientation_block(const std::string& directory, bool sandbox) { + if (sandbox) { + return "\n\n## Working directory\nSandboxed container: shell and file tools execute " + "inside the container, not on the host. Stay inside the mounted workdir " + "unless the user names another path."; + } + return "\n\n## Working directory\n" + dir_key(directory) + + "\nYour shell and file tools execute here. Stay inside " + "it unless the user names another path. If your harness reports a different " + "working directory (a scratch or temp dir), ignore it: it is not the workspace."; +} + +std::string capability_block(bool yolo, bool auto_mode) { + if (yolo || auto_mode) { + return "\n\n## Session capabilities\nApprovals are off and every tool is callable, " + "including shell and file writes. Only a failed " + "tool call proves a limit, so act first and adjust on evidence."; + } + return "\n\n## Session capabilities\nEvery tool is callable, including shell and file writes; " + "some commands ask for approval first. Only a failed " + "tool call proves a limit, so act first and adjust on evidence."; +} + +std::pair, std::string> split_leading_tags(const std::string& text) { + std::vector tags; + std::string rest = text; + while (py::startswith(rest, "@")) { + // LEADING_TAG_RE = @(\S+)\s* (re.match at the start) + auto cps = py::decode(rest); + size_t i = 1; + while (i < cps.size() && !py::is_space_cp(cps[i])) { ++i; } + if (i == 1) { break; } + tags.push_back(py::encode(std::vector(cps.begin() + 1, cps.begin() + static_cast(i)))); + while (i < cps.size() && py::is_space_cp(cps[i])) { ++i; } + rest = py::encode(std::vector(cps.begin() + static_cast(i), cps.end())); + } + return {tags, rest}; +} + +std::string expand_file_mentions(const std::string& text, const std::string& directory) { + // FILE_MENTION_RE = (?(cps.begin() + static_cast(i) + 1, cps.begin() + static_cast(j))); + std::string core = py::rstrip_chars(whole, ",;:!?)"); + std::string punct = whole.substr(core.size()); + std::string expanded = expanduser(core); + std::string path = (!expanded.empty() && expanded[0] == '/') ? expanded : directory + "/" + expanded; + if (!core.empty() && path_exists(path)) { + out += abspath(path) + punct; + } else { + out += py::encode(std::vector(cps.begin() + static_cast(i), cps.begin() + static_cast(j))); + } + i = j; + } + return out; +} + +std::set known_model_ids(const AgentConfig& config, const llm::ModelHealth* health) { + std::set ids; + for (const auto& m : llm::ZenFreeModels()) { ids.insert(m); } + for (const auto& m : llm::PollinationsModels()) { ids.insert(m); } + for (const auto& m : llm::DevplaceFreeRoutes()) { ids.insert(m); } + ids.insert(llm::kOpenrouterModel); + for (const auto* list : {&config.openrouter_models, &config.devplace_models, &config.pollinations_models, &config.zen_models, + &config.opencode_models, &config.ollama_models}) { + for (const auto& item : *list) { if (!item.empty()) { ids.insert(item); } } + } + for (const auto& b : config.backends) { if (!b.model.empty()) { ids.insert(b.model); } } + if (!config.explicit_model.empty()) { ids.insert(config.explicit_model); } + if (health != nullptr) { + for (const char* name : {"openrouter:free", "opencode:free", "ollama:local", "devplace:free", "opencode:box:free", "zen:free", "pollinations:free"}) { + auto cached = health->cached_list(name); + if (cached.has_value()) { for (const auto& i : *cached) { ids.insert(i); } } + } + } + return ids; +} + +// ------------------------------------------------------------------ Agent +Agent::Agent(AgentConfig& cfg, StorePort* st, bool persist_, bool quiet_, int depth_, bool auto_, bool yolo_, const std::string& ctx) + : config(cfg), auto_mode(auto_), yolo(yolo_), ctx_id(ctx), store(st), chat(cfg), tools(*this), + persist(persist_), quiet(quiet_), depth(depth_) { + if (!persist_ || depth_ > 0) { chat.fixed_key = "worker/" + py::uuid4_hex(); } + if (ports().skills != nullptr) { + skills = ports().skills->discover_skills(config.home, fs::current_path().string()); + } + switch_profile(config.profile, true); +} + +void Agent::apply_system() { + std::string content = system_message; + content += ports().skills != nullptr ? ports().skills->skill_catalog(skills) : std::string(); + content += tool_catalog(); + content += orientation_block("", env == "sandbox"); + content += capability_block(yolo, auto_mode); + if (auto_mode) { content += "\n\n" + data::AutonomousNote(); } + if (messages.size() == 0) { messages.push(msg("system", content)); return; } + messages.items()[0] = msg("system", content); +} + +void Agent::sync_chat_scope() { chat.scope = {profile, bot}; } + +void Agent::reset_history() { + messages = json::Value::array(); + messages.push(msg("system", system_message)); + apply_system(); + apply_system(); +} + +void Agent::switch_profile(const std::string& name, bool silent) { + if (messages.size() > 0 && persist) { store->save_session(profile, messages, bot, ctx_id); } + profile = name; + bot = "main"; + store->set_profile(name); + tools.read_files.clear(); + tools.secret_grants.clear(); + auto loaded = store->load_system(name); + std::string text; + if (!loaded.has_value()) { + text = data::DefaultSystem(); + store->save_system(name, text); + } else { + text = *loaded; + } + if (text.find(data::LegacyPasswordNote()) != std::string::npos) { + text = py::replace(text, data::LegacyPasswordNote(), data::VaultMemoryNote()); + store->save_system(name, text); + } + system_message = text; + json::Value restored = persist ? store->load_session(name, "main", ctx_id) : json::Value::array(); + messages = json::Value::array(); + messages.push(msg("system", system_message)); + for (const auto& m : restored.items()) { messages.push(m); } + apply_system(); + sync_chat_scope(); + if (!silent) { + print_out(paint("profile: " + name + " (" + std::to_string(py::len(system_message)) + " chars knowledge, " + + std::to_string(restored.size()) + " restored messages)", Ansi::GREEN)); + } +} + +void Agent::update_system(const std::string& text) { + system_message = text; + if (bot == "main") { store->save_system(profile, text); } + else { store->save_bot_system(profile, bot, text); } + apply_system(); + store->log_event(profile, "system", "remember", "system message updated (" + std::to_string(py::len(text)) + " chars)"); +} + +std::string Agent::switch_bot(const std::string& name, bool silent) { + auto target = store->resolve_bot(profile, name); + if (!target.has_value()) { + std::vector names; + for (const auto& item : store->list_bots(profile).items()) { names.push_back(sv(item.get("name"))); } + return "unknown bot @" + name + " (available: " + py::join(names, ", ") + ")"; + } + if (messages.size() > 0 && persist) { store->save_session(profile, messages, bot, ctx_id); } + bot = *target; + auto loaded = store->load_bot_system(profile, *target); + std::string text; + if (!loaded.has_value()) { + text = data::DefaultSystem(); + if (*target == "main") { store->save_system(profile, text); } + else { store->save_bot_system(profile, *target, text); } + } else { + text = *loaded; + } + system_message = text; + json::Value restored = persist ? store->load_session(profile, *target, ctx_id) : json::Value::array(); + messages = json::Value::array(); + messages.push(msg("system", system_message)); + for (const auto& m : restored.items()) { messages.push(m); } + apply_system(); + sync_chat_scope(); + if (!silent) { + print_out(paint("bot: " + *target + " (" + std::to_string(restored.size()) + " restored messages)", Ansi::GREEN)); + } + return "switched to bot '" + *target + "'"; +} + +void Agent::scrub_secret_from_history() { + for (auto& item : messages.items()) { + const json::Value& content = item.get("content"); + if (content.is_string() && !content.str().empty()) { item.set("content", store->redact(content.str(), profile)); } + if (item.get("tool_calls").truthy()) { + for (auto& recorded : item["tool_calls"].items()) { + json::Value& fn = recorded["function"]; + if (fn.get("arguments").is_string() && !fn.get("arguments").str().empty()) { + fn.set("arguments", store->redact(fn.get("arguments").str(), profile)); + } + } + } + } + chat.reset_muse(); +} + +bool Agent::ask_approval(const std::string& command, bool guidance) { + std::string mode; + if (yolo || auto_mode || !persist) { mode = "auto"; } + else if (prompter) { mode = "prompter"; } + else if (llm::stdin_isatty()) { mode = "tty"; } + else { mode = "headless"; } + auto decide = [&](bool value) { + json::Value meta = json::Value::object(); + meta.set("command", py::head(command, 300)); + meta.set("decision", value); + meta.set("mode", mode); + llm::log_event("INFO", "approval", "decision", std::string("approval ") + (value ? "granted" : "denied"), profile, bot, meta); + return value; + }; + if (yolo || auto_mode) { return decide(true); } + if (!persist) { return decide(config.auto_approve); } + if (prompter) { + auto choice = prompter("approve", command, {"Allow once", "Always allow (approvals off)", "Deny"}); + if (choice.has_value() && *choice == "Always allow (approvals off)") { + user_stepped_in = true; + yolo = true; + config.auto_approve = true; + return decide(true); + } + if (choice.has_value() && *choice == "Allow once") { + user_stepped_in = true; + return decide(true); + } + return decide(false); + } + if (!llm::stdin_isatty()) { return decide(false); } + print_out(paint("run: " + command, Ansi::YELLOW)); + std::string decision = "unknown"; + for (int retry = 0; retry < APPROVAL_MAX_RETRIES; ++retry) { + std::fputs(paint("allow? [y]once [Y]always [n]o ", Ansi::YELLOW).c_str(), stdout); + std::fflush(stdout); + std::string answer; + if (!std::getline(std::cin, answer)) { return decide(false); } + answer = py::strip(answer); + decision = Approval::parse_allow(answer); + if (decision != "unknown") { break; } + print_out(paint("unrecognized answer " + py::repr_str(answer) + ", use y (once), Y (always) or n (deny)", Ansi::YELLOW)); + } + user_stepped_in = true; + if (decision == "always") { + yolo = true; + config.auto_approve = true; + print_out(paint("yolo mode on, approvals off for this session", Ansi::YELLOW)); + return decide(true); + } + if (decision == "once") { return decide(true); } + if (!guidance || auto_mode) { return decide(false); } + std::fputs(paint("denied. What should I do instead? (empty aborts this turn) ", Ansi::YELLOW).c_str(), stdout); + std::fflush(stdout); + std::string typed; + if (!std::getline(std::cin, typed)) { typed.clear(); } + typed = py::strip(typed); + decide(false); + throw Denied(typed); +} + +void Agent::show_call(const llm::ToolCall& call) { + if (quiet) { return; } + std::string preview_args = llm::tool_args_text(call.arguments); + if (call.name == "store_secret") { + try { + json::Value masked = json::Value::loads(!preview_args.empty() ? preview_args : "{}"); + if (!masked.is_object()) { + if (masked.is_array()) { throw std::runtime_error("list indices must be integers or slices, not str"); } + throw std::runtime_error("'" + py_type_name(masked) + "' object does not support item assignment"); + } + masked.set("value", "[hidden]"); + preview_args = masked.dumps(); + } catch (const json::ParseError&) { + preview_args = "[hidden]"; + } + } + if (ui() != nullptr && ui()->glow_enabled()) { + GlowCall d = ui()->glow_describe_call(call.name, preview_args); + std::string target = store->redact(d.target, profile); + std::string extra = store->redact(d.extra, profile); + std::string preview = py::replace(py::head(store->redact(d.pretty, profile), 400), "\n", " "); + last_event_ = LastEvent{call.name, d.verb, target}; + print_out(""); + int inner = std::max(20, ui()->glow_width() - 2); + print_out(ui()->glow_bar(call.name) + ui()->glow_event_row(ui()->glow_icon_for(call.name), d.verb, target, extra, inner, ui()->glow_accent_for(call.name))); + print_out(ui()->glow_bar(call.name) + ui()->glow_preview_row(preview, inner)); + return; + } + std::string preview = py::replace(py::head(store->redact(preview_args, profile), 400), "\n", " "); + print_out(paint("\xE2\x94\x8C\xE2\x94\x80 " + call.name, Ansi::MAGENTA)); + print_out(paint("\xE2\x94\x82 " + preview, Ansi::DIM)); +} + +void Agent::show_diff() { + if (!tools.diff_preview.has_value()) { return; } + DiffPreview preview = *tools.diff_preview; + tools.diff_preview.reset(); + if (ui() == nullptr) { return; } + std::string rendered = ui()->render_diff(preview.path, preview.old_text, preview.new_text, ui()->color_enabled()); + if (!rendered.empty()) { print_out(rendered); } +} + +void Agent::show_result(const std::string& result, double elapsed_ms) { + if (quiet) { return; } + show_diff(); + if (tools.result_note.has_value() && !tools.result_note->empty()) { + std::string note = *tools.result_note; + tools.result_note.reset(); + print_out(paint(note, Ansi::DIM)); + } else { + tools.result_note.reset(); + } + if (ui() != nullptr && ui()->glow_enabled()) { + LastEvent ev = last_event_.value_or(LastEvent{"", "Done", ""}); + bool failed = result == "denied by user" || py::startswith(result, "error:") || py::startswith(result, "[error]") || + py::startswith(result, "[missing]"); + if (ev.name == "fork" && py::startswith(result, "agent ")) { + size_t i = 6; + while (i < result.size() && std::isdigit(static_cast(result[i]))) { ++i; } + if (i > 6 && result.compare(i, 8, " started") == 0) { + std::string num = result.substr(6, i - 6); + ev.target = !ev.target.empty() ? ev.target + " \xC2\xB7 #" + num : "#" + num; + } + } + int inner = std::max(20, ui()->glow_width() - 2); + print_out(ui()->glow_bar(ev.name) + ui()->glow_done_row(ev.verb, ev.target, py::len(result), elapsed_ms, failed, inner)); + last_event_.reset(); + show_lanes(ev.name, result); + return; + } + print_out(paint("\xE2\x94\x94\xE2\x94\x80 " + std::to_string(py::len(result)) + " chars \xC2\xB7 " + + std::to_string(static_cast(elapsed_ms)) + "ms", Ansi::DIM)); +} + +void Agent::show_lanes(const std::string& name, const std::string& result) { + bool boarded = tools.board_shown; + tools.board_shown = false; + bool started = name == "fork" && py::startswith(result, "agent "); + bool collected = name == "poll" && (py::startswith(result, "[done]") || py::startswith(result, "[timeout]")); + bool watching = name == "poll" && py::startswith(result, "[running]"); + if (!(started || collected || watching)) { return; } + if (ui() == nullptr) { return; } + json::Value records = list_agents(profile); + size_t running = 0, pending = 0; + for (const auto& r : records.items()) { + std::string st = sv(r.get("status")); + if (st == "running") { ++running; } + if ((st == "done" || st == "timeout" || st == "error") && !r.get("collected").truthy()) { ++pending; } + } + if (boarded) { + if (collected && running == 0 && pending == 0) { print_out(ui()->glow_event_row("\xE2\x97\x86", "lanes", "", "all collected", 0, "")); } + return; + } + if (running >= 2 || pending > 0) { + for (const auto& row : ui()->glow_lanes(records)) { print_out(row); } + } else if (collected && running == 0 && pending == 0) { + print_out(ui()->glow_event_row("\xE2\x97\x86", "lanes", "", "all collected", 0, "")); + } +} + +void Agent::checkpoint() { + if (!persist) { return; } + store->save_session(profile, messages, bot, ctx_id); + if (turn_open_) { + json::Value st = json::Value::object(); + st.set("open", true); + st.set("goal", turn_goal_); + st.set("steps", turn_steps_); + st.set("tools", turn_tools_); + st.set("updated", py::now_iso()); + store->save_turn_state(profile, bot, st, ctx_id); + } +} + +void Agent::close_turn() { + if (!persist) { turn_open_ = false; return; } + store->save_session(profile, messages, bot, ctx_id); + turn_open_ = false; + if (!turn_goal_.empty()) { + json::Value st = json::Value::object(); + st.set("open", false); + st.set("goal", turn_goal_); + st.set("steps", turn_steps_); + st.set("tools", turn_tools_); + st.set("updated", py::now_iso()); + store->save_turn_state(profile, bot, st, ctx_id); + } +} + +void Agent::phase(const std::string& name, const std::string& detail) { + if (!on_phase) { return; } + try { + on_phase(name, detail); + } catch (...) { + } +} + +// ------------------------------------------------------------------ mentions +std::string Agent::mention_error(const std::string& tag) { + std::vector names; + for (const auto& item : store->list_bots(profile).items()) { names.push_back(sv(item.get("name"))); } + return "unknown bot @" + tag + " (available: " + py::join(names, ", ") + "; providers: " + py::join(llm::ProviderLabels(), ", ") + + "; @path references a file)"; +} + +std::string Agent::mention_confirm(const std::string& provider, const std::string& model, const std::string& bot_sel, + bool cleared, const std::string& directory) { + std::vector bits; + if (cleared && provider.empty() && model.empty()) { bits.push_back("pins cleared"); } + if (!provider.empty()) { bits.push_back("provider pinned to " + provider); } + if (!model.empty()) { bits.push_back("model pinned to " + model); } + if (!bot_sel.empty()) { bits.push_back("switched to bot '" + bot_sel + "'"); } + if (bits.empty()) { bits.push_back("nothing to select"); } + std::string suffix = persist ? "saved as default for " + directory : "not saved (persist off)"; + return py::join(bits, "; ") + " (" + suffix + ")"; +} + +ResolvedTag Agent::resolve_tag(const std::string& tag, const std::string& directory) { + std::string word = py::strip(tag); + std::string lowered = py::lower(word); + auto target = store->resolve_bot(profile, lowered); + if (target.has_value()) { return {"bot", *target, ""}; } + const auto& labels = llm::ProviderLabels(); + if (std::find(labels.begin(), labels.end(), lowered) != labels.end()) { return {"provider", lowered, ""}; } + if (lowered == "molodetz") { return {"route", llm::kDevplaceLabel, llm::kPreferredModel}; } + for (const auto& b : config.backends) { + if (py::lower(py::strip(b.label)) == lowered) { return {"provider", lowered, ""}; } + } + auto ids = known_model_ids(config, &chat.health); + if (ids.count(word)) { return {"model", word, ""}; } + if (lowered == llm::kMentionAuto) { return {"auto", llm::kMentionAuto, ""}; } + std::string core = py::rstrip_chars(word, ",;:!?)"); + std::string expanded = expanduser(core); + std::string path = (!expanded.empty() && expanded[0] == '/') ? expanded : directory + "/" + expanded; + if (!core.empty() && path_exists(path)) { return {"file", abspath(path), ""}; } + return {"unknown", word, ""}; +} + +std::pair, std::optional> Agent::apply_mentions(const std::string& text, bool capture, int max_steps) { + auto [tags, rest] = split_leading_tags(text); + std::string directory = dir_key(); + std::optional sel_provider, sel_model, sel_bot; + bool cleared = false; + std::vector files; + bool loud = !capture && !quiet; + for (const auto& tag : tags) { + ResolvedTag r = resolve_tag(tag, directory); + if (r.kind == "unknown") { + std::string error = mention_error(tag); + if (loud) { print_out(error); } + return {text, error}; + } + if (r.kind == "auto") { cleared = true; sel_provider.reset(); sel_model.reset(); continue; } + if (r.kind == "file") { files.push_back(r.value); continue; } + if (r.kind == "route") { sel_provider = r.value; sel_model = r.value2; continue; } + if (r.kind == "provider") { sel_provider = r.value; } + else if (r.kind == "model") { sel_model = r.value; } + else if (r.kind == "bot") { sel_bot = r.value; } + } + auto truthy = [](const std::optional& s) { return s.has_value() && !s->empty(); }; + if (cleared || sel_provider.has_value()) { chat.provider_pin = sel_provider; } + if (cleared || sel_model.has_value()) { chat.model_pin = sel_model; } + if ((cleared || truthy(sel_provider) || truthy(sel_model) || truthy(sel_bot)) && persist) { + std::vector clear; + if (cleared && !sel_provider.has_value()) { clear.push_back("provider"); } + if (cleared && !sel_model.has_value()) { clear.push_back("model"); } + store->save_dir_defaults(directory, sel_provider, sel_model, sel_bot, clear); + } + std::string prompt = expand_file_mentions(rest, directory); + if (!files.empty()) { + std::vector parts = files; + if (!py::strip(prompt).empty()) { parts.push_back(prompt); } + prompt = py::join(parts, " "); + } + if (tags.empty()) { return {prompt, std::nullopt}; } + if (py::strip(prompt).empty()) { + if (sel_bot.has_value() && *sel_bot != bot) { switch_bot(*sel_bot, true); } + std::string confirm = mention_confirm(sel_provider.value_or(""), sel_model.value_or(""), sel_bot.value_or(""), cleared, directory); + if (loud) { print_out(confirm); } + return {std::nullopt, confirm}; + } + if (sel_bot.has_value() && *sel_bot != bot) { + return {std::nullopt, run_mention(*sel_bot, prompt, text, capture, max_steps)}; + } + return {prompt, std::nullopt}; +} + +// ------------------------------------------------------------------ the loop +std::string Agent::run_turn(const std::string& text, bool capture, int max_steps, bool routed) { + json::Value meta = json::Value::object(); + meta.set("goal", py::head(text, 500)); + meta.set("capture", capture); + meta.set("max_steps", max_steps); + meta.set("depth", depth); + meta.set("routed", routed); + llm::log_event("INFO", "agent", "turn_start", "turn started", profile, bot, meta); + double started = py::now(); + std::string result; + try { + result = run_turn_inner(text, capture, max_steps, routed); + } catch (const std::exception& exc) { + json::Value m = json::Value::object(); + m.set("exec_ms", static_cast((py::now() - started) * 1000)); + m.set("steps", turn_steps_); + m.set("tools_used", turn_tools_); + m.set("outcome", "error"); + m.set("reply_chars", 0); + llm::log_event("INFO", "agent", "turn_end", "turn failed: " + short_error(exc), profile, bot, m); + throw; + } + json::Value m = json::Value::object(); + m.set("exec_ms", static_cast((py::now() - started) * 1000)); + m.set("steps", turn_steps_); + m.set("tools_used", turn_tools_); + m.set("outcome", "ok"); + m.set("reply_chars", static_cast(py::len(result))); + llm::log_event("INFO", "agent", "turn_end", "turn finished", profile, bot, m); + return result; +} + +std::string Agent::run_turn_inner(const std::string& in_text, bool capture, int max_steps, bool routed) { + std::string text = in_text; + apply_system(); + if (!routed) { + auto [prompt, result] = apply_mentions(text, capture, max_steps); + if (result.has_value()) { return *result; } + text = *prompt; + } + messages.push(msg("user", text)); + store->log_event(profile, "user", "message", store->redact(text, profile)); + turn_open_ = true; + turn_goal_ = py::head(text, 500); + turn_steps_ = 0; + turn_tools_ = 0; + checkpoint(); + double started = py::now(); + std::vector backends; + std::string last_text; + bool loud = !capture && !quiet; + llm::StreamSink sink = [](const std::string&) {}; // pieces.append: collected, never read + tools.live = loud; + std::deque seen_actions, seen_results; + int stall = 0; + std::optional> last_pair; + int run_len = 0; + bool nudged = false; + bool broke = false; + for (int step = 0; step < total_step_cap(); ++step) { + if (deadline.has_value() && py::now() > *deadline) { + timed_out = true; + last_text = py::strip(last_text + "\n[time limit reached]"); + broke = true; + break; + } + json::Value payload = select_tools(conversation_text(messages), tools.mcp_schemas()); + int64_t tokens = llm::conversation_tokens(messages, payload); + if (static_cast(tokens) > CONTEXT_CAP * COMPACT_RATIO) { + if (loud) { print_out(paint("compacting context (" + std::to_string(tokens) + " tokens)...", Ansi::DIM)); } + messages = compact_messages(messages, chat).first; + apply_system(); + } + bool spinning = false; + if (loud && ui() != nullptr) { ui()->spinner_start("thinking"); spinning = true; } + phase("thinking"); + llm::ChatReply reply; + try { + reply = chat.complete(messages, payload, &sink, false, deadline); + } catch (const BackendError& exc) { + if (spinning) { ui()->spinner_stop(); } + if (loud) { print_out(paint(std::string("backend error: ") + exc.what(), Ansi::RED)); } + if (messages.size() > 0) { messages.items().pop_back(); } + checkpoint(); + return !last_text.empty() ? last_text : std::string("backend error: ") + exc.what(); + } catch (...) { + if (spinning) { ui()->spinner_stop(); } + throw; + } + if (spinning) { ui()->spinner_stop(); } + reply.content = store->redact(reply.content, profile); + if (loud && !py::strip(reply.content).empty()) { + print_out(ui() != nullptr ? ui()->render_markdown(reply.content) : reply.content); + } + if (std::find(backends.begin(), backends.end(), reply.backend) == backends.end()) { + backends.push_back(reply.backend); + if (loud && (backends.size() > 1 || reply.backend != llm::kMuseLabel)) { + print_out(paint("(via " + reply.backend + ")", Ansi::DIM)); + } + } + if (loud && !reply.reasoning.empty() && reply.content.empty() && reply.tool_calls.empty()) { + print_out(paint(py::head(reply.reasoning, 2000), Ansi::GRAY)); + } + { + std::vector valid; + for (const auto& c : reply.tool_calls) { if (llm::tool_name_valid(c.name)) { valid.push_back(c); } } + reply.tool_calls = valid; + } + json::Value history_calls = json::Value::array(); + for (const auto& c : reply.tool_calls) { + json::Value fn = json::Value::object(); + fn.set("name", c.name); + fn.set("arguments", c.arguments); + json::Value call = json::Value::object(); + call.set("id", c.id); + call.set("type", "function"); + call.set("function", fn); + history_calls.push(call); + } + { + json::Value am = json::Value::object(); + am.set("role", "assistant"); + am.set("content", reply.content.empty() ? json::Value() : json::Value(reply.content)); + am.set("tool_calls", history_calls.size() == 0 ? json::Value() : history_calls); + messages.push(am); + } + if (reply.tool_calls.empty()) { + if (!reply.content.empty()) { last_text = reply.content; } + else if (!reply.reasoning.empty()) { last_text = reply.reasoning; } + if (!last_text.empty()) { + phase("typing"); + store->log_event(profile, "assistant", "message", last_text); + } + broke = true; + break; + } + bool aborted = false; + std::vector> step_pairs; + for (size_t pos = 0; pos < reply.tool_calls.size(); ++pos) { + const llm::ToolCall& call = reply.tool_calls[pos]; + show_call(call); + phase("tool", call.name); + double call_started = py::now(); + std::string result; + try { + result = tools.dispatch(call.name, json::Value(call.arguments)); + } catch (const Denied& denied) { + result = "denied by user"; + show_result(result, (py::now() - call_started) * 1000); + { + json::Value tm = json::Value::object(); + tm.set("role", "tool"); + tm.set("tool_call_id", call.id); + tm.set("content", result); + messages.push(tm); + } + store->log_event(profile, "tool", call.name, result); + step_pairs.emplace_back(llm::action_signature(call.name, call.arguments), llm::result_digest(result)); + for (size_t k = pos + 1; k < reply.tool_calls.size(); ++k) { + json::Value tm = json::Value::object(); + tm.set("role", "tool"); + tm.set("tool_call_id", reply.tool_calls[k].id); + tm.set("content", "skipped: stopped after denial"); + messages.push(tm); + } + if (!denied.guidance().empty()) { + messages.push(msg("user", denied.guidance())); + store->log_event(profile, "user", "guidance", store->redact(denied.guidance(), profile)); + } else { + aborted = true; + last_text = "stopped by user"; + } + break; + } + show_result(result, (py::now() - call_started) * 1000); + { + json::Value tm = json::Value::object(); + tm.set("role", "tool"); + tm.set("tool_call_id", call.id); + tm.set("content", result); + messages.push(tm); + } + store->log_event(profile, "tool", call.name, result); + step_pairs.emplace_back(llm::action_signature(call.name, call.arguments), llm::result_digest(result)); + if (call.name == "store_secret") { scrub_secret_from_history(); } + } + if (aborted) { broke = true; break; } + bool productive = user_stepped_in; + user_stepped_in = false; + for (const auto& pair : step_pairs) { + bool new_action = std::find(seen_actions.begin(), seen_actions.end(), pair.first) == seen_actions.end(); + bool new_result = std::find(seen_results.begin(), seen_results.end(), pair.second) == seen_results.end(); + if (new_action || new_result) { productive = true; } + seen_actions.push_back(pair.first); + seen_results.push_back(pair.second); + if (seen_actions.size() > 400) { seen_actions.pop_front(); } + if (seen_results.size() > 400) { seen_results.pop_front(); } + if (last_pair.has_value() && *last_pair == pair) { + run_len += 1; + } else { + last_pair = pair; + run_len = 1; + nudged = false; + } + } + turn_steps_ += 1; + turn_tools_ += static_cast(step_pairs.size()); + checkpoint(); + if (productive) { stall = 0; } else { stall += 1; } + if (run_len == LOOP_NUDGE_AT && !nudged) { + nudged = true; + std::string nudge = "loop warning: you called " + py::head(last_pair->first, 200) + " " + std::to_string(run_len) + + " times in a row with identical results. Change approach, try different arguments, or stop instead of repeating it."; + messages.push(msg("user", nudge)); + store->log_event(profile, "user", "guidance", nudge); + if (loud) { print_out(paint(nudge, Ansi::YELLOW)); } + } + if (run_len >= LOOP_STOP_AT) { + last_text = "stuck in a loop: " + py::head(last_pair->first, 200) + " repeated " + std::to_string(run_len) + " times with identical results"; + if (loud) { print_out(paint(last_text, Ansi::YELLOW)); } + store->log_event(profile, "system", "stall", last_text); + broke = true; + break; + } + if (stall >= max_steps) { + last_text = "no progress for " + std::to_string(max_steps) + " steps: actions and results keep repeating. Say 'continue' to resume with fresh direction."; + if (loud) { print_out(paint(last_text, Ansi::YELLOW)); } + store->log_event(profile, "system", "stall", last_text); + broke = true; + break; + } + } + if (!broke) { + if (loud) { print_out(paint("step budget exhausted (" + std::to_string(total_step_cap()) + " total steps)", Ansi::YELLOW)); } + } + close_turn(); + double total = py::now() - started; + if (loud) { + if (ui() != nullptr && ui()->glow_enabled()) { print_out(ui()->glow_hairline()); } + char buf[32]; + std::snprintf(buf, sizeof buf, "%.1f", total); + print_out(paint("tokens\xE2\x89\x88" + std::to_string(llm::conversation_tokens(messages)) + " \xC2\xB7 " + py::join(backends, "+") + + " \xC2\xB7 " + buf + "s", Ansi::DIM)); + } + return last_text; +} + +std::pair Agent::run_verified(const std::string& prompt) { + if (depth != 0) { return {run_turn(prompt), true}; } + return verify_loop(prompt, run_turn(prompt)); +} + +bool Agent::ensure_verified(const std::string& goal, const std::string& final_text) { + auto [ft, success] = verify_loop(goal, final_text); + (void)ft; + int64_t failures = success ? 0 : last_verify_failures; + json::Value meta = json::Value::object(); + meta.set("success", success); + meta.set("failures", failures); + llm::log_event("INFO", "agent", "verified", std::string("goal ") + (success ? "delivered" : "not delivered"), profile, bot, meta); + return success; +} + +std::pair Agent::verify_loop(const std::string& in_goal, const std::string& in_final) { + std::string goal = py::head(py::join(py::split_ws(in_goal), " "), VERIFY_MAX_GOAL); + std::string final_text = in_final; + if (final_text == "stopped by user") { + mark_turn_open(goal); + return {final_text, false}; + } + for (int attempt = 0; attempt < 1 + VERIFY_MAX_ROUNDS; ++attempt) { + GoalVerdict verdict = verify_goal(goal, final_text); + last_verify_failures = static_cast(verdict.failures.size()); + show_verdict(verdict, goal); + store->log_event(profile, "system", "verify", verdict.ok ? std::string("ok") : "failed: " + std::to_string(verdict.failures.size()) + " items"); + if (verdict.ok) { return {final_text, true}; } + if (attempt >= VERIFY_MAX_ROUNDS) { break; } + final_text = run_turn(verdict.followup()); + if (final_text == "stopped by user") { + mark_turn_open(goal); + return {final_text, false}; + } + } + mark_turn_open(goal); + return {final_text, false}; +} + +void Agent::mark_turn_open(const std::string& goal) { + if (!persist) { return; } + json::Value st = json::Value::object(); + st.set("open", true); + st.set("goal", py::head(goal, 500)); + st.set("steps", turn_steps_); + st.set("tools", turn_tools_); + st.set("updated", py::now_iso()); + store->save_turn_state(profile, bot, st, ctx_id); +} + +GoalVerdict Agent::verify_goal(const std::string& goal, const std::string& final_text) { + std::string task = GoalVerifier::build_task(goal, final_text, GoalVerifier::transcript(messages), GoalVerifier::tried_actions(messages)); + Runner runner = runner_override; + if (!runner) { + AgentConfig* cfg = &config; + int next_depth = depth + 1; + runner = [cfg, next_depth](const std::string& t, const std::string& p, double timeout) { + return default_runner(*cfg, p, next_depth, t, timeout, VERIFY_STEPS); + }; + } + auto fail = [](const std::string& reason, const std::string& proposal, const std::string& raw) { + GoalVerdict v; + v.ok = false; + v.failures = {GoalFailure{"goal verification", reason, proposal}}; + v.raw = raw; + return v; + }; + int agent_id; + try { + agent_id = spawn_agent(task, profile, VERIFY_TIMEOUT, config, depth + 1, runner); + } catch (const std::exception& exc) { + return fail("verifier did not start: " + short_error(exc), "re-run the request", ""); + } + auto [status, text] = poll_agent(agent_id, VERIFY_TIMEOUT, profile); + if (status == "done" || status == "timeout" || status == "error") { + std::lock_guard guard(AgentRegistry::lock()); + auto it = AgentRegistry::records().find(agent_id); + if (it != AgentRegistry::records().end()) { it->second->collected = true; } + } + if (status != "done") { + return fail("verifier " + status + ": " + py::head(py::join(py::split_ws(text), " "), 200), "re-run the verification", + py::head(text, VERIFY_MAX_COMMENTARY)); + } + std::string full; + { + std::lock_guard guard(AgentRegistry::lock()); + auto it = AgentRegistry::records().find(agent_id); + if (it != AgentRegistry::records().end() && !it->second->result.empty()) { full = it->second->result; } + } + if (full.empty()) { full = text; } + return GoalVerdict::from_text(full); +} + +void Agent::show_verdict(const GoalVerdict& verdict, const std::string& goal) { + if (quiet) { return; } + print_out(paint(verdict.report(goal), verdict.ok ? Ansi::GREEN : Ansi::YELLOW)); +} + +std::string Agent::run_mention(const std::string& target, const std::string& rest, const std::string& original, bool capture, int max_steps) { + json::Value other = json::Value::array(); + if (persist) { + store->save_session(profile, messages, bot, ctx_id); + other = store->load_session(profile, target, ctx_id); + } + auto sys = store->load_bot_system(profile, target); + std::string system = (sys.has_value() && !sys->empty()) ? *sys : data::DefaultSystem(); + json::Value saved_messages = messages; + std::string saved_system = system_message, saved_bot = bot; + messages = json::Value::array(); + messages.push(msg("system", system)); + for (const auto& m : other.items()) { messages.push(m); } + system_message = system; + bot = target; + apply_system(); + sync_chat_scope(); + std::string answer; + try { + answer = run_turn(rest, capture, max_steps, true); + } catch (...) { + if (persist) { store->save_session(profile, messages, target, ctx_id); } + messages = saved_messages; + system_message = saved_system; + bot = saved_bot; + apply_system(); + sync_chat_scope(); + throw; + } + if (persist) { store->save_session(profile, messages, target, ctx_id); } + messages = saved_messages; + system_message = saved_system; + bot = saved_bot; + apply_system(); + sync_chat_scope(); + store->log_event(profile, "user", "message", store->redact(original, profile)); + messages.push(msg("user", original)); + { + json::Value am = json::Value::object(); + am.set("role", "assistant"); + am.set("content", answer); + am.set("reasoning", ""); + am.set("tool_calls", json::Value::array()); + am.set("backend", "mention"); + messages.push(am); + } + if (persist) { store->save_session(profile, messages, bot, ctx_id); } + return answer; +} + +// ------------------------------------------------------------------ runners +std::string default_runner(AgentConfig& config, const std::string& profile, int depth, const std::string& task, double timeout, + int max_steps, AgentRecord* stats) { + if (ports().runtime == nullptr) { throw PortNotWired("runtime"); } + std::unique_ptr store = ports().runtime->open_store(); + ports().runtime->boot_settings(*store); + Agent worker(config, store.get(), false, true, depth); + if (profile != worker.profile) { worker.switch_profile(profile, true); } + worker.deadline = py::now() + timeout; + std::string result; + try { + result = worker.run_turn(task, true, max_steps); + } catch (...) { + if (stats != nullptr) { + std::lock_guard guard(AgentRegistry::lock()); + stats->tools = worker.tools.calls; + } + throw; + } + if (stats != nullptr) { + std::lock_guard guard(AgentRegistry::lock()); + stats->tools = worker.tools.calls; + } + return result; +} + +std::string resume_turn(Agent& agent) { + json::Value state = agent.store->load_turn_state(agent.profile, agent.bot, agent.ctx_id); + if (!state.get("open").truthy()) { + print_out("nothing to resume: last turn for " + agent.profile + "/" + agent.bot + " completed normally"); + return ""; + } + std::string goal = py::str_or_empty(state.get("goal")); + print_out("resuming interrupted turn for " + agent.profile + "/" + agent.bot + ": " + + (state.get("steps").truthy() ? py::str(state.get("steps")) : "0") + " steps, " + + (state.get("tools").truthy() ? py::str(state.get("tools")) : "0") + " tool calls, goal: " + py::head(goal, 200)); + std::string message = "Continue where you left off" + (goal.empty() ? std::string() : ": the interrupted request was: " + goal) + + ". Do not repeat steps already completed above; carry on with the next step."; + return agent.run_turn(message); +} + +int update_models_command(AgentConfig& config) { + llm::ChatClient chat(config); + bool failed = false; + for (const auto& row : chat.update_models()) { + if (!row.count.has_value()) { + failed = true; + print_out(row.label + ": skipped, source unreachable or empty, old list kept"); + continue; + } + print_out(row.label + ": " + std::to_string(*row.count) + " models, +" + std::to_string(row.added.size()) + " new, -" + + std::to_string(row.removed.size()) + " removed"); + for (const auto& item : row.added) { print_out(" + " + item); } + for (const auto& item : row.removed) { print_out(" - " + item); } + } + return failed ? 1 : 0; +} + } // namespace detz::agent diff --git a/taicpp/src/agent/agent.h b/taicpp/src/agent/agent.h index 2a5b0e7..f2c70ed 100644 --- a/taicpp/src/agent/agent.h +++ b/taicpp/src/agent/agent.h @@ -1,38 +1,141 @@ #pragma once +// The Agent of tai.py (19910-20535): profile/bot/session handling, approval +// gating, mentions, the turn loop (run_turn), strict goal verification, plus +// the module-level helpers select_tools / tool_catalog / compact_messages. +#include "../llm/chat_client.h" +#include "../llm/json.h" +#include "agent_config.h" +#include "consts.h" +#include "goal.h" +#include "ports.h" +#include "registry.h" +#include "tools.h" +#include #include -#include +#include +#include #include #include #include namespace detz::agent { -struct ToolResult { bool ok{true}; std::string out; }; -struct ToolSpec { std::string name, desc; }; -class ToolRegistry { - public: - using Fn = std::function&)>; - void add(const ToolSpec& s, Fn fn); - std::optional call(const std::string& name, const std::map& args); - std::vector list() const; - private: - std::map> tools_; -}; -struct Approval { bool approved{false}; std::string reason; }; -class ApprovalGate { - public: - Approval review(const std::string& tool, const std::map& args); - void set_auto(bool v) { auto_ = v; } - private: - bool auto_{false}; -}; -struct GoalVerdict { bool met{false}; std::string note; }; -GoalVerdict verify_goal(const std::string& goal, const std::string& evidence); -struct RagStats { size_t hits{0}, chunks{0}; double ms{0}; }; -RagStats rag_stats(size_t hits, size_t chunks, double ms); + +// prompter(kind, text, options) -> answer; kind is "approve", "ask" or "ask". +// nullopt means no answer (input closed). +using Prompter = std::function(const std::string& kind, const std::string& text, + const std::vector& options)>; +using StreamFactory = std::function(const std::string& command)>; +using PhaseListener = std::function; + +// ---- module-level helpers of tai.py (9109-9230, 19061-19108) ---- +bool tool_env_ok(const std::string& name); +std::string tool_catalog(); +std::string conversation_text(const json::Value& messages); +// Offered tool schemas: core always, others by name/tag/description word match. +json::Value select_tools(const std::string& text, const json::Value& extra = json::Value()); +const std::vector& tool_schemas(); // TOOL_SCHEMAS parsed from data.cpp +// (messages, changed) +std::pair compact_messages(const json::Value& messages, llm::ChatClient& chat, int keep = KEEP_TURNS); +std::string dir_key(const std::string& directory = ""); +std::string orientation_block(const std::string& directory = "", bool sandbox = false); +std::string capability_block(bool yolo = false, bool auto_mode = false); +std::pair, std::string> split_leading_tags(const std::string& text); +std::string expand_file_mentions(const std::string& text, const std::string& directory); +std::set known_model_ids(const AgentConfig& config, const llm::ModelHealth* health); + +struct ResolvedTag { std::string kind; std::string value; std::string value2; }; + class Agent { public: - explicit Agent(ToolRegistry reg); - std::string run(const std::string& goal); + Agent(AgentConfig& config, StorePort* store, bool persist = true, bool quiet = false, int depth = 0, + bool auto_mode = false, bool yolo = false, const std::string& ctx = ""); + + // ---- state mirrored from the Python attributes ---- + AgentConfig& config; + bool auto_mode; + bool yolo; + std::string ctx_id; + StorePort* store; + llm::ChatClient chat; + Tools tools; + std::string profile; + std::string bot{"main"}; + std::string system_message; + json::Value messages = json::Value::array(); + json::Value skills = json::Value::object(); + std::string env{"home"}; + bool persist; + bool quiet; + int depth; + std::optional deadline; + bool timed_out{false}; + Runner runner_override; // empty: default_runner + bool user_stepped_in{false}; + Prompter prompter; // empty: none + PhaseListener on_phase; // empty: none + StreamFactory stream_factory; // empty: none + int64_t last_verify_failures{1}; + + // ---- profile / bot / system ---- + void apply_system(); + void sync_chat_scope(); + void reset_history(); + void switch_profile(const std::string& name, bool silent = false); + void update_system(const std::string& text); + std::string switch_bot(const std::string& name, bool silent = false); + void scrub_secret_from_history(); + + // ---- approvals ---- + bool ask_approval(const std::string& command, bool guidance = true); // may throw Denied + + // ---- display ---- + void show_call(const llm::ToolCall& call); + void show_result(const std::string& result, double elapsed_ms); + void show_lanes(const std::string& name, const std::string& result); + void show_diff(); + + // ---- turn state ---- + void checkpoint(); + void close_turn(); + void phase(const std::string& name, const std::string& detail = ""); + + // ---- mentions ---- + std::string mention_error(const std::string& tag); + std::string mention_confirm(const std::string& provider, const std::string& model, const std::string& bot_sel, + bool cleared, const std::string& directory); + // (prompt text or nullopt, result or nullopt) + std::pair, std::optional> apply_mentions(const std::string& text, bool capture = false, + int max_steps = MAX_STEPS); + ResolvedTag resolve_tag(const std::string& tag, const std::string& directory); + + // ---- the loop ---- + std::string run_turn(const std::string& text, bool capture = false, int max_steps = MAX_STEPS, bool routed = false); + std::pair run_verified(const std::string& prompt); + bool ensure_verified(const std::string& goal, const std::string& final_text); + std::pair verify_loop(const std::string& goal, const std::string& final_text); + void mark_turn_open(const std::string& goal); + GoalVerdict verify_goal(const std::string& goal, const std::string& final_text); + void show_verdict(const GoalVerdict& verdict, const std::string& goal); + std::string run_mention(const std::string& target, const std::string& rest, const std::string& original, + bool capture = false, int max_steps = MAX_STEPS); + + // turn counters (read by Tools / tests) + int64_t turn_steps() const { return turn_steps_; } + int64_t turn_tools() const { return turn_tools_; } + private: - ToolRegistry reg_; + std::string run_turn_inner(const std::string& text, bool capture, int max_steps, bool routed); + bool turn_open_{false}; + std::string turn_goal_; + int64_t turn_steps_{0}; + int64_t turn_tools_{0}; + struct LastEvent { std::string name, verb, target; }; + std::optional last_event_; }; + +int& total_step_cap(); // Consts.TOTAL_STEP_CAP, live-patchable +// resume_turn(agent): continue an interrupted turn (tai.py 40391). +std::string resume_turn(Agent& agent); +// update_models_command(config): prints the discovery report, returns the exit code (40444). +int update_models_command(AgentConfig& config); + } // namespace detz::agent diff --git a/taicpp/src/agent/agent_config.h b/taicpp/src/agent/agent_config.h new file mode 100644 index 0000000..5cc674a --- /dev/null +++ b/taicpp/src/agent/agent_config.h @@ -0,0 +1,18 @@ +#pragma once +// The Config attributes the agent core reads (tai.py Config, 5022-5060), on +// top of the LLM-facing ones. The config module's Config converts into this. +#include "../llm/ports.h" +#include +namespace detz::agent { + +struct AgentConfig : public llm::LlmConfig { + std::string profiles_dir; + std::string audio_dir; + std::string db_path; + std::string history_path; + std::string profile{"default"}; + std::string voice{"en-US-EmmaMultilingualNeural"}; + bool auto_approve{false}; +}; + +} // namespace detz::agent diff --git a/taicpp/src/agent/consts.h b/taicpp/src/agent/consts.h new file mode 100644 index 0000000..67331ad --- /dev/null +++ b/taicpp/src/agent/consts.h @@ -0,0 +1,98 @@ +#pragma once +// Agent-core constants, copied literally from tai.py (see the line numbers). +#include +#include +namespace detz::agent { + +// 108-115, 295-298, 301-327 +inline constexpr int WORKER_STEPS = 12; +inline constexpr int CREATE_SKILL_STEPS = 40; +inline constexpr int FORK_MAX_DEPTH = 2; +inline constexpr int AGENTS_MAX_PARALLEL = 8; +inline constexpr int AGENTS_MAX_RECORDS = 64; +inline constexpr int SCHEDULER_MAX_FIRES = 4; +inline constexpr int SCHEDULER_LEASE_MARGIN = 30; +inline constexpr int CONTEXT_CAP = 32000; +inline constexpr double COMPACT_RATIO = 0.8; +inline constexpr int KEEP_TURNS = 6; +inline constexpr int MAX_STEPS = 25; +inline constexpr int TOTAL_STEP_CAP = 500; // Consts.TOTAL_STEP_CAP (live-patchable) +inline constexpr int SYSINFO_TIMEOUT = 10; // Consts.SYSINFO_TIMEOUT +inline constexpr int LOOP_NUDGE_AT = 3; +inline constexpr int LOOP_STOP_AT = 5; +inline constexpr int SYSTEM_MAX_CHARS = 12000; +inline constexpr int SHELL_TIMEOUT = 120; +inline constexpr int SCHEDULER_INTERVAL = 30; +inline constexpr int APPROVAL_MAX_RETRIES = 3; +inline constexpr int VERIFY_MAX_ROUNDS = 9; +inline constexpr int VERIFY_TIMEOUT = 120; +inline constexpr int VERIFY_STEPS = 6; +inline constexpr int VERIFY_MAX_CHARS = 6000; +inline constexpr int VERIFY_MAX_TRIED = 12; +inline constexpr int VERIFY_MAX_LINE = 400; +inline constexpr int VERIFY_MAX_FAILURES = 8; +inline constexpr int VERIFY_MAX_GOAL = 2000; +inline constexpr int VERIFY_MAX_ITEM = 200; +inline constexpr int VERIFY_MAX_REASON = 500; +inline constexpr int VERIFY_MAX_PROPOSAL = 500; +inline constexpr int VERIFY_MAX_COMMENTARY = 1000; +inline constexpr int EVAL_K = 5; +inline constexpr int EVAL_FETCH = 10; +inline constexpr int EVAL_MAX_RECORDS = 50; +inline constexpr int EVAL_MAX_GOLDENS = 200; +inline constexpr int EVAL_MAX_PAIRS = 200; +inline constexpr int EVAL_MIN_TOKEN = 6; +inline constexpr int EVAL_PREFIX_MIN = 4; +inline constexpr int RAG_STATS_EVENTS = 5000; +inline constexpr int RAG_STATS_TOP = 20; +inline constexpr int RAG_STATS_QUERIES = 20; +inline constexpr int RAG_STATS_SAMPLE = 10; +inline constexpr int RAG_STATS_KINDS = 30; +// 604-618 +inline constexpr int SPILL_LIMIT = 6000; +inline constexpr int AUDIT_MAX_CHARS = 20000; +inline constexpr int FILE_RECORD_MAX = 50000; +inline constexpr int SHELL_SNAPSHOT_MAX_FILES = 25; +inline constexpr int SHELL_HISTORY_LINES = 50; +inline constexpr int SHELL_HISTORY_CHARS = 4000; +inline constexpr int SHELL_HISTORY_READ_BYTES = 32768; +inline constexpr int TERMINAL_LIST_MAX_PANES = 50; +inline constexpr int TAIL_TERMINALS_DEFAULT_LINES = 3; +inline constexpr int TAIL_TERMINALS_MAX_LINES = 10; +inline constexpr int TAIL_TERMINALS_MAX_PANES = 20; +inline constexpr int TAIL_PER_PANE_CHARS = 800; +// 767-769 +inline constexpr int CHUNK_MAX_LINES = 60; +inline constexpr int CHUNK_OVERLAP_LINES = 3; +inline constexpr int CHUNK_MAX_CHARS = 4000; +// 9086-9087, 17001..: ask_user +inline constexpr int ASK_USER_MAX_OPTIONS = 12; +inline constexpr int ASK_USER_MAX_ANSWER = 2000; +// plexer / tui control +inline constexpr int PLEXER_FLOW_TIMEOUT_MIN = 5; +inline constexpr int PLEXER_FLOW_TIMEOUT_MAX = 600; +inline constexpr const char* PLEXER_SOCKET_ENV = "TAI_PLEXER_SOCKET"; +inline constexpr const char* PLEXER_PANE_ENV = "TAI_PLEXER_PANE"; +inline constexpr int GRR_CONTROL_WAIT = 10; +// MCP (35766-35797) +inline constexpr const char* MCP_TOOL_PREFIX = "mcp_"; +inline constexpr int MCP_RESOLVER_TIMEOUT = 300; +inline constexpr int MCP_RESOLVER_STEPS = 12; +inline constexpr int MCP_OAUTH_TIMEOUT = 300; +inline constexpr int MCP_MAX_TOOLS = 100; +// shell tool +inline constexpr const char* MUSE_LABEL_FOR_REPLY = "muse"; + +const std::vector& RecordKinds(); // RECORD_KINDS +const std::vector& SafeCommands(); // SAFE_COMMANDS +const std::vector& FetchMethods(); // FETCH_METHODS +const std::vector& CommandNames(); // COMMANDS + +// Regex validators (re.match with ^...$; a trailing "\n" is accepted like Python's $). +bool profile_ok(const std::string& s); // PROFILE_RE +bool bot_ok(const std::string& s); // BOT_RE +bool secret_ok(const std::string& s); // SECRET_RE +bool skill_name_ok(const std::string& s);// SKILL_NAME_RE +bool mcp_server_ok(const std::string& s);// MCP_SERVER_RE + +} // namespace detz::agent diff --git a/taicpp/src/agent/data.cpp b/taicpp/src/agent/data.cpp new file mode 100644 index 0000000..5753398 --- /dev/null +++ b/taicpp/src/agent/data.cpp @@ -0,0 +1,219 @@ +// GENERATED by src/agent/gen/gen_data.py from tai.py (version 1.44.2). Do not edit. +#include "data.h" +namespace detz::agent::data { +const std::vector& ToolSchemaJson() { + static const std::vector v = { + R"TAIRAW({"type": "function", "function": {"name": "shell", "description": "Run a shell command. Output is truncated. On home, destructive commands ask the user first. On sandbox, commands run isolated in the container and need no approval. Pass secrets to expose named vault secrets as TAI_SECRET_ variables for that command only; using secrets asks the user first. For local commands only: use web_fetch for HTTP(S), never curl or wget.", "parameters": {"type": "object", "properties": {"command": {"type": "string"}, "workdir": {"type": "string"}, "secrets": {"type": "array"}}, "required": ["command"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "read_file", "description": "Read a text file. Large files are truncated.", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "write_file", "description": "Write content to a file, creating parent directories. Overwrites existing files.", "parameters": {"type": "object", "properties": {"path": {"type": "string"}, "content": {"type": "string"}}, "required": ["path", "content"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "edit_file", "description": "Replace one exact text match in a file. Fails unless the match is unique.", "parameters": {"type": "object", "properties": {"path": {"type": "string"}, "find": {"type": "string"}, "replace": {"type": "string"}}, "required": ["path", "find", "replace"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "web_search", "description": "Search the web. Set images for image search, content to include fetched page text.", "parameters": {"type": "object", "properties": {"query": {"type": "string"}, "images": {"type": "boolean"}, "content": {"type": "boolean"}}, "required": ["query"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "describe_image", "description": "Describe an image with a vision model (rsearch). Pass an http(s) url or a local file path (PNG, JPEG, GIF, WebP, BMP, max 10 MB).", "parameters": {"type": "object", "properties": {"source": {"type": "string"}}, "required": ["source"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "web_fetch", "description": "HTTP client: fetch a URL with any method and return status, content type, size, and content. GET by default; pass method (POST, PUT, PATCH, DELETE, HEAD, OPTIONS), headers, and a string body for APIs. HTML becomes readable text, JSON stays raw, raw forces the raw body. Use this for all HTTP(S), never curl or wget via shell. Pass auth_secret to send a vault secret as an Authorization header; the value stays sealed and secret use asks the user first.", "parameters": {"type": "object", "properties": {"url": {"type": "string"}, "method": {"type": "string"}, "headers": {"type": "object"}, "body": {"type": "string"}, "timeout": {"type": "integer"}, "raw": {"type": "boolean"}, "auth_secret": {"type": "string"}, "auth_header": {"type": "string"}, "auth_prefix": {"type": "string"}}, "required": ["url"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "speak", "description": "Synthesize text to speech with a free voice, save the MP3, and play it when a player exists.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "voice": {"type": "string"}}, "required": ["text"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "listen", "description": "Record from the microphone for some seconds and transcribe it. Needs OS audio tools.", "parameters": {"type": "object", "properties": {"seconds": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "remember", "description": "Update your own system message with new knowledge or behavior. The instruction is merged into the existing system message, unrelated parts stay intact. Use it to forget as well by instructing what to remove. Execute by default when behavior, preferences, or durable facts need updating. Never pass passwords, tokens, or secret values here; store those with store_secret instead.", "parameters": {"type": "object", "properties": {"instruction": {"type": "string"}}, "required": ["instruction"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "recall", "description": "Search past session memory of the current profile by keyword, optionally filtered by tags.", "parameters": {"type": "object", "properties": {"query": {"type": "string"}, "tags": {"type": "array"}}, "required": ["query"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "load_skill", "description": "Load a skill by name. Returns full instructions plus bundled file paths. Missing skills with a blueprint are researched and built on demand.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "get_current_terminal_content", "description": "Capture visible text of the user's current terminal pane including scrollback, plus recent shell history. With no target the pane resolves straight from the TMUX_PANE or TAI_PLEXER_PANE environment variable, no scan needed, so prefer this over list_terminals and tail_terminals whenever you need the current terminal. Pass target (a pane id like %3 or session:window.pane from list_terminals) to capture another open terminal instead. Works inside tmux, under tai's own multiplexer, or against a running tmux server.", "parameters": {"type": "object", "properties": {"lines": {"type": "integer"}, "target": {"type": "string"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "list_terminals", "description": "List every open tmux terminal: all panes in all sessions with ids, current commands, and working directories, plus session/window/pane counts. Only for an overview of ALL terminals; for the user's current terminal call get_current_terminal_content with no target instead (it resolves via env). Use a pane id as target for get_current_terminal_content. Needs a running tmux server.", "parameters": {"type": "object", "properties": {}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tail_terminals", "description": "One overview of all open tmux terminals: the last lines (default 3) of each at once, labeled with pane ids so you can spot which terminal holds what, then capture it fully with get_current_terminal_content. Only for an overview of ALL terminals; for the user's current terminal call get_current_terminal_content with no target instead (it resolves via env). Needs a running tmux server.", "parameters": {"type": "object", "properties": {"lines": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "export_terminal_log", "description": "Only available inside a live tmux session. Exports the full pane scrollback (tmux capture-pane -pS -, the entire history, not just what's visible) to a log file under tai's home dir, then chunks it and indexes the chunks for retrieval with SQLite FTS5 + bm25 (lexical only, no embeddings). From then on, use the search tool (kind chunk) to answer questions about anything that happened in this terminal; it returns the actual chunk text, not just a snippet.", "parameters": {"type": "object", "properties": {"tags": {"type": "array"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "plexer_send", "description": "Only available when a tai-managed terminal session is active (either tai's own native wrapper, or an active tmux pane). Sends literal keystrokes into that session's shell, as if typed. Set enter to also press Enter and actually run it. No output is returned by this call; follow up with plexer_read.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "enter": {"type": "boolean"}}, "required": ["text"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "plexer_read", "description": "Only available when a tai-managed terminal session is active. Reads back the last N lines the session's shell has produced (its actual rendered output, not a tool-call summary).", "parameters": {"type": "object", "properties": {"lines": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "plexer_type", "description": "Only available when a tai-managed terminal session is active. Types text into the session one character at a time with a delay between keystrokes, for a demo/animated-typing effect, then presses Enter unless enter is set to false.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "delay": {"type": "number"}, "enter": {"type": "boolean"}}, "required": ["text"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "plexer_status", "description": "Only available when a tai-managed terminal session is active. Reports which backend is managing the current session (tai's own native pty wrapper, or tmux) and backend-specific details (pid/socket/log, or pane id).", "parameters": {"type": "object", "properties": {}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "plexer_show", "description": "Only available when a tai-managed terminal session is active. Shows an interactive UI flow on the user's terminal and waits for the outcome: one spec with screens, each with body text, fields and actions. Field types: choice (options, single select), text (free input), info (display only). Fields and screens take show_if conditions on earlier answers ({field: expected}); next routes on ({action: screen}, a screen id, $done, $cancel). Example: {screens: [{id: main, body: Deploy?, fields: [{id: env, type: choice, options: [staging, prod]}], actions: [{id: ok, label: OK}], next: $done}]}. Returns {status: done/cancelled/timeout, answers: {field: value}, action}. Full contract: uiflow.md. Only for real blockers a human must decide; in autonomous mode and for background workers it is unavailable, so decide yourself and state the assumption.", "parameters": {"type": "object", "properties": {"spec": {"type": "object"}, "timeout": {"type": "integer"}}, "required": ["spec"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_screen", "description": "Only available while a grr TUI (tai.py --tui) runs on this machine. Returns what its screen shows now (route, size, focus, the text). Set until to wait for that text to appear (timeout seconds), set async to get a job id at once and collect it with tui_result.", "parameters": {"type": "object", "properties": {"until": {"type": "string"}, "timeout": {"type": "integer"}, "async": {"type": "boolean"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_keys", "description": "Only available while a grr TUI runs. Presses keys in it like a user: names such as down, up, enter, escape, space, tab, f1, ctrl+n, ctrl+p, or single characters. Async calls return a job id for tui_result.", "parameters": {"type": "object", "properties": {"keys": {"type": "array", "items": {"type": "string"}}, "async": {"type": "boolean"}}, "required": ["keys"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_type", "description": "Only available while a grr TUI runs. Types text into the focused box (a newline presses Enter). Async calls return a job id for tui_result.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "async": {"type": "boolean"}}, "required": ["text"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_click", "description": "Only available while a grr TUI runs. Clicks the cell at column x, row y (0 based, as in tui_screen text). Async calls return a job id for tui_result.", "parameters": {"type": "object", "properties": {"x": {"type": "integer"}, "y": {"type": "integer"}, "async": {"type": "boolean"}}, "required": ["x", "y"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_goto", "description": "Only available while a grr TUI runs. Opens a route such as /chat/new, /bots, /settings, /automations, /bot/. Async calls return a job id for tui_result.", "parameters": {"type": "object", "properties": {"route": {"type": "string"}, "async": {"type": "boolean"}}, "required": ["route"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tui_result", "description": "Only available while a grr TUI runs. Collects an async tui_* job by id, waiting up to wait seconds; the state is running, done or error.", "parameters": {"type": "object", "properties": {"id": {"type": "integer"}, "wait": {"type": "integer"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "fork", "description": "Spawn a background subagent with its own context that works while you continue. Returns an agent id immediately. Collect its summarized result with poll. Subagents get a smaller step budget and a time limit.", "parameters": {"type": "object", "properties": {"task": {"type": "string"}, "timeout": {"type": "integer"}, "profile": {"type": "string"}}, "required": ["task"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "poll", "description": "Collect a background subagent result by id. Waits up to wait seconds, then reports running or the result.", "parameters": {"type": "object", "properties": {"id": {"type": "integer"}, "wait": {"type": "integer"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "ask_user", "description": "Ask the user one question mid-turn and wait for their answer. Use this instead of ending your turn with questions in prose: an empty tool_calls list ends the turn at once and one-shot runs then exit, so prose questions are never answered. Pass the question plus optional options; the user picks an option (by number or text) or types free input, and their answer comes back as the tool result. Only for real blockers a human must decide; in autonomous mode and for background workers it is unavailable, so decide yourself and state the assumption.", "parameters": {"type": "object", "properties": {"question": {"type": "string"}, "options": {"type": "array"}}, "required": ["question"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "sysinfo", "description": "Inspect the host machine where tai runs. Runs os, python, venv, root, container, binaries, cpu, and disk checks in parallel and reports each with timing. Pass checks to run a named subset.", "parameters": {"type": "object", "properties": {"checks": {"type": "array"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "create_skill", "description": "Create a new agent skill by name from a brief. Runs a dedicated deep-research worker on this machine (it calls sysinfo first, then researches with web search and shell until it has verified information) and writes SKILL.md plus supporting files into the skill directory. Runs synchronously and can take many minutes. Scope project writes under ./.tai/skills (the default), scope home under ~/.tai/skills.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}, "brief": {"type": "string"}, "scope": {"type": "string"}, "timeout": {"type": "integer"}}, "required": ["name", "brief"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "install_skill", "description": "Install a skill that already exists somewhere else, instead of building one from scratch. source can be a local directory containing SKILL.md (or one subdirectory that does), a bare SKILL.md file, a local .zip, or a git URL (needs git installed; most sources need nothing beyond the standard library). Validates the SKILL.md frontmatter, then lands it under tai's own skill directories: copy vendors a private snapshot, symlink points at the source in place (the default when the source already sits in a recognized skills folder like .claude/skills or .agents/skills, so there is one canonical copy instead of a drifting duplicate). Scope defaults like create_skill: project when the cwd looks like a project, else home. Overwriting an existing skill of the same name asks the user first unless force is set.", "parameters": {"type": "object", "properties": {"source": {"type": "string"}, "name": {"type": "string"}, "scope": {"type": "string"}, "mode": {"type": "string"}, "force": {"type": "boolean"}}, "required": ["source"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "store_secret", "description": "Store a password, token, or secret in the sealed vault under a name. Values are encrypted at rest, never shown back, and only usable by reference; prefer /secret set in the REPL so the value never enters the conversation. Optional username, host, port, notes, expires (ISO datetime), and tags describe it; name and value suffice, never pester the user for more.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}, "value": {"type": "string"}, "username": {"type": "string"}, "host": {"type": "string"}, "port": {"type": "integer"}, "notes": {"type": "string"}, "expires": {"type": "string"}, "tags": {"type": "array"}}, "required": ["name", "value"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "list_secrets", "description": "List vault secret names with metadata and tags. Values are never revealed.", "parameters": {"type": "object", "properties": {}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "delete_secret", "description": "Delete a vault secret by name. Asks the user first.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "schedule", "description": "Schedule a prompt to run at a future time or on a repeating interval. Exactly one of at (ISO datetime, naive means local time) or every (interval seconds, minimum 60) is required. When due, a background subagent runs the prompt without blocking anyone; inspect outcomes with schedules. Asks the user first. Optional tags label it.", "parameters": {"type": "object", "properties": {"at": {"type": "string"}, "every": {"type": "integer"}, "prompt": {"type": "string"}, "name": {"type": "string"}, "timeout": {"type": "integer"}, "profile": {"type": "string"}, "tags": {"type": "array"}}, "required": ["prompt"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "unschedule", "description": "Delete a scheduled prompt by id. Asks the user first.", "parameters": {"type": "object", "properties": {"id": {"type": "integer"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "schedules", "description": "List scheduled prompts with due times and last outcomes.", "parameters": {"type": "object", "properties": {}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "record_save", "description": "Save text as a tagged record in the vault and get back a mem: id. Use for anything too big or too durable for chat: research notes, command output, transcripts. Kind is one of note, output, file, research, transcript. Tag rules: lowercase singular, reuse tags from the tags tool, at most 5; known words in the text attach automatically and the record links to recent same-tag records.", "parameters": {"type": "object", "properties": {"title": {"type": "string"}, "content": {"type": "string"}, "kind": {"type": "string"}, "tags": {"type": "array"}}, "required": ["content"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "record_read", "description": "Read a slice of a record by mem: id. Offset and limit page through huge records without loading them whole.", "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "offset": {"type": "integer"}, "limit": {"type": "integer"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "record_search", "description": "Search records by text, kind, and tags. Returns mem: ids with sizes so you can page in only what you need.", "parameters": {"type": "object", "properties": {"query": {"type": "string"}, "kind": {"type": "string"}, "tags": {"type": "array"}, "limit": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "record_delete", "description": "Delete a record by mem: id. Asks the user first.", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "graph_link", "description": "Link two vault nodes with a relation: mem: records, secret: secrets, sched: schedules. Both endpoints must exist.", "parameters": {"type": "object", "properties": {"src": {"type": "string"}, "dst": {"type": "string"}, "relation": {"type": "string"}}, "required": ["src", "dst"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "graph_query", "description": "Show the neighborhood of a vault node (mem:, secret:, sched:) by breadth-first traversal. Depth caps at 4.", "parameters": {"type": "object", "properties": {"node": {"type": "string"}, "depth": {"type": "integer"}, "limit": {"type": "integer"}}, "required": ["node"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "delete_file", "description": "Delete a file. Requires reading it first; the full pre-image stays in the audit trail so restore can undelete. Asks the user first.", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "audit", "description": "Show the file audit trail for time travel: every write, edit, delete, shell snapshot, release, and restore with sizes and messages. Filter by exact path or tag, or rank by a full-text query over paths and messages.", "parameters": {"type": "object", "properties": {"path": {"type": "string"}, "tag": {"type": "string"}, "query": {"type": "string"}, "limit": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "restore", "description": "Restore a file to an audit row: the post-image for writes and edits, the pre-image for deletes and shell snapshots. Refuses truncated images. Asks the user first.", "parameters": {"type": "object", "properties": {"id": {"type": "integer"}}, "required": ["id"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "release", "description": "Cut a release of the running script: bump VERSION by part (patch for fixes, minor for features, major for breaking changes), keep a versioned backup, log the message in the audit trail. Asks the user first.", "parameters": {"type": "object", "properties": {"part": {"type": "string"}, "message": {"type": "string"}}, "required": ["part", "message"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "tags", "description": "List vault tags with usage counts, most used first. Consult before tagging so new items reuse established tags instead of inventing synonyms.", "parameters": {"type": "object", "properties": {"prefix": {"type": "string"}, "limit": {"type": "integer"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "install", "description": "Manage tai installations: binary, command-not-found hook and plexer auto-wrap hook for bash/zsh/fish, venv, scheduler service, telegram service, container. Status reports what exists. Install adds missing pieces (skipping a shell hook loudly if that shell isn't on the machine), upgrade refreshes in place and restarts services, reinstall rebuilds artifacts, uninstall removes them. Service changes take effect immediately. The vault is never touched by any action. Non-status actions ask the user first.", "parameters": {"type": "object", "properties": {"action": {"type": "string"}, "targets": {"type": "array"}}, "required": ["action"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "create_bot", "description": "Create a bot in the current profile: a name plus its own system message and history. Give at least one of description, rules, or behavior; optional nicknames register as @mention aliases (a short form of the name registers automatically). Names are lowercase.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}, "description": {"type": "string"}, "rules": {"type": "string"}, "behavior": {"type": "string"}, "nicknames": {"type": "array"}}, "required": ["name"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "search", "description": "Search everything at once with ranked full-text matching (SQLite FTS5 + bm25, no vectors): records, episodic events, the audit trail, and RAG chunks. Chunks are the retrieval-grade granular hits (e.g. one slice of an exported terminal log) and come back with their actual text, not just a snippet; use the chunk's parent id with record_read to see more surrounding context. One call replaces paging through each store separately. expand adds one graph hop per record hit.", "parameters": {"type": "object", "properties": {"query": {"type": "string"}, "kinds": {"type": "array"}, "limit": {"type": "integer"}, "expand": {"type": "boolean"}}, "required": ["query"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_add", "description": "Register an MCP server from pasted JSON/TOML config or a github URL. Normalizes the config, shows the exact spawn for approval, verifies with a live handshake, stores the connection, and enables it for the profile. Github URLs resolve via registry, package files, and README run blocks, verified live before storing.", "parameters": {"type": "object", "properties": {"source": {"type": "string"}, "name": {"type": "string"}, "profile": {"type": "string"}}, "required": ["source"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_list", "description": "List registered MCP servers with transport, enabled profiles, tool counts, and origin.", "parameters": {"type": "object", "properties": {}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_enable", "description": "Enable an MCP server for a profile (current when omitted).", "parameters": {"type": "object", "properties": {"server": {"type": "string"}, "profile": {"type": "string"}}, "required": ["server"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_disable", "description": "Disable an MCP server for a profile (current when omitted).", "parameters": {"type": "object", "properties": {"server": {"type": "string"}, "profile": {"type": "string"}}, "required": ["server"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_remove", "description": "Delete an MCP server with its profile rows, pins, and cached tools. Asks first.", "parameters": {"type": "object", "properties": {"server": {"type": "string"}}, "required": ["server"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_tools", "description": "List cached MCP tools with descriptions and pin state, for one server or all.", "parameters": {"type": "object", "properties": {"server": {"type": "string"}}, "required": []}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_call", "description": "Call one MCP tool explicitly: full path with drift check, approval, caps, and audit. Bridged mcp__ names call the same path.", "parameters": {"type": "object", "properties": {"server": {"type": "string"}, "tool": {"type": "string"}, "args": {"type": "object"}}, "required": ["server", "tool"]}}})TAIRAW", + R"TAIRAW({"type": "function", "function": {"name": "mcp_login", "description": "OAuth login to a remote MCP server: opens the browser, waits for approval, stores tokens sealed in the vault, then verifies and enables. Run in a REPL, not headless.", "parameters": {"type": "object", "properties": {"server": {"type": "string"}, "profile": {"type": "string"}}, "required": ["server"]}}})TAIRAW", + }; + return v; +} +const std::vector>>& ToolTags() { + static const std::vector>> v = { + {"shell", {"run", "execute", "command", "bash", "terminal", "script"}}, + {"read_file", {"read", "open", "view", "contents"}}, + {"write_file", {"write", "create", "overwrite"}}, + {"edit_file", {"edit", "change", "modify", "replace", "patch", "fix", "alter"}}, + {"web_search", {"search", "web", "internet", "lookup", "google"}}, + {"describe_image", {"image", "describe", "vision", "picture", "photo", "screenshot"}}, + {"web_fetch", {"fetch", "download", "url", "link", "page", "curl", "wget"}}, + {"speak", {"speak", "say", "voice", "talk", "tts"}}, + {"listen", {"listen", "hear", "microphone", "mic", "transcribe", "stt", "dictate"}}, + {"remember", {"remember", "memorize", "preference"}}, + {"recall", {"recall", "memory", "history", "past"}}, + {"load_skill", {"skill", "capability"}}, + {"get_current_terminal_content", {"terminal", "tmux", "pane", "scrollback", "screen", "history", "bash"}}, + {"list_terminals", {"terminal", "tmux", "panes", "sessions", "windows", "list", "open", "count"}}, + {"tail_terminals", {"terminal", "tmux", "panes", "tail", "peek", "overview", "every"}}, + {"export_terminal_log", {"terminal", "tmux", "export", "log", "capture", "history", "rag", "index"}}, + {"plexer_send", {"plexer", "terminal", "tmux", "keys", "send-keys", "type", "keystroke"}}, + {"plexer_read", {"plexer", "terminal", "tmux", "pane", "read", "capture"}}, + {"plexer_type", {"plexer", "terminal", "tmux", "type", "animate", "keystroke", "demo"}}, + {"plexer_status", {"plexer", "terminal", "tmux", "status", "session"}}, + {"plexer_show", {"plexer", "dialog", "flow", "menu", "form", "wizard", "screen", "ui"}}, + {"tui_screen", {"tui", "gui", "screen", "interface", "ui"}}, + {"tui_keys", {"tui", "gui", "keys", "press", "keyboard", "navigate"}}, + {"tui_type", {"tui", "gui", "type", "input", "text"}}, + {"tui_click", {"tui", "gui", "click", "mouse"}}, + {"tui_goto", {"tui", "gui", "open", "route", "navigate", "goto"}}, + {"tui_result", {"tui", "gui", "result", "async", "job"}}, + {"fork", {"fork", "subagent", "background", "parallel", "delegate", "spawn"}}, + {"poll", {"poll", "collect"}}, + {"ask_user", {"ask", "question", "clarify", "confirm", "choose", "answer", "input"}}, + {"sysinfo", {"sysinfo", "system", "host", "machine", "hardware", "specs", "installed", "python", "container"}}, + {"create_skill", {"skill", "create", "author", "blueprint"}}, + {"install_skill", {"skill", "install", "import", "adopt", "copy", "symlink"}}, + {"store_secret", {"secret", "password", "token", "credential", "passwd", "vault", "apikey"}}, + {"list_secrets", {"secrets", "vault", "credentials"}}, + {"delete_secret", {"secret", "remove", "revoke"}}, + {"schedule", {"schedule", "cron", "appointment", "reminder", "recurring", "periodic", "later", "daily", "hourly", "interval"}}, + {"unschedule", {"unschedule", "cancel"}}, + {"schedules", {"schedules", "upcoming", "planned", "agenda"}}, + {"record_save", {"record", "note", "document", "memo"}}, + {"record_read", {"record", "page"}}, + {"record_search", {"record", "find"}}, + {"record_delete", {"record", "remove"}}, + {"graph_link", {"graph", "link", "connect", "relate", "edge", "relation"}}, + {"graph_query", {"graph", "neighbors", "neighborhood", "traverse", "related"}}, + {"delete_file", {"delete", "remove", "erase"}}, + {"audit", {"audit", "trail", "changes", "blame", "history", "version"}}, + {"restore", {"restore", "undo", "revert", "recover", "rollback", "undelete", "previous"}}, + {"release", {"release", "version", "bump", "changelog", "publish"}}, + {"tags", {"tags", "tag", "label", "categorize", "taxonomy", "vocabulary"}}, + {"install", {"install", "uninstall", "reinstall", "upgrade", "setup", "service", "systemd", "venv", "hook", "deploy", "zsh", "fish"}}, + {"create_bot", {"bot", "bots", "persona", "character", "mention"}}, + {"search", {"search", "find", "lookup", "everything"}}, + {"mcp_add", {"mcp", "server", "register", "connect", "integrate"}}, + {"mcp_list", {"mcp", "servers", "connections"}}, + {"mcp_enable", {"mcp", "enable", "activate"}}, + {"mcp_disable", {"mcp", "disable", "deactivate"}}, + {"mcp_remove", {"mcp", "remove", "delete", "unregister"}}, + {"mcp_tools", {"mcp", "tools"}}, + {"mcp_call", {"mcp", "call", "invoke"}}, + {"mcp_login", {"mcp", "login", "oauth", "auth"}}, + }; + return v; +} +const std::vector& CoreTools() { + static const std::vector v = {"shell", "read_file", "write_file", "remember", "recall", "load_skill", "fork", "poll", "ask_user"}; + return v; +} +const std::vector& LazyNameStop() { + static const std::vector v = {"get", "set", "list", "add"}; + return v; +} +const std::string& DefaultSystem() { + static const std::string s = + R"TAIRAW(You are tai, a professional autonomous assistant. You act with tools, verify results, and report concisely. Rules: prefer the smallest change that solves the task; never silently ignore errors; call ask_user when requirements are ambiguous (never end your turn with open questions in prose: one-shot runs exit and prose questions are never answered); keep answers short and factual. Tools: shell and file tools for the local machine, web_search and web_fetch for the internet, web_fetch handles every HTTP method with headers and bodies so never curl or wget through shell, speak for voice output, recall to search past session memory. Memory: call remember whenever you learn durable facts, preferences, or standing behavior changes. Store passwords, tokens, and secrets with store_secret and reference them by name; never write secret values into memory. Name and value suffice for secrets; never pester the user for more. Call remember with a forget instruction to drop outdated knowledge. Delegation: do quick, interactive, or memory-changing work yourself; fork background subagents for independent, long, or context-heavy work and collect them with poll; schedule future work instead of waiting on it. Records: save big or durable findings with record_save and tag generously; large tool outputs spill to mem: ids automatically, page them with record_read, connect related nodes with graph_link. Tags: strict rules, lowercase singular only, shortest common word for the subject, one canonical tag per subject, at most 5 per item, always reuse existing tags from the tags tool instead of inventing synonyms. Content words matching known tags attach automatically, and new records link to recent same-tag records. Lazy tools: only everyday tools load by default; name any other tool or topic in your reply text and it becomes callable on your next step. Bots: one identity holds many bots; a bot is only a system message plus its own history, all vault data is shared. @name routes one turn to another bot and files the exchange in both histories; /bot switches, create_bot makes bots. Terminal: the user's question is very likely about what is visible in their terminal, especially right after a terminal capture or when launched with a single question straight from a shell; resolve vague references (this, that, it, the error) against that content, capture it with get_current_terminal_content instead of asking, and answer from what you see. Retrieval: use search first for anything stored: one ranked full-text query across records, events, and audit plus graph expansion beats paging and guessing. Chain tool calls until the task is done. Never narrate a planned action in text without emitting its tool call: text without tool calls ends your turn and the action will not happen.)TAIRAW"; + return s; +} +const std::string& MergeSystem() { + static const std::string s = + R"TAIRAW(You maintain an AI assistant system prompt. You receive the current system prompt and one memory instruction. Rewrite the system prompt to incorporate the instruction: add new facts, update changed behavior, or remove forgotten items. Never persist passwords, tokens, or other secret values; if the instruction holds one, note only that a secret exists and must be stored with store_secret. Preserve everything unrelated. Keep it organized with short sections. Output ONLY the rewritten system prompt, no explanation, no code fences.)TAIRAW"; + return s; +} +const std::string& CompactSystem() { + static const std::string s = + R"TAIRAW(You compress assistant session history into a dense handoff summary. Cover: goal, completed work with outcomes, current state, key decisions, files touched, errors and fixes, pending next steps. Output only the summary, no preamble.)TAIRAW"; + return s; +} +const std::string& AutonomousNote() { + static const std::string s = + R"TAIRAW(Autonomous mode: never ask the user anything this session (the ask_user tool refuses, do not call it). When requirements are unclear, blocked, or need input, research deeply with web_search, web_fetch, shell, and subagents, then decide and proceed. State assumptions briefly in your reply.)TAIRAW"; + return s; +} +const std::string& LegacyPasswordNote() { + static const std::string s = + R"TAIRAW(especially passwords (collect and keep every password the user shares), preferences, or standing behavior changes)TAIRAW"; + return s; +} +const std::string& VaultMemoryNote() { + static const std::string s = + R"TAIRAW(preferences, or standing behavior changes. Store passwords, tokens, and secrets with store_secret and reference them by name; never write secret values into memory)TAIRAW"; + return s; +} +const std::string& McpResolverBrief() { + static const std::string s = + R"TAIRAW(You must resolve the GitHub repo %s/%s into an MCP server run contract. +Use web_fetch to read, in this order, until one tells you how to run the server: +https://api.github.com/repos/%s/%s/contents/mcp.json +https://api.github.com/repos/%s/%s/contents/.mcp.json +https://api.github.com/repos/%s/%s/contents/server.json +https://api.github.com/repos/%s/%s/contents/package.json +https://api.github.com/repos/%s/%s/contents/README.md +https://raw.githubusercontent.com/%s/%s/HEAD/README.md +(github api file blobs are base64 under content, decode them yourself.) +Prefer an explicit stdio command (command + args + env) from the repo docs; npx -y for npm packages and uvx for python packages are acceptable when the README shows them. If the repo is only usable as a remote http endpoint, return its url instead. +Reply with EXACTLY one fenced block and nothing else outside it: +```json +{"command": "npx", "args": ["-y", "pkg"], "env": {}} +``` +or {"url": "https://..."} for remote servers, or {"error": "short reason", "evidence": "what you checked"} when nothing runnable exists. Never invent package names or urls you did not read.)TAIRAW"; + return s; +} +const std::string& GoalVerifierTask() { + static const std::string s = + R"TAIRAW(You are a strict acceptance verifier. The main agent was asked to achieve a GOAL. Decide only from evidence whether the goal is actually delivered. Be very strict: claims without evidence are failures. User praise (perfect, great, fine) is not evidence. A denied, skipped or timed out tool call means its effect did not happen. Partial work is failure unless the goal explicitly allows partial work. You may use read-only tools to inspect machine state; judge from the transcript when a check is unavailable. + +GOAL: +%s + +FINAL ANSWER of the main agent: +%s + +TRANSCRIPT (oldest first, tail truncated): +%s + +ALREADY TRIED (never propose these again verbatim, move forward): +%s + +Reply with exactly one JSON object and nothing else: +{"success": true or false, "failures": [{"item": "...", "reason": "...", "proposal": "..."}], "commentary": "..."} +Rules: success is true only when every part of the goal is delivered with evidence. When success is false, list every unmet item (at most %d) with its reason and one concrete next proposal that differs from ALREADY TRIED.)TAIRAW"; + return s; +} +const std::string& CurlHint() { + static const std::string s = + R"TAIRAW(hint: use the web_fetch tool for HTTP(S) (methods, headers, bodies, status) instead of curl/wget via shell)TAIRAW"; + return s; +} +} // namespace detz::agent::data diff --git a/taicpp/src/agent/data.h b/taicpp/src/agent/data.h new file mode 100644 index 0000000..7f24142 --- /dev/null +++ b/taicpp/src/agent/data.h @@ -0,0 +1,27 @@ +#pragma once +// Byte-exact data copied from tai.py by src/agent/gen/gen_data.py. +#include +#include +#include +namespace detz::agent::data { +// json.dumps(schema) of each TOOL_SCHEMAS entry, in tai.py order (61 entries). +const std::vector& ToolSchemaJson(); +// TOOL_TAGS in tai.py insertion order. +const std::vector>>& ToolTags(); +const std::vector& CoreTools(); // CORE_TOOLS +const std::vector& LazyNameStop(); // LAZY_NAME_STOP +const std::string& DefaultSystem(); +const std::string& MergeSystem(); +const std::string& CompactSystem(); +const std::string& AutonomousNote(); +const std::string& LegacyPasswordNote(); +const std::string& VaultMemoryNote(); +const std::string& McpResolverBrief(); // contains 14 "%s" slots +const std::string& GoalVerifierTask(); // GoalVerifier.TASK, 5 slots (4 %s, 1 %d) +const std::string& CurlHint(); +// Python html.entities.html5 (names keep the ';' form and the legacy no-';' +// forms), html._invalid_charrefs and html._invalid_codepoints: html.unescape. +const std::vector>& Html5Entities(); +const std::vector>& HtmlInvalidCharrefs(); +const std::vector& HtmlInvalidCodepoints(); +} // namespace detz::agent::data diff --git a/taicpp/src/agent/data_html.cpp b/taicpp/src/agent/data_html.cpp new file mode 100644 index 0000000..c059ac4 --- /dev/null +++ b/taicpp/src/agent/data_html.cpp @@ -0,0 +1,2283 @@ +// GENERATED by src/agent/gen/gen_data.py html from Python 3.14.4 html.entities. Do not edit. +#include "data.h" +namespace detz::agent::data { +const std::vector>& Html5Entities() { + static const std::vector> v = { + {"AElig", "\303\206"}, + {"AElig;", "\303\206"}, + {"AMP", "&"}, + {"AMP;", "&"}, + {"Aacute", "\303\201"}, + {"Aacute;", "\303\201"}, + {"Abreve;", "\304\202"}, + {"Acirc", "\303\202"}, + {"Acirc;", "\303\202"}, + {"Acy;", "\320\220"}, + {"Afr;", "\360\235\224\204"}, + {"Agrave", "\303\200"}, + {"Agrave;", "\303\200"}, + {"Alpha;", "\316\221"}, + {"Amacr;", "\304\200"}, + {"And;", "\342\251\223"}, + {"Aogon;", "\304\204"}, + {"Aopf;", "\360\235\224\270"}, + {"ApplyFunction;", "\342\201\241"}, + {"Aring", "\303\205"}, + {"Aring;", "\303\205"}, + {"Ascr;", "\360\235\222\234"}, + {"Assign;", "\342\211\224"}, + {"Atilde", "\303\203"}, + {"Atilde;", "\303\203"}, + {"Auml", "\303\204"}, + {"Auml;", "\303\204"}, + {"Backslash;", "\342\210\226"}, + {"Barv;", "\342\253\247"}, + {"Barwed;", "\342\214\206"}, + {"Bcy;", "\320\221"}, + {"Because;", "\342\210\265"}, + {"Bernoullis;", "\342\204\254"}, + {"Beta;", "\316\222"}, + {"Bfr;", "\360\235\224\205"}, + {"Bopf;", "\360\235\224\271"}, + {"Breve;", "\313\230"}, + {"Bscr;", "\342\204\254"}, + {"Bumpeq;", "\342\211\216"}, + {"CHcy;", "\320\247"}, + {"COPY", "\302\251"}, + {"COPY;", "\302\251"}, + {"Cacute;", "\304\206"}, + {"Cap;", "\342\213\222"}, + {"CapitalDifferentialD;", "\342\205\205"}, + {"Cayleys;", "\342\204\255"}, + {"Ccaron;", "\304\214"}, + {"Ccedil", "\303\207"}, + {"Ccedil;", "\303\207"}, + {"Ccirc;", "\304\210"}, + {"Cconint;", "\342\210\260"}, + {"Cdot;", "\304\212"}, + {"Cedilla;", "\302\270"}, + {"CenterDot;", "\302\267"}, + {"Cfr;", "\342\204\255"}, + {"Chi;", "\316\247"}, + {"CircleDot;", "\342\212\231"}, + {"CircleMinus;", "\342\212\226"}, + {"CirclePlus;", "\342\212\225"}, + {"CircleTimes;", "\342\212\227"}, + {"ClockwiseContourIntegral;", "\342\210\262"}, + {"CloseCurlyDoubleQuote;", "\342\200\235"}, + {"CloseCurlyQuote;", "\342\200\231"}, + {"Colon;", "\342\210\267"}, + {"Colone;", "\342\251\264"}, + {"Congruent;", "\342\211\241"}, + {"Conint;", "\342\210\257"}, + {"ContourIntegral;", "\342\210\256"}, + {"Copf;", "\342\204\202"}, + {"Coproduct;", "\342\210\220"}, + {"CounterClockwiseContourIntegral;", "\342\210\263"}, + {"Cross;", "\342\250\257"}, + {"Cscr;", "\360\235\222\236"}, + {"Cup;", "\342\213\223"}, + {"CupCap;", "\342\211\215"}, + {"DD;", "\342\205\205"}, + {"DDotrahd;", "\342\244\221"}, + {"DJcy;", "\320\202"}, + {"DScy;", "\320\205"}, + {"DZcy;", "\320\217"}, + {"Dagger;", "\342\200\241"}, + {"Darr;", "\342\206\241"}, + {"Dashv;", "\342\253\244"}, + {"Dcaron;", "\304\216"}, + {"Dcy;", "\320\224"}, + {"Del;", "\342\210\207"}, + {"Delta;", "\316\224"}, + {"Dfr;", "\360\235\224\207"}, + {"DiacriticalAcute;", "\302\264"}, + {"DiacriticalDot;", "\313\231"}, + {"DiacriticalDoubleAcute;", "\313\235"}, + {"DiacriticalGrave;", "`"}, + {"DiacriticalTilde;", "\313\234"}, + {"Diamond;", "\342\213\204"}, + {"DifferentialD;", "\342\205\206"}, + {"Dopf;", "\360\235\224\273"}, + {"Dot;", "\302\250"}, + {"DotDot;", "\342\203\234"}, + {"DotEqual;", "\342\211\220"}, + {"DoubleContourIntegral;", "\342\210\257"}, + {"DoubleDot;", "\302\250"}, + {"DoubleDownArrow;", "\342\207\223"}, + {"DoubleLeftArrow;", "\342\207\220"}, + {"DoubleLeftRightArrow;", "\342\207\224"}, + {"DoubleLeftTee;", "\342\253\244"}, + {"DoubleLongLeftArrow;", "\342\237\270"}, + {"DoubleLongLeftRightArrow;", "\342\237\272"}, + {"DoubleLongRightArrow;", "\342\237\271"}, + {"DoubleRightArrow;", "\342\207\222"}, + {"DoubleRightTee;", "\342\212\250"}, + {"DoubleUpArrow;", "\342\207\221"}, + {"DoubleUpDownArrow;", "\342\207\225"}, + {"DoubleVerticalBar;", "\342\210\245"}, + {"DownArrow;", "\342\206\223"}, + {"DownArrowBar;", "\342\244\223"}, + {"DownArrowUpArrow;", "\342\207\265"}, + {"DownBreve;", "\314\221"}, + {"DownLeftRightVector;", "\342\245\220"}, + {"DownLeftTeeVector;", "\342\245\236"}, + {"DownLeftVector;", "\342\206\275"}, + {"DownLeftVectorBar;", "\342\245\226"}, + {"DownRightTeeVector;", "\342\245\237"}, + {"DownRightVector;", "\342\207\201"}, + {"DownRightVectorBar;", "\342\245\227"}, + {"DownTee;", "\342\212\244"}, + {"DownTeeArrow;", "\342\206\247"}, + {"Downarrow;", "\342\207\223"}, + {"Dscr;", "\360\235\222\237"}, + {"Dstrok;", "\304\220"}, + {"ENG;", "\305\212"}, + {"ETH", "\303\220"}, + {"ETH;", "\303\220"}, + {"Eacute", "\303\211"}, + {"Eacute;", "\303\211"}, + {"Ecaron;", "\304\232"}, + {"Ecirc", "\303\212"}, + {"Ecirc;", "\303\212"}, + {"Ecy;", "\320\255"}, + {"Edot;", "\304\226"}, + {"Efr;", "\360\235\224\210"}, + {"Egrave", "\303\210"}, + {"Egrave;", "\303\210"}, + {"Element;", "\342\210\210"}, + {"Emacr;", "\304\222"}, + {"EmptySmallSquare;", "\342\227\273"}, + {"EmptyVerySmallSquare;", "\342\226\253"}, + {"Eogon;", "\304\230"}, + {"Eopf;", "\360\235\224\274"}, + {"Epsilon;", "\316\225"}, + {"Equal;", "\342\251\265"}, + {"EqualTilde;", "\342\211\202"}, + {"Equilibrium;", "\342\207\214"}, + {"Escr;", "\342\204\260"}, + {"Esim;", "\342\251\263"}, + {"Eta;", "\316\227"}, + {"Euml", "\303\213"}, + {"Euml;", "\303\213"}, + {"Exists;", "\342\210\203"}, + {"ExponentialE;", "\342\205\207"}, + {"Fcy;", "\320\244"}, + {"Ffr;", "\360\235\224\211"}, + {"FilledSmallSquare;", "\342\227\274"}, + {"FilledVerySmallSquare;", "\342\226\252"}, + {"Fopf;", "\360\235\224\275"}, + {"ForAll;", "\342\210\200"}, + {"Fouriertrf;", "\342\204\261"}, + {"Fscr;", "\342\204\261"}, + {"GJcy;", "\320\203"}, + {"GT", ">"}, + {"GT;", ">"}, + {"Gamma;", "\316\223"}, + {"Gammad;", "\317\234"}, + {"Gbreve;", "\304\236"}, + {"Gcedil;", "\304\242"}, + {"Gcirc;", "\304\234"}, + {"Gcy;", "\320\223"}, + {"Gdot;", "\304\240"}, + {"Gfr;", "\360\235\224\212"}, + {"Gg;", "\342\213\231"}, + {"Gopf;", "\360\235\224\276"}, + {"GreaterEqual;", "\342\211\245"}, + {"GreaterEqualLess;", "\342\213\233"}, + {"GreaterFullEqual;", "\342\211\247"}, + {"GreaterGreater;", "\342\252\242"}, + {"GreaterLess;", "\342\211\267"}, + {"GreaterSlantEqual;", "\342\251\276"}, + {"GreaterTilde;", "\342\211\263"}, + {"Gscr;", "\360\235\222\242"}, + {"Gt;", "\342\211\253"}, + {"HARDcy;", "\320\252"}, + {"Hacek;", "\313\207"}, + {"Hat;", "^"}, + {"Hcirc;", "\304\244"}, + {"Hfr;", "\342\204\214"}, + {"HilbertSpace;", "\342\204\213"}, + {"Hopf;", "\342\204\215"}, + {"HorizontalLine;", "\342\224\200"}, + {"Hscr;", "\342\204\213"}, + {"Hstrok;", "\304\246"}, + {"HumpDownHump;", "\342\211\216"}, + {"HumpEqual;", "\342\211\217"}, + {"IEcy;", "\320\225"}, + {"IJlig;", "\304\262"}, + {"IOcy;", "\320\201"}, + {"Iacute", "\303\215"}, + {"Iacute;", "\303\215"}, + {"Icirc", "\303\216"}, + {"Icirc;", "\303\216"}, + {"Icy;", "\320\230"}, + {"Idot;", "\304\260"}, + {"Ifr;", "\342\204\221"}, + {"Igrave", "\303\214"}, + {"Igrave;", "\303\214"}, + {"Im;", "\342\204\221"}, + {"Imacr;", "\304\252"}, + {"ImaginaryI;", "\342\205\210"}, + {"Implies;", "\342\207\222"}, + {"Int;", "\342\210\254"}, + {"Integral;", "\342\210\253"}, + {"Intersection;", "\342\213\202"}, + {"InvisibleComma;", "\342\201\243"}, + {"InvisibleTimes;", "\342\201\242"}, + {"Iogon;", "\304\256"}, + {"Iopf;", "\360\235\225\200"}, + {"Iota;", "\316\231"}, + {"Iscr;", "\342\204\220"}, + {"Itilde;", "\304\250"}, + {"Iukcy;", "\320\206"}, + {"Iuml", "\303\217"}, + {"Iuml;", "\303\217"}, + {"Jcirc;", "\304\264"}, + {"Jcy;", "\320\231"}, + {"Jfr;", "\360\235\224\215"}, + {"Jopf;", "\360\235\225\201"}, + {"Jscr;", "\360\235\222\245"}, + {"Jsercy;", "\320\210"}, + {"Jukcy;", "\320\204"}, + {"KHcy;", "\320\245"}, + {"KJcy;", "\320\214"}, + {"Kappa;", "\316\232"}, + {"Kcedil;", "\304\266"}, + {"Kcy;", "\320\232"}, + {"Kfr;", "\360\235\224\216"}, + {"Kopf;", "\360\235\225\202"}, + {"Kscr;", "\360\235\222\246"}, + {"LJcy;", "\320\211"}, + {"LT", "<"}, + {"LT;", "<"}, + {"Lacute;", "\304\271"}, + {"Lambda;", "\316\233"}, + {"Lang;", "\342\237\252"}, + {"Laplacetrf;", "\342\204\222"}, + {"Larr;", "\342\206\236"}, + {"Lcaron;", "\304\275"}, + {"Lcedil;", "\304\273"}, + {"Lcy;", "\320\233"}, + {"LeftAngleBracket;", "\342\237\250"}, + {"LeftArrow;", "\342\206\220"}, + {"LeftArrowBar;", "\342\207\244"}, + {"LeftArrowRightArrow;", "\342\207\206"}, + {"LeftCeiling;", "\342\214\210"}, + {"LeftDoubleBracket;", "\342\237\246"}, + {"LeftDownTeeVector;", "\342\245\241"}, + {"LeftDownVector;", "\342\207\203"}, + {"LeftDownVectorBar;", "\342\245\231"}, + {"LeftFloor;", "\342\214\212"}, + {"LeftRightArrow;", "\342\206\224"}, + {"LeftRightVector;", "\342\245\216"}, + {"LeftTee;", "\342\212\243"}, + {"LeftTeeArrow;", "\342\206\244"}, + {"LeftTeeVector;", "\342\245\232"}, + {"LeftTriangle;", "\342\212\262"}, + {"LeftTriangleBar;", "\342\247\217"}, + {"LeftTriangleEqual;", "\342\212\264"}, + {"LeftUpDownVector;", "\342\245\221"}, + {"LeftUpTeeVector;", "\342\245\240"}, + {"LeftUpVector;", "\342\206\277"}, + {"LeftUpVectorBar;", "\342\245\230"}, + {"LeftVector;", "\342\206\274"}, + {"LeftVectorBar;", "\342\245\222"}, + {"Leftarrow;", "\342\207\220"}, + {"Leftrightarrow;", "\342\207\224"}, + {"LessEqualGreater;", "\342\213\232"}, + {"LessFullEqual;", "\342\211\246"}, + {"LessGreater;", "\342\211\266"}, + {"LessLess;", "\342\252\241"}, + {"LessSlantEqual;", "\342\251\275"}, + {"LessTilde;", "\342\211\262"}, + {"Lfr;", "\360\235\224\217"}, + {"Ll;", "\342\213\230"}, + {"Lleftarrow;", "\342\207\232"}, + {"Lmidot;", "\304\277"}, + {"LongLeftArrow;", "\342\237\265"}, + {"LongLeftRightArrow;", "\342\237\267"}, + {"LongRightArrow;", "\342\237\266"}, + {"Longleftarrow;", "\342\237\270"}, + {"Longleftrightarrow;", "\342\237\272"}, + {"Longrightarrow;", "\342\237\271"}, + {"Lopf;", "\360\235\225\203"}, + {"LowerLeftArrow;", "\342\206\231"}, + {"LowerRightArrow;", "\342\206\230"}, + {"Lscr;", "\342\204\222"}, + {"Lsh;", "\342\206\260"}, + {"Lstrok;", "\305\201"}, + {"Lt;", "\342\211\252"}, + {"Map;", "\342\244\205"}, + {"Mcy;", "\320\234"}, + {"MediumSpace;", "\342\201\237"}, + {"Mellintrf;", "\342\204\263"}, + {"Mfr;", "\360\235\224\220"}, + {"MinusPlus;", "\342\210\223"}, + {"Mopf;", "\360\235\225\204"}, + {"Mscr;", "\342\204\263"}, + {"Mu;", "\316\234"}, + {"NJcy;", "\320\212"}, + {"Nacute;", "\305\203"}, + {"Ncaron;", "\305\207"}, + {"Ncedil;", "\305\205"}, + {"Ncy;", "\320\235"}, + {"NegativeMediumSpace;", "\342\200\213"}, + {"NegativeThickSpace;", "\342\200\213"}, + {"NegativeThinSpace;", "\342\200\213"}, + {"NegativeVeryThinSpace;", "\342\200\213"}, + {"NestedGreaterGreater;", "\342\211\253"}, + {"NestedLessLess;", "\342\211\252"}, + {"NewLine;", "\012"}, + {"Nfr;", "\360\235\224\221"}, + {"NoBreak;", "\342\201\240"}, + {"NonBreakingSpace;", "\302\240"}, + {"Nopf;", "\342\204\225"}, + {"Not;", "\342\253\254"}, + {"NotCongruent;", "\342\211\242"}, + {"NotCupCap;", "\342\211\255"}, + {"NotDoubleVerticalBar;", "\342\210\246"}, + {"NotElement;", "\342\210\211"}, + {"NotEqual;", "\342\211\240"}, + {"NotEqualTilde;", "\342\211\202\314\270"}, + {"NotExists;", "\342\210\204"}, + {"NotGreater;", "\342\211\257"}, + {"NotGreaterEqual;", "\342\211\261"}, + {"NotGreaterFullEqual;", "\342\211\247\314\270"}, + {"NotGreaterGreater;", "\342\211\253\314\270"}, + {"NotGreaterLess;", "\342\211\271"}, + {"NotGreaterSlantEqual;", "\342\251\276\314\270"}, + {"NotGreaterTilde;", "\342\211\265"}, + {"NotHumpDownHump;", "\342\211\216\314\270"}, + {"NotHumpEqual;", "\342\211\217\314\270"}, + {"NotLeftTriangle;", "\342\213\252"}, + {"NotLeftTriangleBar;", "\342\247\217\314\270"}, + {"NotLeftTriangleEqual;", "\342\213\254"}, + {"NotLess;", "\342\211\256"}, + {"NotLessEqual;", "\342\211\260"}, + {"NotLessGreater;", "\342\211\270"}, + {"NotLessLess;", "\342\211\252\314\270"}, + {"NotLessSlantEqual;", "\342\251\275\314\270"}, + {"NotLessTilde;", "\342\211\264"}, + {"NotNestedGreaterGreater;", "\342\252\242\314\270"}, + {"NotNestedLessLess;", "\342\252\241\314\270"}, + {"NotPrecedes;", "\342\212\200"}, + {"NotPrecedesEqual;", "\342\252\257\314\270"}, + {"NotPrecedesSlantEqual;", "\342\213\240"}, + {"NotReverseElement;", "\342\210\214"}, + {"NotRightTriangle;", "\342\213\253"}, + {"NotRightTriangleBar;", "\342\247\220\314\270"}, + {"NotRightTriangleEqual;", "\342\213\255"}, + {"NotSquareSubset;", "\342\212\217\314\270"}, + {"NotSquareSubsetEqual;", "\342\213\242"}, + {"NotSquareSuperset;", "\342\212\220\314\270"}, + {"NotSquareSupersetEqual;", "\342\213\243"}, + {"NotSubset;", "\342\212\202\342\203\222"}, + {"NotSubsetEqual;", "\342\212\210"}, + {"NotSucceeds;", "\342\212\201"}, + {"NotSucceedsEqual;", "\342\252\260\314\270"}, + {"NotSucceedsSlantEqual;", "\342\213\241"}, + {"NotSucceedsTilde;", "\342\211\277\314\270"}, + {"NotSuperset;", "\342\212\203\342\203\222"}, + {"NotSupersetEqual;", "\342\212\211"}, + {"NotTilde;", "\342\211\201"}, + {"NotTildeEqual;", "\342\211\204"}, + {"NotTildeFullEqual;", "\342\211\207"}, + {"NotTildeTilde;", "\342\211\211"}, + {"NotVerticalBar;", "\342\210\244"}, + {"Nscr;", "\360\235\222\251"}, + {"Ntilde", "\303\221"}, + {"Ntilde;", "\303\221"}, + {"Nu;", "\316\235"}, + {"OElig;", "\305\222"}, + {"Oacute", "\303\223"}, + {"Oacute;", "\303\223"}, + {"Ocirc", "\303\224"}, + {"Ocirc;", "\303\224"}, + {"Ocy;", "\320\236"}, + {"Odblac;", "\305\220"}, + {"Ofr;", "\360\235\224\222"}, + {"Ograve", "\303\222"}, + {"Ograve;", "\303\222"}, + {"Omacr;", "\305\214"}, + {"Omega;", "\316\251"}, + {"Omicron;", "\316\237"}, + {"Oopf;", "\360\235\225\206"}, + {"OpenCurlyDoubleQuote;", "\342\200\234"}, + {"OpenCurlyQuote;", "\342\200\230"}, + {"Or;", "\342\251\224"}, + {"Oscr;", "\360\235\222\252"}, + {"Oslash", "\303\230"}, + {"Oslash;", "\303\230"}, + {"Otilde", "\303\225"}, + {"Otilde;", "\303\225"}, + {"Otimes;", "\342\250\267"}, + {"Ouml", "\303\226"}, + {"Ouml;", "\303\226"}, + {"OverBar;", "\342\200\276"}, + {"OverBrace;", "\342\217\236"}, + {"OverBracket;", "\342\216\264"}, + {"OverParenthesis;", "\342\217\234"}, + {"PartialD;", "\342\210\202"}, + {"Pcy;", "\320\237"}, + {"Pfr;", "\360\235\224\223"}, + {"Phi;", "\316\246"}, + {"Pi;", "\316\240"}, + {"PlusMinus;", "\302\261"}, + {"Poincareplane;", "\342\204\214"}, + {"Popf;", "\342\204\231"}, + {"Pr;", "\342\252\273"}, + {"Precedes;", "\342\211\272"}, + {"PrecedesEqual;", "\342\252\257"}, + {"PrecedesSlantEqual;", "\342\211\274"}, + {"PrecedesTilde;", "\342\211\276"}, + {"Prime;", "\342\200\263"}, + {"Product;", "\342\210\217"}, + {"Proportion;", "\342\210\267"}, + {"Proportional;", "\342\210\235"}, + {"Pscr;", "\360\235\222\253"}, + {"Psi;", "\316\250"}, + {"QUOT", "\""}, + {"QUOT;", "\""}, + {"Qfr;", "\360\235\224\224"}, + {"Qopf;", "\342\204\232"}, + {"Qscr;", "\360\235\222\254"}, + {"RBarr;", "\342\244\220"}, + {"REG", "\302\256"}, + {"REG;", "\302\256"}, + {"Racute;", "\305\224"}, + {"Rang;", "\342\237\253"}, + {"Rarr;", "\342\206\240"}, + {"Rarrtl;", "\342\244\226"}, + {"Rcaron;", "\305\230"}, + {"Rcedil;", "\305\226"}, + {"Rcy;", "\320\240"}, + {"Re;", "\342\204\234"}, + {"ReverseElement;", "\342\210\213"}, + {"ReverseEquilibrium;", "\342\207\213"}, + {"ReverseUpEquilibrium;", "\342\245\257"}, + {"Rfr;", "\342\204\234"}, + {"Rho;", "\316\241"}, + {"RightAngleBracket;", "\342\237\251"}, + {"RightArrow;", "\342\206\222"}, + {"RightArrowBar;", "\342\207\245"}, + {"RightArrowLeftArrow;", "\342\207\204"}, + {"RightCeiling;", "\342\214\211"}, + {"RightDoubleBracket;", "\342\237\247"}, + {"RightDownTeeVector;", "\342\245\235"}, + {"RightDownVector;", "\342\207\202"}, + {"RightDownVectorBar;", "\342\245\225"}, + {"RightFloor;", "\342\214\213"}, + {"RightTee;", "\342\212\242"}, + {"RightTeeArrow;", "\342\206\246"}, + {"RightTeeVector;", "\342\245\233"}, + {"RightTriangle;", "\342\212\263"}, + {"RightTriangleBar;", "\342\247\220"}, + {"RightTriangleEqual;", "\342\212\265"}, + {"RightUpDownVector;", "\342\245\217"}, + {"RightUpTeeVector;", "\342\245\234"}, + {"RightUpVector;", "\342\206\276"}, + {"RightUpVectorBar;", "\342\245\224"}, + {"RightVector;", "\342\207\200"}, + {"RightVectorBar;", "\342\245\223"}, + {"Rightarrow;", "\342\207\222"}, + {"Ropf;", "\342\204\235"}, + {"RoundImplies;", "\342\245\260"}, + {"Rrightarrow;", "\342\207\233"}, + {"Rscr;", "\342\204\233"}, + {"Rsh;", "\342\206\261"}, + {"RuleDelayed;", "\342\247\264"}, + {"SHCHcy;", "\320\251"}, + {"SHcy;", "\320\250"}, + {"SOFTcy;", "\320\254"}, + {"Sacute;", "\305\232"}, + {"Sc;", "\342\252\274"}, + {"Scaron;", "\305\240"}, + {"Scedil;", "\305\236"}, + {"Scirc;", "\305\234"}, + {"Scy;", "\320\241"}, + {"Sfr;", "\360\235\224\226"}, + {"ShortDownArrow;", "\342\206\223"}, + {"ShortLeftArrow;", "\342\206\220"}, + {"ShortRightArrow;", "\342\206\222"}, + {"ShortUpArrow;", "\342\206\221"}, + {"Sigma;", "\316\243"}, + {"SmallCircle;", "\342\210\230"}, + {"Sopf;", "\360\235\225\212"}, + {"Sqrt;", "\342\210\232"}, + {"Square;", "\342\226\241"}, + {"SquareIntersection;", "\342\212\223"}, + {"SquareSubset;", "\342\212\217"}, + {"SquareSubsetEqual;", "\342\212\221"}, + {"SquareSuperset;", "\342\212\220"}, + {"SquareSupersetEqual;", "\342\212\222"}, + {"SquareUnion;", "\342\212\224"}, + {"Sscr;", "\360\235\222\256"}, + {"Star;", "\342\213\206"}, + {"Sub;", "\342\213\220"}, + {"Subset;", "\342\213\220"}, + {"SubsetEqual;", "\342\212\206"}, + {"Succeeds;", "\342\211\273"}, + {"SucceedsEqual;", "\342\252\260"}, + {"SucceedsSlantEqual;", "\342\211\275"}, + {"SucceedsTilde;", "\342\211\277"}, + {"SuchThat;", "\342\210\213"}, + {"Sum;", "\342\210\221"}, + {"Sup;", "\342\213\221"}, + {"Superset;", "\342\212\203"}, + {"SupersetEqual;", "\342\212\207"}, + {"Supset;", "\342\213\221"}, + {"THORN", "\303\236"}, + {"THORN;", "\303\236"}, + {"TRADE;", "\342\204\242"}, + {"TSHcy;", "\320\213"}, + {"TScy;", "\320\246"}, + {"Tab;", "\011"}, + {"Tau;", "\316\244"}, + {"Tcaron;", "\305\244"}, + {"Tcedil;", "\305\242"}, + {"Tcy;", "\320\242"}, + {"Tfr;", "\360\235\224\227"}, + {"Therefore;", "\342\210\264"}, + {"Theta;", "\316\230"}, + {"ThickSpace;", "\342\201\237\342\200\212"}, + {"ThinSpace;", "\342\200\211"}, + {"Tilde;", "\342\210\274"}, + {"TildeEqual;", "\342\211\203"}, + {"TildeFullEqual;", "\342\211\205"}, + {"TildeTilde;", "\342\211\210"}, + {"Topf;", "\360\235\225\213"}, + {"TripleDot;", "\342\203\233"}, + {"Tscr;", "\360\235\222\257"}, + {"Tstrok;", "\305\246"}, + {"Uacute", "\303\232"}, + {"Uacute;", "\303\232"}, + {"Uarr;", "\342\206\237"}, + {"Uarrocir;", "\342\245\211"}, + {"Ubrcy;", "\320\216"}, + {"Ubreve;", "\305\254"}, + {"Ucirc", "\303\233"}, + {"Ucirc;", "\303\233"}, + {"Ucy;", "\320\243"}, + {"Udblac;", "\305\260"}, + {"Ufr;", "\360\235\224\230"}, + {"Ugrave", "\303\231"}, + {"Ugrave;", "\303\231"}, + {"Umacr;", "\305\252"}, + {"UnderBar;", "_"}, + {"UnderBrace;", "\342\217\237"}, + {"UnderBracket;", "\342\216\265"}, + {"UnderParenthesis;", "\342\217\235"}, + {"Union;", "\342\213\203"}, + {"UnionPlus;", "\342\212\216"}, + {"Uogon;", "\305\262"}, + {"Uopf;", "\360\235\225\214"}, + {"UpArrow;", "\342\206\221"}, + {"UpArrowBar;", "\342\244\222"}, + {"UpArrowDownArrow;", "\342\207\205"}, + {"UpDownArrow;", "\342\206\225"}, + {"UpEquilibrium;", "\342\245\256"}, + {"UpTee;", "\342\212\245"}, + {"UpTeeArrow;", "\342\206\245"}, + {"Uparrow;", "\342\207\221"}, + {"Updownarrow;", "\342\207\225"}, + {"UpperLeftArrow;", "\342\206\226"}, + {"UpperRightArrow;", "\342\206\227"}, + {"Upsi;", "\317\222"}, + {"Upsilon;", "\316\245"}, + {"Uring;", "\305\256"}, + {"Uscr;", "\360\235\222\260"}, + {"Utilde;", "\305\250"}, + {"Uuml", "\303\234"}, + {"Uuml;", "\303\234"}, + {"VDash;", "\342\212\253"}, + {"Vbar;", "\342\253\253"}, + {"Vcy;", "\320\222"}, + {"Vdash;", "\342\212\251"}, + {"Vdashl;", "\342\253\246"}, + {"Vee;", "\342\213\201"}, + {"Verbar;", "\342\200\226"}, + {"Vert;", "\342\200\226"}, + {"VerticalBar;", "\342\210\243"}, + {"VerticalLine;", "|"}, + {"VerticalSeparator;", "\342\235\230"}, + {"VerticalTilde;", "\342\211\200"}, + {"VeryThinSpace;", "\342\200\212"}, + {"Vfr;", "\360\235\224\231"}, + {"Vopf;", "\360\235\225\215"}, + {"Vscr;", "\360\235\222\261"}, + {"Vvdash;", "\342\212\252"}, + {"Wcirc;", "\305\264"}, + {"Wedge;", "\342\213\200"}, + {"Wfr;", "\360\235\224\232"}, + {"Wopf;", "\360\235\225\216"}, + {"Wscr;", "\360\235\222\262"}, + {"Xfr;", "\360\235\224\233"}, + {"Xi;", "\316\236"}, + {"Xopf;", "\360\235\225\217"}, + {"Xscr;", "\360\235\222\263"}, + {"YAcy;", "\320\257"}, + {"YIcy;", "\320\207"}, + {"YUcy;", "\320\256"}, + {"Yacute", "\303\235"}, + {"Yacute;", "\303\235"}, + {"Ycirc;", "\305\266"}, + {"Ycy;", "\320\253"}, + {"Yfr;", "\360\235\224\234"}, + {"Yopf;", "\360\235\225\220"}, + {"Yscr;", "\360\235\222\264"}, + {"Yuml;", "\305\270"}, + {"ZHcy;", "\320\226"}, + {"Zacute;", "\305\271"}, + {"Zcaron;", "\305\275"}, + {"Zcy;", "\320\227"}, + {"Zdot;", "\305\273"}, + {"ZeroWidthSpace;", "\342\200\213"}, + {"Zeta;", "\316\226"}, + {"Zfr;", "\342\204\250"}, + {"Zopf;", "\342\204\244"}, + {"Zscr;", "\360\235\222\265"}, + {"aacute", "\303\241"}, + {"aacute;", "\303\241"}, + {"abreve;", "\304\203"}, + {"ac;", "\342\210\276"}, + {"acE;", "\342\210\276\314\263"}, + {"acd;", "\342\210\277"}, + {"acirc", "\303\242"}, + {"acirc;", "\303\242"}, + {"acute", "\302\264"}, + {"acute;", "\302\264"}, + {"acy;", "\320\260"}, + {"aelig", "\303\246"}, + {"aelig;", "\303\246"}, + {"af;", "\342\201\241"}, + {"afr;", "\360\235\224\236"}, + {"agrave", "\303\240"}, + {"agrave;", "\303\240"}, + {"alefsym;", "\342\204\265"}, + {"aleph;", "\342\204\265"}, + {"alpha;", "\316\261"}, + {"amacr;", "\304\201"}, + {"amalg;", "\342\250\277"}, + {"amp", "&"}, + {"amp;", "&"}, + {"and;", "\342\210\247"}, + {"andand;", "\342\251\225"}, + {"andd;", "\342\251\234"}, + {"andslope;", "\342\251\230"}, + {"andv;", "\342\251\232"}, + {"ang;", "\342\210\240"}, + {"ange;", "\342\246\244"}, + {"angle;", "\342\210\240"}, + {"angmsd;", "\342\210\241"}, + {"angmsdaa;", "\342\246\250"}, + {"angmsdab;", "\342\246\251"}, + {"angmsdac;", "\342\246\252"}, + {"angmsdad;", "\342\246\253"}, + {"angmsdae;", "\342\246\254"}, + {"angmsdaf;", "\342\246\255"}, + {"angmsdag;", "\342\246\256"}, + {"angmsdah;", "\342\246\257"}, + {"angrt;", "\342\210\237"}, + {"angrtvb;", "\342\212\276"}, + {"angrtvbd;", "\342\246\235"}, + {"angsph;", "\342\210\242"}, + {"angst;", "\303\205"}, + {"angzarr;", "\342\215\274"}, + {"aogon;", "\304\205"}, + {"aopf;", "\360\235\225\222"}, + {"ap;", "\342\211\210"}, + {"apE;", "\342\251\260"}, + {"apacir;", "\342\251\257"}, + {"ape;", "\342\211\212"}, + {"apid;", "\342\211\213"}, + {"apos;", "'"}, + {"approx;", "\342\211\210"}, + {"approxeq;", "\342\211\212"}, + {"aring", "\303\245"}, + {"aring;", "\303\245"}, + {"ascr;", "\360\235\222\266"}, + {"ast;", "*"}, + {"asymp;", "\342\211\210"}, + {"asympeq;", "\342\211\215"}, + {"atilde", "\303\243"}, + {"atilde;", "\303\243"}, + {"auml", "\303\244"}, + {"auml;", "\303\244"}, + {"awconint;", "\342\210\263"}, + {"awint;", "\342\250\221"}, + {"bNot;", "\342\253\255"}, + {"backcong;", "\342\211\214"}, + {"backepsilon;", "\317\266"}, + {"backprime;", "\342\200\265"}, + {"backsim;", "\342\210\275"}, + {"backsimeq;", "\342\213\215"}, + {"barvee;", "\342\212\275"}, + {"barwed;", "\342\214\205"}, + {"barwedge;", "\342\214\205"}, + {"bbrk;", "\342\216\265"}, + {"bbrktbrk;", "\342\216\266"}, + {"bcong;", "\342\211\214"}, + {"bcy;", "\320\261"}, + {"bdquo;", "\342\200\236"}, + {"becaus;", "\342\210\265"}, + {"because;", "\342\210\265"}, + {"bemptyv;", "\342\246\260"}, + {"bepsi;", "\317\266"}, + {"bernou;", "\342\204\254"}, + {"beta;", "\316\262"}, + {"beth;", "\342\204\266"}, + {"between;", "\342\211\254"}, + {"bfr;", "\360\235\224\237"}, + {"bigcap;", "\342\213\202"}, + {"bigcirc;", "\342\227\257"}, + {"bigcup;", "\342\213\203"}, + {"bigodot;", "\342\250\200"}, + {"bigoplus;", "\342\250\201"}, + {"bigotimes;", "\342\250\202"}, + {"bigsqcup;", "\342\250\206"}, + {"bigstar;", "\342\230\205"}, + {"bigtriangledown;", "\342\226\275"}, + {"bigtriangleup;", "\342\226\263"}, + {"biguplus;", "\342\250\204"}, + {"bigvee;", "\342\213\201"}, + {"bigwedge;", "\342\213\200"}, + {"bkarow;", "\342\244\215"}, + {"blacklozenge;", "\342\247\253"}, + {"blacksquare;", "\342\226\252"}, + {"blacktriangle;", "\342\226\264"}, + {"blacktriangledown;", "\342\226\276"}, + {"blacktriangleleft;", "\342\227\202"}, + {"blacktriangleright;", "\342\226\270"}, + {"blank;", "\342\220\243"}, + {"blk12;", "\342\226\222"}, + {"blk14;", "\342\226\221"}, + {"blk34;", "\342\226\223"}, + {"block;", "\342\226\210"}, + {"bne;", "=\342\203\245"}, + {"bnequiv;", "\342\211\241\342\203\245"}, + {"bnot;", "\342\214\220"}, + {"bopf;", "\360\235\225\223"}, + {"bot;", "\342\212\245"}, + {"bottom;", "\342\212\245"}, + {"bowtie;", "\342\213\210"}, + {"boxDL;", "\342\225\227"}, + {"boxDR;", "\342\225\224"}, + {"boxDl;", "\342\225\226"}, + {"boxDr;", "\342\225\223"}, + {"boxH;", "\342\225\220"}, + {"boxHD;", "\342\225\246"}, + {"boxHU;", "\342\225\251"}, + {"boxHd;", "\342\225\244"}, + {"boxHu;", "\342\225\247"}, + {"boxUL;", "\342\225\235"}, + {"boxUR;", "\342\225\232"}, + {"boxUl;", "\342\225\234"}, + {"boxUr;", "\342\225\231"}, + {"boxV;", "\342\225\221"}, + {"boxVH;", "\342\225\254"}, + {"boxVL;", "\342\225\243"}, + {"boxVR;", "\342\225\240"}, + {"boxVh;", "\342\225\253"}, + {"boxVl;", "\342\225\242"}, + {"boxVr;", "\342\225\237"}, + {"boxbox;", "\342\247\211"}, + {"boxdL;", "\342\225\225"}, + {"boxdR;", "\342\225\222"}, + {"boxdl;", "\342\224\220"}, + {"boxdr;", "\342\224\214"}, + {"boxh;", "\342\224\200"}, + {"boxhD;", "\342\225\245"}, + {"boxhU;", "\342\225\250"}, + {"boxhd;", "\342\224\254"}, + {"boxhu;", "\342\224\264"}, + {"boxminus;", "\342\212\237"}, + {"boxplus;", "\342\212\236"}, + {"boxtimes;", "\342\212\240"}, + {"boxuL;", "\342\225\233"}, + {"boxuR;", "\342\225\230"}, + {"boxul;", "\342\224\230"}, + {"boxur;", "\342\224\224"}, + {"boxv;", "\342\224\202"}, + {"boxvH;", "\342\225\252"}, + {"boxvL;", "\342\225\241"}, + {"boxvR;", "\342\225\236"}, + {"boxvh;", "\342\224\274"}, + {"boxvl;", "\342\224\244"}, + {"boxvr;", "\342\224\234"}, + {"bprime;", "\342\200\265"}, + {"breve;", "\313\230"}, + {"brvbar", "\302\246"}, + {"brvbar;", "\302\246"}, + {"bscr;", "\360\235\222\267"}, + {"bsemi;", "\342\201\217"}, + {"bsim;", "\342\210\275"}, + {"bsime;", "\342\213\215"}, + {"bsol;", "\\"}, + {"bsolb;", "\342\247\205"}, + {"bsolhsub;", "\342\237\210"}, + {"bull;", "\342\200\242"}, + {"bullet;", "\342\200\242"}, + {"bump;", "\342\211\216"}, + {"bumpE;", "\342\252\256"}, + {"bumpe;", "\342\211\217"}, + {"bumpeq;", "\342\211\217"}, + {"cacute;", "\304\207"}, + {"cap;", "\342\210\251"}, + {"capand;", "\342\251\204"}, + {"capbrcup;", "\342\251\211"}, + {"capcap;", "\342\251\213"}, + {"capcup;", "\342\251\207"}, + {"capdot;", "\342\251\200"}, + {"caps;", "\342\210\251\357\270\200"}, + {"caret;", "\342\201\201"}, + {"caron;", "\313\207"}, + {"ccaps;", "\342\251\215"}, + {"ccaron;", "\304\215"}, + {"ccedil", "\303\247"}, + {"ccedil;", "\303\247"}, + {"ccirc;", "\304\211"}, + {"ccups;", "\342\251\214"}, + {"ccupssm;", "\342\251\220"}, + {"cdot;", "\304\213"}, + {"cedil", "\302\270"}, + {"cedil;", "\302\270"}, + {"cemptyv;", "\342\246\262"}, + {"cent", "\302\242"}, + {"cent;", "\302\242"}, + {"centerdot;", "\302\267"}, + {"cfr;", "\360\235\224\240"}, + {"chcy;", "\321\207"}, + {"check;", "\342\234\223"}, + {"checkmark;", "\342\234\223"}, + {"chi;", "\317\207"}, + {"cir;", "\342\227\213"}, + {"cirE;", "\342\247\203"}, + {"circ;", "\313\206"}, + {"circeq;", "\342\211\227"}, + {"circlearrowleft;", "\342\206\272"}, + {"circlearrowright;", "\342\206\273"}, + {"circledR;", "\302\256"}, + {"circledS;", "\342\223\210"}, + {"circledast;", "\342\212\233"}, + {"circledcirc;", "\342\212\232"}, + {"circleddash;", "\342\212\235"}, + {"cire;", "\342\211\227"}, + {"cirfnint;", "\342\250\220"}, + {"cirmid;", "\342\253\257"}, + {"cirscir;", "\342\247\202"}, + {"clubs;", "\342\231\243"}, + {"clubsuit;", "\342\231\243"}, + {"colon;", ":"}, + {"colone;", "\342\211\224"}, + {"coloneq;", "\342\211\224"}, + {"comma;", ","}, + {"commat;", "@"}, + {"comp;", "\342\210\201"}, + {"compfn;", "\342\210\230"}, + {"complement;", "\342\210\201"}, + {"complexes;", "\342\204\202"}, + {"cong;", "\342\211\205"}, + {"congdot;", "\342\251\255"}, + {"conint;", "\342\210\256"}, + {"copf;", "\360\235\225\224"}, + {"coprod;", "\342\210\220"}, + {"copy", "\302\251"}, + {"copy;", "\302\251"}, + {"copysr;", "\342\204\227"}, + {"crarr;", "\342\206\265"}, + {"cross;", "\342\234\227"}, + {"cscr;", "\360\235\222\270"}, + {"csub;", "\342\253\217"}, + {"csube;", "\342\253\221"}, + {"csup;", "\342\253\220"}, + {"csupe;", "\342\253\222"}, + {"ctdot;", "\342\213\257"}, + {"cudarrl;", "\342\244\270"}, + {"cudarrr;", "\342\244\265"}, + {"cuepr;", "\342\213\236"}, + {"cuesc;", "\342\213\237"}, + {"cularr;", "\342\206\266"}, + {"cularrp;", "\342\244\275"}, + {"cup;", "\342\210\252"}, + {"cupbrcap;", "\342\251\210"}, + {"cupcap;", "\342\251\206"}, + {"cupcup;", "\342\251\212"}, + {"cupdot;", "\342\212\215"}, + {"cupor;", "\342\251\205"}, + {"cups;", "\342\210\252\357\270\200"}, + {"curarr;", "\342\206\267"}, + {"curarrm;", "\342\244\274"}, + {"curlyeqprec;", "\342\213\236"}, + {"curlyeqsucc;", "\342\213\237"}, + {"curlyvee;", "\342\213\216"}, + {"curlywedge;", "\342\213\217"}, + {"curren", "\302\244"}, + {"curren;", "\302\244"}, + {"curvearrowleft;", "\342\206\266"}, + {"curvearrowright;", "\342\206\267"}, + {"cuvee;", "\342\213\216"}, + {"cuwed;", "\342\213\217"}, + {"cwconint;", "\342\210\262"}, + {"cwint;", "\342\210\261"}, + {"cylcty;", "\342\214\255"}, + {"dArr;", "\342\207\223"}, + {"dHar;", "\342\245\245"}, + {"dagger;", "\342\200\240"}, + {"daleth;", "\342\204\270"}, + {"darr;", "\342\206\223"}, + {"dash;", "\342\200\220"}, + {"dashv;", "\342\212\243"}, + {"dbkarow;", "\342\244\217"}, + {"dblac;", "\313\235"}, + {"dcaron;", "\304\217"}, + {"dcy;", "\320\264"}, + {"dd;", "\342\205\206"}, + {"ddagger;", "\342\200\241"}, + {"ddarr;", "\342\207\212"}, + {"ddotseq;", "\342\251\267"}, + {"deg", "\302\260"}, + {"deg;", "\302\260"}, + {"delta;", "\316\264"}, + {"demptyv;", "\342\246\261"}, + {"dfisht;", "\342\245\277"}, + {"dfr;", "\360\235\224\241"}, + {"dharl;", "\342\207\203"}, + {"dharr;", "\342\207\202"}, + {"diam;", "\342\213\204"}, + {"diamond;", "\342\213\204"}, + {"diamondsuit;", "\342\231\246"}, + {"diams;", "\342\231\246"}, + {"die;", "\302\250"}, + {"digamma;", "\317\235"}, + {"disin;", "\342\213\262"}, + {"div;", "\303\267"}, + {"divide", "\303\267"}, + {"divide;", "\303\267"}, + {"divideontimes;", "\342\213\207"}, + {"divonx;", "\342\213\207"}, + {"djcy;", "\321\222"}, + {"dlcorn;", "\342\214\236"}, + {"dlcrop;", "\342\214\215"}, + {"dollar;", "$"}, + {"dopf;", "\360\235\225\225"}, + {"dot;", "\313\231"}, + {"doteq;", "\342\211\220"}, + {"doteqdot;", "\342\211\221"}, + {"dotminus;", "\342\210\270"}, + {"dotplus;", "\342\210\224"}, + {"dotsquare;", "\342\212\241"}, + {"doublebarwedge;", "\342\214\206"}, + {"downarrow;", "\342\206\223"}, + {"downdownarrows;", "\342\207\212"}, + {"downharpoonleft;", "\342\207\203"}, + {"downharpoonright;", "\342\207\202"}, + {"drbkarow;", "\342\244\220"}, + {"drcorn;", "\342\214\237"}, + {"drcrop;", "\342\214\214"}, + {"dscr;", "\360\235\222\271"}, + {"dscy;", "\321\225"}, + {"dsol;", "\342\247\266"}, + {"dstrok;", "\304\221"}, + {"dtdot;", "\342\213\261"}, + {"dtri;", "\342\226\277"}, + {"dtrif;", "\342\226\276"}, + {"duarr;", "\342\207\265"}, + {"duhar;", "\342\245\257"}, + {"dwangle;", "\342\246\246"}, + {"dzcy;", "\321\237"}, + {"dzigrarr;", "\342\237\277"}, + {"eDDot;", "\342\251\267"}, + {"eDot;", "\342\211\221"}, + {"eacute", "\303\251"}, + {"eacute;", "\303\251"}, + {"easter;", "\342\251\256"}, + {"ecaron;", "\304\233"}, + {"ecir;", "\342\211\226"}, + {"ecirc", "\303\252"}, + {"ecirc;", "\303\252"}, + {"ecolon;", "\342\211\225"}, + {"ecy;", "\321\215"}, + {"edot;", "\304\227"}, + {"ee;", "\342\205\207"}, + {"efDot;", "\342\211\222"}, + {"efr;", "\360\235\224\242"}, + {"eg;", "\342\252\232"}, + {"egrave", "\303\250"}, + {"egrave;", "\303\250"}, + {"egs;", "\342\252\226"}, + {"egsdot;", "\342\252\230"}, + {"el;", "\342\252\231"}, + {"elinters;", "\342\217\247"}, + {"ell;", "\342\204\223"}, + {"els;", "\342\252\225"}, + {"elsdot;", "\342\252\227"}, + {"emacr;", "\304\223"}, + {"empty;", "\342\210\205"}, + {"emptyset;", "\342\210\205"}, + {"emptyv;", "\342\210\205"}, + {"emsp13;", "\342\200\204"}, + {"emsp14;", "\342\200\205"}, + {"emsp;", "\342\200\203"}, + {"eng;", "\305\213"}, + {"ensp;", "\342\200\202"}, + {"eogon;", "\304\231"}, + {"eopf;", "\360\235\225\226"}, + {"epar;", "\342\213\225"}, + {"eparsl;", "\342\247\243"}, + {"eplus;", "\342\251\261"}, + {"epsi;", "\316\265"}, + {"epsilon;", "\316\265"}, + {"epsiv;", "\317\265"}, + {"eqcirc;", "\342\211\226"}, + {"eqcolon;", "\342\211\225"}, + {"eqsim;", "\342\211\202"}, + {"eqslantgtr;", "\342\252\226"}, + {"eqslantless;", "\342\252\225"}, + {"equals;", "="}, + {"equest;", "\342\211\237"}, + {"equiv;", "\342\211\241"}, + {"equivDD;", "\342\251\270"}, + {"eqvparsl;", "\342\247\245"}, + {"erDot;", "\342\211\223"}, + {"erarr;", "\342\245\261"}, + {"escr;", "\342\204\257"}, + {"esdot;", "\342\211\220"}, + {"esim;", "\342\211\202"}, + {"eta;", "\316\267"}, + {"eth", "\303\260"}, + {"eth;", "\303\260"}, + {"euml", "\303\253"}, + {"euml;", "\303\253"}, + {"euro;", "\342\202\254"}, + {"excl;", "!"}, + {"exist;", "\342\210\203"}, + {"expectation;", "\342\204\260"}, + {"exponentiale;", "\342\205\207"}, + {"fallingdotseq;", "\342\211\222"}, + {"fcy;", "\321\204"}, + {"female;", "\342\231\200"}, + {"ffilig;", "\357\254\203"}, + {"fflig;", "\357\254\200"}, + {"ffllig;", "\357\254\204"}, + {"ffr;", "\360\235\224\243"}, + {"filig;", "\357\254\201"}, + {"fjlig;", "fj"}, + {"flat;", "\342\231\255"}, + {"fllig;", "\357\254\202"}, + {"fltns;", "\342\226\261"}, + {"fnof;", "\306\222"}, + {"fopf;", "\360\235\225\227"}, + {"forall;", "\342\210\200"}, + {"fork;", "\342\213\224"}, + {"forkv;", "\342\253\231"}, + {"fpartint;", "\342\250\215"}, + {"frac12", "\302\275"}, + {"frac12;", "\302\275"}, + {"frac13;", "\342\205\223"}, + {"frac14", "\302\274"}, + {"frac14;", "\302\274"}, + {"frac15;", "\342\205\225"}, + {"frac16;", "\342\205\231"}, + {"frac18;", "\342\205\233"}, + {"frac23;", "\342\205\224"}, + {"frac25;", "\342\205\226"}, + {"frac34", "\302\276"}, + {"frac34;", "\302\276"}, + {"frac35;", "\342\205\227"}, + {"frac38;", "\342\205\234"}, + {"frac45;", "\342\205\230"}, + {"frac56;", "\342\205\232"}, + {"frac58;", "\342\205\235"}, + {"frac78;", "\342\205\236"}, + {"frasl;", "\342\201\204"}, + {"frown;", "\342\214\242"}, + {"fscr;", "\360\235\222\273"}, + {"gE;", "\342\211\247"}, + {"gEl;", "\342\252\214"}, + {"gacute;", "\307\265"}, + {"gamma;", "\316\263"}, + {"gammad;", "\317\235"}, + {"gap;", "\342\252\206"}, + {"gbreve;", "\304\237"}, + {"gcirc;", "\304\235"}, + {"gcy;", "\320\263"}, + {"gdot;", "\304\241"}, + {"ge;", "\342\211\245"}, + {"gel;", "\342\213\233"}, + {"geq;", "\342\211\245"}, + {"geqq;", "\342\211\247"}, + {"geqslant;", "\342\251\276"}, + {"ges;", "\342\251\276"}, + {"gescc;", "\342\252\251"}, + {"gesdot;", "\342\252\200"}, + {"gesdoto;", "\342\252\202"}, + {"gesdotol;", "\342\252\204"}, + {"gesl;", "\342\213\233\357\270\200"}, + {"gesles;", "\342\252\224"}, + {"gfr;", "\360\235\224\244"}, + {"gg;", "\342\211\253"}, + {"ggg;", "\342\213\231"}, + {"gimel;", "\342\204\267"}, + {"gjcy;", "\321\223"}, + {"gl;", "\342\211\267"}, + {"glE;", "\342\252\222"}, + {"gla;", "\342\252\245"}, + {"glj;", "\342\252\244"}, + {"gnE;", "\342\211\251"}, + {"gnap;", "\342\252\212"}, + {"gnapprox;", "\342\252\212"}, + {"gne;", "\342\252\210"}, + {"gneq;", "\342\252\210"}, + {"gneqq;", "\342\211\251"}, + {"gnsim;", "\342\213\247"}, + {"gopf;", "\360\235\225\230"}, + {"grave;", "`"}, + {"gscr;", "\342\204\212"}, + {"gsim;", "\342\211\263"}, + {"gsime;", "\342\252\216"}, + {"gsiml;", "\342\252\220"}, + {"gt", ">"}, + {"gt;", ">"}, + {"gtcc;", "\342\252\247"}, + {"gtcir;", "\342\251\272"}, + {"gtdot;", "\342\213\227"}, + {"gtlPar;", "\342\246\225"}, + {"gtquest;", "\342\251\274"}, + {"gtrapprox;", "\342\252\206"}, + {"gtrarr;", "\342\245\270"}, + {"gtrdot;", "\342\213\227"}, + {"gtreqless;", "\342\213\233"}, + {"gtreqqless;", "\342\252\214"}, + {"gtrless;", "\342\211\267"}, + {"gtrsim;", "\342\211\263"}, + {"gvertneqq;", "\342\211\251\357\270\200"}, + {"gvnE;", "\342\211\251\357\270\200"}, + {"hArr;", "\342\207\224"}, + {"hairsp;", "\342\200\212"}, + {"half;", "\302\275"}, + {"hamilt;", "\342\204\213"}, + {"hardcy;", "\321\212"}, + {"harr;", "\342\206\224"}, + {"harrcir;", "\342\245\210"}, + {"harrw;", "\342\206\255"}, + {"hbar;", "\342\204\217"}, + {"hcirc;", "\304\245"}, + {"hearts;", "\342\231\245"}, + {"heartsuit;", "\342\231\245"}, + {"hellip;", "\342\200\246"}, + {"hercon;", "\342\212\271"}, + {"hfr;", "\360\235\224\245"}, + {"hksearow;", "\342\244\245"}, + {"hkswarow;", "\342\244\246"}, + {"hoarr;", "\342\207\277"}, + {"homtht;", "\342\210\273"}, + {"hookleftarrow;", "\342\206\251"}, + {"hookrightarrow;", "\342\206\252"}, + {"hopf;", "\360\235\225\231"}, + {"horbar;", "\342\200\225"}, + {"hscr;", "\360\235\222\275"}, + {"hslash;", "\342\204\217"}, + {"hstrok;", "\304\247"}, + {"hybull;", "\342\201\203"}, + {"hyphen;", "\342\200\220"}, + {"iacute", "\303\255"}, + {"iacute;", "\303\255"}, + {"ic;", "\342\201\243"}, + {"icirc", "\303\256"}, + {"icirc;", "\303\256"}, + {"icy;", "\320\270"}, + {"iecy;", "\320\265"}, + {"iexcl", "\302\241"}, + {"iexcl;", "\302\241"}, + {"iff;", "\342\207\224"}, + {"ifr;", "\360\235\224\246"}, + {"igrave", "\303\254"}, + {"igrave;", "\303\254"}, + {"ii;", "\342\205\210"}, + {"iiiint;", "\342\250\214"}, + {"iiint;", "\342\210\255"}, + {"iinfin;", "\342\247\234"}, + {"iiota;", "\342\204\251"}, + {"ijlig;", "\304\263"}, + {"imacr;", "\304\253"}, + {"image;", "\342\204\221"}, + {"imagline;", "\342\204\220"}, + {"imagpart;", "\342\204\221"}, + {"imath;", "\304\261"}, + {"imof;", "\342\212\267"}, + {"imped;", "\306\265"}, + {"in;", "\342\210\210"}, + {"incare;", "\342\204\205"}, + {"infin;", "\342\210\236"}, + {"infintie;", "\342\247\235"}, + {"inodot;", "\304\261"}, + {"int;", "\342\210\253"}, + {"intcal;", "\342\212\272"}, + {"integers;", "\342\204\244"}, + {"intercal;", "\342\212\272"}, + {"intlarhk;", "\342\250\227"}, + {"intprod;", "\342\250\274"}, + {"iocy;", "\321\221"}, + {"iogon;", "\304\257"}, + {"iopf;", "\360\235\225\232"}, + {"iota;", "\316\271"}, + {"iprod;", "\342\250\274"}, + {"iquest", "\302\277"}, + {"iquest;", "\302\277"}, + {"iscr;", "\360\235\222\276"}, + {"isin;", "\342\210\210"}, + {"isinE;", "\342\213\271"}, + {"isindot;", "\342\213\265"}, + {"isins;", "\342\213\264"}, + {"isinsv;", "\342\213\263"}, + {"isinv;", "\342\210\210"}, + {"it;", "\342\201\242"}, + {"itilde;", "\304\251"}, + {"iukcy;", "\321\226"}, + {"iuml", "\303\257"}, + {"iuml;", "\303\257"}, + {"jcirc;", "\304\265"}, + {"jcy;", "\320\271"}, + {"jfr;", "\360\235\224\247"}, + {"jmath;", "\310\267"}, + {"jopf;", "\360\235\225\233"}, + {"jscr;", "\360\235\222\277"}, + {"jsercy;", "\321\230"}, + {"jukcy;", "\321\224"}, + {"kappa;", "\316\272"}, + {"kappav;", "\317\260"}, + {"kcedil;", "\304\267"}, + {"kcy;", "\320\272"}, + {"kfr;", "\360\235\224\250"}, + {"kgreen;", "\304\270"}, + {"khcy;", "\321\205"}, + {"kjcy;", "\321\234"}, + {"kopf;", "\360\235\225\234"}, + {"kscr;", "\360\235\223\200"}, + {"lAarr;", "\342\207\232"}, + {"lArr;", "\342\207\220"}, + {"lAtail;", "\342\244\233"}, + {"lBarr;", "\342\244\216"}, + {"lE;", "\342\211\246"}, + {"lEg;", "\342\252\213"}, + {"lHar;", "\342\245\242"}, + {"lacute;", "\304\272"}, + {"laemptyv;", "\342\246\264"}, + {"lagran;", "\342\204\222"}, + {"lambda;", "\316\273"}, + {"lang;", "\342\237\250"}, + {"langd;", "\342\246\221"}, + {"langle;", "\342\237\250"}, + {"lap;", "\342\252\205"}, + {"laquo", "\302\253"}, + {"laquo;", "\302\253"}, + {"larr;", "\342\206\220"}, + {"larrb;", "\342\207\244"}, + {"larrbfs;", "\342\244\237"}, + {"larrfs;", "\342\244\235"}, + {"larrhk;", "\342\206\251"}, + {"larrlp;", "\342\206\253"}, + {"larrpl;", "\342\244\271"}, + {"larrsim;", "\342\245\263"}, + {"larrtl;", "\342\206\242"}, + {"lat;", "\342\252\253"}, + {"latail;", "\342\244\231"}, + {"late;", "\342\252\255"}, + {"lates;", "\342\252\255\357\270\200"}, + {"lbarr;", "\342\244\214"}, + {"lbbrk;", "\342\235\262"}, + {"lbrace;", "{"}, + {"lbrack;", "["}, + {"lbrke;", "\342\246\213"}, + {"lbrksld;", "\342\246\217"}, + {"lbrkslu;", "\342\246\215"}, + {"lcaron;", "\304\276"}, + {"lcedil;", "\304\274"}, + {"lceil;", "\342\214\210"}, + {"lcub;", "{"}, + {"lcy;", "\320\273"}, + {"ldca;", "\342\244\266"}, + {"ldquo;", "\342\200\234"}, + {"ldquor;", "\342\200\236"}, + {"ldrdhar;", "\342\245\247"}, + {"ldrushar;", "\342\245\213"}, + {"ldsh;", "\342\206\262"}, + {"le;", "\342\211\244"}, + {"leftarrow;", "\342\206\220"}, + {"leftarrowtail;", "\342\206\242"}, + {"leftharpoondown;", "\342\206\275"}, + {"leftharpoonup;", "\342\206\274"}, + {"leftleftarrows;", "\342\207\207"}, + {"leftrightarrow;", "\342\206\224"}, + {"leftrightarrows;", "\342\207\206"}, + {"leftrightharpoons;", "\342\207\213"}, + {"leftrightsquigarrow;", "\342\206\255"}, + {"leftthreetimes;", "\342\213\213"}, + {"leg;", "\342\213\232"}, + {"leq;", "\342\211\244"}, + {"leqq;", "\342\211\246"}, + {"leqslant;", "\342\251\275"}, + {"les;", "\342\251\275"}, + {"lescc;", "\342\252\250"}, + {"lesdot;", "\342\251\277"}, + {"lesdoto;", "\342\252\201"}, + {"lesdotor;", "\342\252\203"}, + {"lesg;", "\342\213\232\357\270\200"}, + {"lesges;", "\342\252\223"}, + {"lessapprox;", "\342\252\205"}, + {"lessdot;", "\342\213\226"}, + {"lesseqgtr;", "\342\213\232"}, + {"lesseqqgtr;", "\342\252\213"}, + {"lessgtr;", "\342\211\266"}, + {"lesssim;", "\342\211\262"}, + {"lfisht;", "\342\245\274"}, + {"lfloor;", "\342\214\212"}, + {"lfr;", "\360\235\224\251"}, + {"lg;", "\342\211\266"}, + {"lgE;", "\342\252\221"}, + {"lhard;", "\342\206\275"}, + {"lharu;", "\342\206\274"}, + {"lharul;", "\342\245\252"}, + {"lhblk;", "\342\226\204"}, + {"ljcy;", "\321\231"}, + {"ll;", "\342\211\252"}, + {"llarr;", "\342\207\207"}, + {"llcorner;", "\342\214\236"}, + {"llhard;", "\342\245\253"}, + {"lltri;", "\342\227\272"}, + {"lmidot;", "\305\200"}, + {"lmoust;", "\342\216\260"}, + {"lmoustache;", "\342\216\260"}, + {"lnE;", "\342\211\250"}, + {"lnap;", "\342\252\211"}, + {"lnapprox;", "\342\252\211"}, + {"lne;", "\342\252\207"}, + {"lneq;", "\342\252\207"}, + {"lneqq;", "\342\211\250"}, + {"lnsim;", "\342\213\246"}, + {"loang;", "\342\237\254"}, + {"loarr;", "\342\207\275"}, + {"lobrk;", "\342\237\246"}, + {"longleftarrow;", "\342\237\265"}, + {"longleftrightarrow;", "\342\237\267"}, + {"longmapsto;", "\342\237\274"}, + {"longrightarrow;", "\342\237\266"}, + {"looparrowleft;", "\342\206\253"}, + {"looparrowright;", "\342\206\254"}, + {"lopar;", "\342\246\205"}, + {"lopf;", "\360\235\225\235"}, + {"loplus;", "\342\250\255"}, + {"lotimes;", "\342\250\264"}, + {"lowast;", "\342\210\227"}, + {"lowbar;", "_"}, + {"loz;", "\342\227\212"}, + {"lozenge;", "\342\227\212"}, + {"lozf;", "\342\247\253"}, + {"lpar;", "("}, + {"lparlt;", "\342\246\223"}, + {"lrarr;", "\342\207\206"}, + {"lrcorner;", "\342\214\237"}, + {"lrhar;", "\342\207\213"}, + {"lrhard;", "\342\245\255"}, + {"lrm;", "\342\200\216"}, + {"lrtri;", "\342\212\277"}, + {"lsaquo;", "\342\200\271"}, + {"lscr;", "\360\235\223\201"}, + {"lsh;", "\342\206\260"}, + {"lsim;", "\342\211\262"}, + {"lsime;", "\342\252\215"}, + {"lsimg;", "\342\252\217"}, + {"lsqb;", "["}, + {"lsquo;", "\342\200\230"}, + {"lsquor;", "\342\200\232"}, + {"lstrok;", "\305\202"}, + {"lt", "<"}, + {"lt;", "<"}, + {"ltcc;", "\342\252\246"}, + {"ltcir;", "\342\251\271"}, + {"ltdot;", "\342\213\226"}, + {"lthree;", "\342\213\213"}, + {"ltimes;", "\342\213\211"}, + {"ltlarr;", "\342\245\266"}, + {"ltquest;", "\342\251\273"}, + {"ltrPar;", "\342\246\226"}, + {"ltri;", "\342\227\203"}, + {"ltrie;", "\342\212\264"}, + {"ltrif;", "\342\227\202"}, + {"lurdshar;", "\342\245\212"}, + {"luruhar;", "\342\245\246"}, + {"lvertneqq;", "\342\211\250\357\270\200"}, + {"lvnE;", "\342\211\250\357\270\200"}, + {"mDDot;", "\342\210\272"}, + {"macr", "\302\257"}, + {"macr;", "\302\257"}, + {"male;", "\342\231\202"}, + {"malt;", "\342\234\240"}, + {"maltese;", "\342\234\240"}, + {"map;", "\342\206\246"}, + {"mapsto;", "\342\206\246"}, + {"mapstodown;", "\342\206\247"}, + {"mapstoleft;", "\342\206\244"}, + {"mapstoup;", "\342\206\245"}, + {"marker;", "\342\226\256"}, + {"mcomma;", "\342\250\251"}, + {"mcy;", "\320\274"}, + {"mdash;", "\342\200\224"}, + {"measuredangle;", "\342\210\241"}, + {"mfr;", "\360\235\224\252"}, + {"mho;", "\342\204\247"}, + {"micro", "\302\265"}, + {"micro;", "\302\265"}, + {"mid;", "\342\210\243"}, + {"midast;", "*"}, + {"midcir;", "\342\253\260"}, + {"middot", "\302\267"}, + {"middot;", "\302\267"}, + {"minus;", "\342\210\222"}, + {"minusb;", "\342\212\237"}, + {"minusd;", "\342\210\270"}, + {"minusdu;", "\342\250\252"}, + {"mlcp;", "\342\253\233"}, + {"mldr;", "\342\200\246"}, + {"mnplus;", "\342\210\223"}, + {"models;", "\342\212\247"}, + {"mopf;", "\360\235\225\236"}, + {"mp;", "\342\210\223"}, + {"mscr;", "\360\235\223\202"}, + {"mstpos;", "\342\210\276"}, + {"mu;", "\316\274"}, + {"multimap;", "\342\212\270"}, + {"mumap;", "\342\212\270"}, + {"nGg;", "\342\213\231\314\270"}, + {"nGt;", "\342\211\253\342\203\222"}, + {"nGtv;", "\342\211\253\314\270"}, + {"nLeftarrow;", "\342\207\215"}, + {"nLeftrightarrow;", "\342\207\216"}, + {"nLl;", "\342\213\230\314\270"}, + {"nLt;", "\342\211\252\342\203\222"}, + {"nLtv;", "\342\211\252\314\270"}, + {"nRightarrow;", "\342\207\217"}, + {"nVDash;", "\342\212\257"}, + {"nVdash;", "\342\212\256"}, + {"nabla;", "\342\210\207"}, + {"nacute;", "\305\204"}, + {"nang;", "\342\210\240\342\203\222"}, + {"nap;", "\342\211\211"}, + {"napE;", "\342\251\260\314\270"}, + {"napid;", "\342\211\213\314\270"}, + {"napos;", "\305\211"}, + {"napprox;", "\342\211\211"}, + {"natur;", "\342\231\256"}, + {"natural;", "\342\231\256"}, + {"naturals;", "\342\204\225"}, + {"nbsp", "\302\240"}, + {"nbsp;", "\302\240"}, + {"nbump;", "\342\211\216\314\270"}, + {"nbumpe;", "\342\211\217\314\270"}, + {"ncap;", "\342\251\203"}, + {"ncaron;", "\305\210"}, + {"ncedil;", "\305\206"}, + {"ncong;", "\342\211\207"}, + {"ncongdot;", "\342\251\255\314\270"}, + {"ncup;", "\342\251\202"}, + {"ncy;", "\320\275"}, + {"ndash;", "\342\200\223"}, + {"ne;", "\342\211\240"}, + {"neArr;", "\342\207\227"}, + {"nearhk;", "\342\244\244"}, + {"nearr;", "\342\206\227"}, + {"nearrow;", "\342\206\227"}, + {"nedot;", "\342\211\220\314\270"}, + {"nequiv;", "\342\211\242"}, + {"nesear;", "\342\244\250"}, + {"nesim;", "\342\211\202\314\270"}, + {"nexist;", "\342\210\204"}, + {"nexists;", "\342\210\204"}, + {"nfr;", "\360\235\224\253"}, + {"ngE;", "\342\211\247\314\270"}, + {"nge;", "\342\211\261"}, + {"ngeq;", "\342\211\261"}, + {"ngeqq;", "\342\211\247\314\270"}, + {"ngeqslant;", "\342\251\276\314\270"}, + {"nges;", "\342\251\276\314\270"}, + {"ngsim;", "\342\211\265"}, + {"ngt;", "\342\211\257"}, + {"ngtr;", "\342\211\257"}, + {"nhArr;", "\342\207\216"}, + {"nharr;", "\342\206\256"}, + {"nhpar;", "\342\253\262"}, + {"ni;", "\342\210\213"}, + {"nis;", "\342\213\274"}, + {"nisd;", "\342\213\272"}, + {"niv;", "\342\210\213"}, + {"njcy;", "\321\232"}, + {"nlArr;", "\342\207\215"}, + {"nlE;", "\342\211\246\314\270"}, + {"nlarr;", "\342\206\232"}, + {"nldr;", "\342\200\245"}, + {"nle;", "\342\211\260"}, + {"nleftarrow;", "\342\206\232"}, + {"nleftrightarrow;", "\342\206\256"}, + {"nleq;", "\342\211\260"}, + {"nleqq;", "\342\211\246\314\270"}, + {"nleqslant;", "\342\251\275\314\270"}, + {"nles;", "\342\251\275\314\270"}, + {"nless;", "\342\211\256"}, + {"nlsim;", "\342\211\264"}, + {"nlt;", "\342\211\256"}, + {"nltri;", "\342\213\252"}, + {"nltrie;", "\342\213\254"}, + {"nmid;", "\342\210\244"}, + {"nopf;", "\360\235\225\237"}, + {"not", "\302\254"}, + {"not;", "\302\254"}, + {"notin;", "\342\210\211"}, + {"notinE;", "\342\213\271\314\270"}, + {"notindot;", "\342\213\265\314\270"}, + {"notinva;", "\342\210\211"}, + {"notinvb;", "\342\213\267"}, + {"notinvc;", "\342\213\266"}, + {"notni;", "\342\210\214"}, + {"notniva;", "\342\210\214"}, + {"notnivb;", "\342\213\276"}, + {"notnivc;", "\342\213\275"}, + {"npar;", "\342\210\246"}, + {"nparallel;", "\342\210\246"}, + {"nparsl;", "\342\253\275\342\203\245"}, + {"npart;", "\342\210\202\314\270"}, + {"npolint;", "\342\250\224"}, + {"npr;", "\342\212\200"}, + {"nprcue;", "\342\213\240"}, + {"npre;", "\342\252\257\314\270"}, + {"nprec;", "\342\212\200"}, + {"npreceq;", "\342\252\257\314\270"}, + {"nrArr;", "\342\207\217"}, + {"nrarr;", "\342\206\233"}, + {"nrarrc;", "\342\244\263\314\270"}, + {"nrarrw;", "\342\206\235\314\270"}, + {"nrightarrow;", "\342\206\233"}, + {"nrtri;", "\342\213\253"}, + {"nrtrie;", "\342\213\255"}, + {"nsc;", "\342\212\201"}, + {"nsccue;", "\342\213\241"}, + {"nsce;", "\342\252\260\314\270"}, + {"nscr;", "\360\235\223\203"}, + {"nshortmid;", "\342\210\244"}, + {"nshortparallel;", "\342\210\246"}, + {"nsim;", "\342\211\201"}, + {"nsime;", "\342\211\204"}, + {"nsimeq;", "\342\211\204"}, + {"nsmid;", "\342\210\244"}, + {"nspar;", "\342\210\246"}, + {"nsqsube;", "\342\213\242"}, + {"nsqsupe;", "\342\213\243"}, + {"nsub;", "\342\212\204"}, + {"nsubE;", "\342\253\205\314\270"}, + {"nsube;", "\342\212\210"}, + {"nsubset;", "\342\212\202\342\203\222"}, + {"nsubseteq;", "\342\212\210"}, + {"nsubseteqq;", "\342\253\205\314\270"}, + {"nsucc;", "\342\212\201"}, + {"nsucceq;", "\342\252\260\314\270"}, + {"nsup;", "\342\212\205"}, + {"nsupE;", "\342\253\206\314\270"}, + {"nsupe;", "\342\212\211"}, + {"nsupset;", "\342\212\203\342\203\222"}, + {"nsupseteq;", "\342\212\211"}, + {"nsupseteqq;", "\342\253\206\314\270"}, + {"ntgl;", "\342\211\271"}, + {"ntilde", "\303\261"}, + {"ntilde;", "\303\261"}, + {"ntlg;", "\342\211\270"}, + {"ntriangleleft;", "\342\213\252"}, + {"ntrianglelefteq;", "\342\213\254"}, + {"ntriangleright;", "\342\213\253"}, + {"ntrianglerighteq;", "\342\213\255"}, + {"nu;", "\316\275"}, + {"num;", "#"}, + {"numero;", "\342\204\226"}, + {"numsp;", "\342\200\207"}, + {"nvDash;", "\342\212\255"}, + {"nvHarr;", "\342\244\204"}, + {"nvap;", "\342\211\215\342\203\222"}, + {"nvdash;", "\342\212\254"}, + {"nvge;", "\342\211\245\342\203\222"}, + {"nvgt;", ">\342\203\222"}, + {"nvinfin;", "\342\247\236"}, + {"nvlArr;", "\342\244\202"}, + {"nvle;", "\342\211\244\342\203\222"}, + {"nvlt;", "<\342\203\222"}, + {"nvltrie;", "\342\212\264\342\203\222"}, + {"nvrArr;", "\342\244\203"}, + {"nvrtrie;", "\342\212\265\342\203\222"}, + {"nvsim;", "\342\210\274\342\203\222"}, + {"nwArr;", "\342\207\226"}, + {"nwarhk;", "\342\244\243"}, + {"nwarr;", "\342\206\226"}, + {"nwarrow;", "\342\206\226"}, + {"nwnear;", "\342\244\247"}, + {"oS;", "\342\223\210"}, + {"oacute", "\303\263"}, + {"oacute;", "\303\263"}, + {"oast;", "\342\212\233"}, + {"ocir;", "\342\212\232"}, + {"ocirc", "\303\264"}, + {"ocirc;", "\303\264"}, + {"ocy;", "\320\276"}, + {"odash;", "\342\212\235"}, + {"odblac;", "\305\221"}, + {"odiv;", "\342\250\270"}, + {"odot;", "\342\212\231"}, + {"odsold;", "\342\246\274"}, + {"oelig;", "\305\223"}, + {"ofcir;", "\342\246\277"}, + {"ofr;", "\360\235\224\254"}, + {"ogon;", "\313\233"}, + {"ograve", "\303\262"}, + {"ograve;", "\303\262"}, + {"ogt;", "\342\247\201"}, + {"ohbar;", "\342\246\265"}, + {"ohm;", "\316\251"}, + {"oint;", "\342\210\256"}, + {"olarr;", "\342\206\272"}, + {"olcir;", "\342\246\276"}, + {"olcross;", "\342\246\273"}, + {"oline;", "\342\200\276"}, + {"olt;", "\342\247\200"}, + {"omacr;", "\305\215"}, + {"omega;", "\317\211"}, + {"omicron;", "\316\277"}, + {"omid;", "\342\246\266"}, + {"ominus;", "\342\212\226"}, + {"oopf;", "\360\235\225\240"}, + {"opar;", "\342\246\267"}, + {"operp;", "\342\246\271"}, + {"oplus;", "\342\212\225"}, + {"or;", "\342\210\250"}, + {"orarr;", "\342\206\273"}, + {"ord;", "\342\251\235"}, + {"order;", "\342\204\264"}, + {"orderof;", "\342\204\264"}, + {"ordf", "\302\252"}, + {"ordf;", "\302\252"}, + {"ordm", "\302\272"}, + {"ordm;", "\302\272"}, + {"origof;", "\342\212\266"}, + {"oror;", "\342\251\226"}, + {"orslope;", "\342\251\227"}, + {"orv;", "\342\251\233"}, + {"oscr;", "\342\204\264"}, + {"oslash", "\303\270"}, + {"oslash;", "\303\270"}, + {"osol;", "\342\212\230"}, + {"otilde", "\303\265"}, + {"otilde;", "\303\265"}, + {"otimes;", "\342\212\227"}, + {"otimesas;", "\342\250\266"}, + {"ouml", "\303\266"}, + {"ouml;", "\303\266"}, + {"ovbar;", "\342\214\275"}, + {"par;", "\342\210\245"}, + {"para", "\302\266"}, + {"para;", "\302\266"}, + {"parallel;", "\342\210\245"}, + {"parsim;", "\342\253\263"}, + {"parsl;", "\342\253\275"}, + {"part;", "\342\210\202"}, + {"pcy;", "\320\277"}, + {"percnt;", "%"}, + {"period;", "."}, + {"permil;", "\342\200\260"}, + {"perp;", "\342\212\245"}, + {"pertenk;", "\342\200\261"}, + {"pfr;", "\360\235\224\255"}, + {"phi;", "\317\206"}, + {"phiv;", "\317\225"}, + {"phmmat;", "\342\204\263"}, + {"phone;", "\342\230\216"}, + {"pi;", "\317\200"}, + {"pitchfork;", "\342\213\224"}, + {"piv;", "\317\226"}, + {"planck;", "\342\204\217"}, + {"planckh;", "\342\204\216"}, + {"plankv;", "\342\204\217"}, + {"plus;", "+"}, + {"plusacir;", "\342\250\243"}, + {"plusb;", "\342\212\236"}, + {"pluscir;", "\342\250\242"}, + {"plusdo;", "\342\210\224"}, + {"plusdu;", "\342\250\245"}, + {"pluse;", "\342\251\262"}, + {"plusmn", "\302\261"}, + {"plusmn;", "\302\261"}, + {"plussim;", "\342\250\246"}, + {"plustwo;", "\342\250\247"}, + {"pm;", "\302\261"}, + {"pointint;", "\342\250\225"}, + {"popf;", "\360\235\225\241"}, + {"pound", "\302\243"}, + {"pound;", "\302\243"}, + {"pr;", "\342\211\272"}, + {"prE;", "\342\252\263"}, + {"prap;", "\342\252\267"}, + {"prcue;", "\342\211\274"}, + {"pre;", "\342\252\257"}, + {"prec;", "\342\211\272"}, + {"precapprox;", "\342\252\267"}, + {"preccurlyeq;", "\342\211\274"}, + {"preceq;", "\342\252\257"}, + {"precnapprox;", "\342\252\271"}, + {"precneqq;", "\342\252\265"}, + {"precnsim;", "\342\213\250"}, + {"precsim;", "\342\211\276"}, + {"prime;", "\342\200\262"}, + {"primes;", "\342\204\231"}, + {"prnE;", "\342\252\265"}, + {"prnap;", "\342\252\271"}, + {"prnsim;", "\342\213\250"}, + {"prod;", "\342\210\217"}, + {"profalar;", "\342\214\256"}, + {"profline;", "\342\214\222"}, + {"profsurf;", "\342\214\223"}, + {"prop;", "\342\210\235"}, + {"propto;", "\342\210\235"}, + {"prsim;", "\342\211\276"}, + {"prurel;", "\342\212\260"}, + {"pscr;", "\360\235\223\205"}, + {"psi;", "\317\210"}, + {"puncsp;", "\342\200\210"}, + {"qfr;", "\360\235\224\256"}, + {"qint;", "\342\250\214"}, + {"qopf;", "\360\235\225\242"}, + {"qprime;", "\342\201\227"}, + {"qscr;", "\360\235\223\206"}, + {"quaternions;", "\342\204\215"}, + {"quatint;", "\342\250\226"}, + {"quest;", "?"}, + {"questeq;", "\342\211\237"}, + {"quot", "\""}, + {"quot;", "\""}, + {"rAarr;", "\342\207\233"}, + {"rArr;", "\342\207\222"}, + {"rAtail;", "\342\244\234"}, + {"rBarr;", "\342\244\217"}, + {"rHar;", "\342\245\244"}, + {"race;", "\342\210\275\314\261"}, + {"racute;", "\305\225"}, + {"radic;", "\342\210\232"}, + {"raemptyv;", "\342\246\263"}, + {"rang;", "\342\237\251"}, + {"rangd;", "\342\246\222"}, + {"range;", "\342\246\245"}, + {"rangle;", "\342\237\251"}, + {"raquo", "\302\273"}, + {"raquo;", "\302\273"}, + {"rarr;", "\342\206\222"}, + {"rarrap;", "\342\245\265"}, + {"rarrb;", "\342\207\245"}, + {"rarrbfs;", "\342\244\240"}, + {"rarrc;", "\342\244\263"}, + {"rarrfs;", "\342\244\236"}, + {"rarrhk;", "\342\206\252"}, + {"rarrlp;", "\342\206\254"}, + {"rarrpl;", "\342\245\205"}, + {"rarrsim;", "\342\245\264"}, + {"rarrtl;", "\342\206\243"}, + {"rarrw;", "\342\206\235"}, + {"ratail;", "\342\244\232"}, + {"ratio;", "\342\210\266"}, + {"rationals;", "\342\204\232"}, + {"rbarr;", "\342\244\215"}, + {"rbbrk;", "\342\235\263"}, + {"rbrace;", "}"}, + {"rbrack;", "]"}, + {"rbrke;", "\342\246\214"}, + {"rbrksld;", "\342\246\216"}, + {"rbrkslu;", "\342\246\220"}, + {"rcaron;", "\305\231"}, + {"rcedil;", "\305\227"}, + {"rceil;", "\342\214\211"}, + {"rcub;", "}"}, + {"rcy;", "\321\200"}, + {"rdca;", "\342\244\267"}, + {"rdldhar;", "\342\245\251"}, + {"rdquo;", "\342\200\235"}, + {"rdquor;", "\342\200\235"}, + {"rdsh;", "\342\206\263"}, + {"real;", "\342\204\234"}, + {"realine;", "\342\204\233"}, + {"realpart;", "\342\204\234"}, + {"reals;", "\342\204\235"}, + {"rect;", "\342\226\255"}, + {"reg", "\302\256"}, + {"reg;", "\302\256"}, + {"rfisht;", "\342\245\275"}, + {"rfloor;", "\342\214\213"}, + {"rfr;", "\360\235\224\257"}, + {"rhard;", "\342\207\201"}, + {"rharu;", "\342\207\200"}, + {"rharul;", "\342\245\254"}, + {"rho;", "\317\201"}, + {"rhov;", "\317\261"}, + {"rightarrow;", "\342\206\222"}, + {"rightarrowtail;", "\342\206\243"}, + {"rightharpoondown;", "\342\207\201"}, + {"rightharpoonup;", "\342\207\200"}, + {"rightleftarrows;", "\342\207\204"}, + {"rightleftharpoons;", "\342\207\214"}, + {"rightrightarrows;", "\342\207\211"}, + {"rightsquigarrow;", "\342\206\235"}, + {"rightthreetimes;", "\342\213\214"}, + {"ring;", "\313\232"}, + {"risingdotseq;", "\342\211\223"}, + {"rlarr;", "\342\207\204"}, + {"rlhar;", "\342\207\214"}, + {"rlm;", "\342\200\217"}, + {"rmoust;", "\342\216\261"}, + {"rmoustache;", "\342\216\261"}, + {"rnmid;", "\342\253\256"}, + {"roang;", "\342\237\255"}, + {"roarr;", "\342\207\276"}, + {"robrk;", "\342\237\247"}, + {"ropar;", "\342\246\206"}, + {"ropf;", "\360\235\225\243"}, + {"roplus;", "\342\250\256"}, + {"rotimes;", "\342\250\265"}, + {"rpar;", ")"}, + {"rpargt;", "\342\246\224"}, + {"rppolint;", "\342\250\222"}, + {"rrarr;", "\342\207\211"}, + {"rsaquo;", "\342\200\272"}, + {"rscr;", "\360\235\223\207"}, + {"rsh;", "\342\206\261"}, + {"rsqb;", "]"}, + {"rsquo;", "\342\200\231"}, + {"rsquor;", "\342\200\231"}, + {"rthree;", "\342\213\214"}, + {"rtimes;", "\342\213\212"}, + {"rtri;", "\342\226\271"}, + {"rtrie;", "\342\212\265"}, + {"rtrif;", "\342\226\270"}, + {"rtriltri;", "\342\247\216"}, + {"ruluhar;", "\342\245\250"}, + {"rx;", "\342\204\236"}, + {"sacute;", "\305\233"}, + {"sbquo;", "\342\200\232"}, + {"sc;", "\342\211\273"}, + {"scE;", "\342\252\264"}, + {"scap;", "\342\252\270"}, + {"scaron;", "\305\241"}, + {"sccue;", "\342\211\275"}, + {"sce;", "\342\252\260"}, + {"scedil;", "\305\237"}, + {"scirc;", "\305\235"}, + {"scnE;", "\342\252\266"}, + {"scnap;", "\342\252\272"}, + {"scnsim;", "\342\213\251"}, + {"scpolint;", "\342\250\223"}, + {"scsim;", "\342\211\277"}, + {"scy;", "\321\201"}, + {"sdot;", "\342\213\205"}, + {"sdotb;", "\342\212\241"}, + {"sdote;", "\342\251\246"}, + {"seArr;", "\342\207\230"}, + {"searhk;", "\342\244\245"}, + {"searr;", "\342\206\230"}, + {"searrow;", "\342\206\230"}, + {"sect", "\302\247"}, + {"sect;", "\302\247"}, + {"semi;", ";"}, + {"seswar;", "\342\244\251"}, + {"setminus;", "\342\210\226"}, + {"setmn;", "\342\210\226"}, + {"sext;", "\342\234\266"}, + {"sfr;", "\360\235\224\260"}, + {"sfrown;", "\342\214\242"}, + {"sharp;", "\342\231\257"}, + {"shchcy;", "\321\211"}, + {"shcy;", "\321\210"}, + {"shortmid;", "\342\210\243"}, + {"shortparallel;", "\342\210\245"}, + {"shy", "\302\255"}, + {"shy;", "\302\255"}, + {"sigma;", "\317\203"}, + {"sigmaf;", "\317\202"}, + {"sigmav;", "\317\202"}, + {"sim;", "\342\210\274"}, + {"simdot;", "\342\251\252"}, + {"sime;", "\342\211\203"}, + {"simeq;", "\342\211\203"}, + {"simg;", "\342\252\236"}, + {"simgE;", "\342\252\240"}, + {"siml;", "\342\252\235"}, + {"simlE;", "\342\252\237"}, + {"simne;", "\342\211\206"}, + {"simplus;", "\342\250\244"}, + {"simrarr;", "\342\245\262"}, + {"slarr;", "\342\206\220"}, + {"smallsetminus;", "\342\210\226"}, + {"smashp;", "\342\250\263"}, + {"smeparsl;", "\342\247\244"}, + {"smid;", "\342\210\243"}, + {"smile;", "\342\214\243"}, + {"smt;", "\342\252\252"}, + {"smte;", "\342\252\254"}, + {"smtes;", "\342\252\254\357\270\200"}, + {"softcy;", "\321\214"}, + {"sol;", "/"}, + {"solb;", "\342\247\204"}, + {"solbar;", "\342\214\277"}, + {"sopf;", "\360\235\225\244"}, + {"spades;", "\342\231\240"}, + {"spadesuit;", "\342\231\240"}, + {"spar;", "\342\210\245"}, + {"sqcap;", "\342\212\223"}, + {"sqcaps;", "\342\212\223\357\270\200"}, + {"sqcup;", "\342\212\224"}, + {"sqcups;", "\342\212\224\357\270\200"}, + {"sqsub;", "\342\212\217"}, + {"sqsube;", "\342\212\221"}, + {"sqsubset;", "\342\212\217"}, + {"sqsubseteq;", "\342\212\221"}, + {"sqsup;", "\342\212\220"}, + {"sqsupe;", "\342\212\222"}, + {"sqsupset;", "\342\212\220"}, + {"sqsupseteq;", "\342\212\222"}, + {"squ;", "\342\226\241"}, + {"square;", "\342\226\241"}, + {"squarf;", "\342\226\252"}, + {"squf;", "\342\226\252"}, + {"srarr;", "\342\206\222"}, + {"sscr;", "\360\235\223\210"}, + {"ssetmn;", "\342\210\226"}, + {"ssmile;", "\342\214\243"}, + {"sstarf;", "\342\213\206"}, + {"star;", "\342\230\206"}, + {"starf;", "\342\230\205"}, + {"straightepsilon;", "\317\265"}, + {"straightphi;", "\317\225"}, + {"strns;", "\302\257"}, + {"sub;", "\342\212\202"}, + {"subE;", "\342\253\205"}, + {"subdot;", "\342\252\275"}, + {"sube;", "\342\212\206"}, + {"subedot;", "\342\253\203"}, + {"submult;", "\342\253\201"}, + {"subnE;", "\342\253\213"}, + {"subne;", "\342\212\212"}, + {"subplus;", "\342\252\277"}, + {"subrarr;", "\342\245\271"}, + {"subset;", "\342\212\202"}, + {"subseteq;", "\342\212\206"}, + {"subseteqq;", "\342\253\205"}, + {"subsetneq;", "\342\212\212"}, + {"subsetneqq;", "\342\253\213"}, + {"subsim;", "\342\253\207"}, + {"subsub;", "\342\253\225"}, + {"subsup;", "\342\253\223"}, + {"succ;", "\342\211\273"}, + {"succapprox;", "\342\252\270"}, + {"succcurlyeq;", "\342\211\275"}, + {"succeq;", "\342\252\260"}, + {"succnapprox;", "\342\252\272"}, + {"succneqq;", "\342\252\266"}, + {"succnsim;", "\342\213\251"}, + {"succsim;", "\342\211\277"}, + {"sum;", "\342\210\221"}, + {"sung;", "\342\231\252"}, + {"sup1", "\302\271"}, + {"sup1;", "\302\271"}, + {"sup2", "\302\262"}, + {"sup2;", "\302\262"}, + {"sup3", "\302\263"}, + {"sup3;", "\302\263"}, + {"sup;", "\342\212\203"}, + {"supE;", "\342\253\206"}, + {"supdot;", "\342\252\276"}, + {"supdsub;", "\342\253\230"}, + {"supe;", "\342\212\207"}, + {"supedot;", "\342\253\204"}, + {"suphsol;", "\342\237\211"}, + {"suphsub;", "\342\253\227"}, + {"suplarr;", "\342\245\273"}, + {"supmult;", "\342\253\202"}, + {"supnE;", "\342\253\214"}, + {"supne;", "\342\212\213"}, + {"supplus;", "\342\253\200"}, + {"supset;", "\342\212\203"}, + {"supseteq;", "\342\212\207"}, + {"supseteqq;", "\342\253\206"}, + {"supsetneq;", "\342\212\213"}, + {"supsetneqq;", "\342\253\214"}, + {"supsim;", "\342\253\210"}, + {"supsub;", "\342\253\224"}, + {"supsup;", "\342\253\226"}, + {"swArr;", "\342\207\231"}, + {"swarhk;", "\342\244\246"}, + {"swarr;", "\342\206\231"}, + {"swarrow;", "\342\206\231"}, + {"swnwar;", "\342\244\252"}, + {"szlig", "\303\237"}, + {"szlig;", "\303\237"}, + {"target;", "\342\214\226"}, + {"tau;", "\317\204"}, + {"tbrk;", "\342\216\264"}, + {"tcaron;", "\305\245"}, + {"tcedil;", "\305\243"}, + {"tcy;", "\321\202"}, + {"tdot;", "\342\203\233"}, + {"telrec;", "\342\214\225"}, + {"tfr;", "\360\235\224\261"}, + {"there4;", "\342\210\264"}, + {"therefore;", "\342\210\264"}, + {"theta;", "\316\270"}, + {"thetasym;", "\317\221"}, + {"thetav;", "\317\221"}, + {"thickapprox;", "\342\211\210"}, + {"thicksim;", "\342\210\274"}, + {"thinsp;", "\342\200\211"}, + {"thkap;", "\342\211\210"}, + {"thksim;", "\342\210\274"}, + {"thorn", "\303\276"}, + {"thorn;", "\303\276"}, + {"tilde;", "\313\234"}, + {"times", "\303\227"}, + {"times;", "\303\227"}, + {"timesb;", "\342\212\240"}, + {"timesbar;", "\342\250\261"}, + {"timesd;", "\342\250\260"}, + {"tint;", "\342\210\255"}, + {"toea;", "\342\244\250"}, + {"top;", "\342\212\244"}, + {"topbot;", "\342\214\266"}, + {"topcir;", "\342\253\261"}, + {"topf;", "\360\235\225\245"}, + {"topfork;", "\342\253\232"}, + {"tosa;", "\342\244\251"}, + {"tprime;", "\342\200\264"}, + {"trade;", "\342\204\242"}, + {"triangle;", "\342\226\265"}, + {"triangledown;", "\342\226\277"}, + {"triangleleft;", "\342\227\203"}, + {"trianglelefteq;", "\342\212\264"}, + {"triangleq;", "\342\211\234"}, + {"triangleright;", "\342\226\271"}, + {"trianglerighteq;", "\342\212\265"}, + {"tridot;", "\342\227\254"}, + {"trie;", "\342\211\234"}, + {"triminus;", "\342\250\272"}, + {"triplus;", "\342\250\271"}, + {"trisb;", "\342\247\215"}, + {"tritime;", "\342\250\273"}, + {"trpezium;", "\342\217\242"}, + {"tscr;", "\360\235\223\211"}, + {"tscy;", "\321\206"}, + {"tshcy;", "\321\233"}, + {"tstrok;", "\305\247"}, + {"twixt;", "\342\211\254"}, + {"twoheadleftarrow;", "\342\206\236"}, + {"twoheadrightarrow;", "\342\206\240"}, + {"uArr;", "\342\207\221"}, + {"uHar;", "\342\245\243"}, + {"uacute", "\303\272"}, + {"uacute;", "\303\272"}, + {"uarr;", "\342\206\221"}, + {"ubrcy;", "\321\236"}, + {"ubreve;", "\305\255"}, + {"ucirc", "\303\273"}, + {"ucirc;", "\303\273"}, + {"ucy;", "\321\203"}, + {"udarr;", "\342\207\205"}, + {"udblac;", "\305\261"}, + {"udhar;", "\342\245\256"}, + {"ufisht;", "\342\245\276"}, + {"ufr;", "\360\235\224\262"}, + {"ugrave", "\303\271"}, + {"ugrave;", "\303\271"}, + {"uharl;", "\342\206\277"}, + {"uharr;", "\342\206\276"}, + {"uhblk;", "\342\226\200"}, + {"ulcorn;", "\342\214\234"}, + {"ulcorner;", "\342\214\234"}, + {"ulcrop;", "\342\214\217"}, + {"ultri;", "\342\227\270"}, + {"umacr;", "\305\253"}, + {"uml", "\302\250"}, + {"uml;", "\302\250"}, + {"uogon;", "\305\263"}, + {"uopf;", "\360\235\225\246"}, + {"uparrow;", "\342\206\221"}, + {"updownarrow;", "\342\206\225"}, + {"upharpoonleft;", "\342\206\277"}, + {"upharpoonright;", "\342\206\276"}, + {"uplus;", "\342\212\216"}, + {"upsi;", "\317\205"}, + {"upsih;", "\317\222"}, + {"upsilon;", "\317\205"}, + {"upuparrows;", "\342\207\210"}, + {"urcorn;", "\342\214\235"}, + {"urcorner;", "\342\214\235"}, + {"urcrop;", "\342\214\216"}, + {"uring;", "\305\257"}, + {"urtri;", "\342\227\271"}, + {"uscr;", "\360\235\223\212"}, + {"utdot;", "\342\213\260"}, + {"utilde;", "\305\251"}, + {"utri;", "\342\226\265"}, + {"utrif;", "\342\226\264"}, + {"uuarr;", "\342\207\210"}, + {"uuml", "\303\274"}, + {"uuml;", "\303\274"}, + {"uwangle;", "\342\246\247"}, + {"vArr;", "\342\207\225"}, + {"vBar;", "\342\253\250"}, + {"vBarv;", "\342\253\251"}, + {"vDash;", "\342\212\250"}, + {"vangrt;", "\342\246\234"}, + {"varepsilon;", "\317\265"}, + {"varkappa;", "\317\260"}, + {"varnothing;", "\342\210\205"}, + {"varphi;", "\317\225"}, + {"varpi;", "\317\226"}, + {"varpropto;", "\342\210\235"}, + {"varr;", "\342\206\225"}, + {"varrho;", "\317\261"}, + {"varsigma;", "\317\202"}, + {"varsubsetneq;", "\342\212\212\357\270\200"}, + {"varsubsetneqq;", "\342\253\213\357\270\200"}, + {"varsupsetneq;", "\342\212\213\357\270\200"}, + {"varsupsetneqq;", "\342\253\214\357\270\200"}, + {"vartheta;", "\317\221"}, + {"vartriangleleft;", "\342\212\262"}, + {"vartriangleright;", "\342\212\263"}, + {"vcy;", "\320\262"}, + {"vdash;", "\342\212\242"}, + {"vee;", "\342\210\250"}, + {"veebar;", "\342\212\273"}, + {"veeeq;", "\342\211\232"}, + {"vellip;", "\342\213\256"}, + {"verbar;", "|"}, + {"vert;", "|"}, + {"vfr;", "\360\235\224\263"}, + {"vltri;", "\342\212\262"}, + {"vnsub;", "\342\212\202\342\203\222"}, + {"vnsup;", "\342\212\203\342\203\222"}, + {"vopf;", "\360\235\225\247"}, + {"vprop;", "\342\210\235"}, + {"vrtri;", "\342\212\263"}, + {"vscr;", "\360\235\223\213"}, + {"vsubnE;", "\342\253\213\357\270\200"}, + {"vsubne;", "\342\212\212\357\270\200"}, + {"vsupnE;", "\342\253\214\357\270\200"}, + {"vsupne;", "\342\212\213\357\270\200"}, + {"vzigzag;", "\342\246\232"}, + {"wcirc;", "\305\265"}, + {"wedbar;", "\342\251\237"}, + {"wedge;", "\342\210\247"}, + {"wedgeq;", "\342\211\231"}, + {"weierp;", "\342\204\230"}, + {"wfr;", "\360\235\224\264"}, + {"wopf;", "\360\235\225\250"}, + {"wp;", "\342\204\230"}, + {"wr;", "\342\211\200"}, + {"wreath;", "\342\211\200"}, + {"wscr;", "\360\235\223\214"}, + {"xcap;", "\342\213\202"}, + {"xcirc;", "\342\227\257"}, + {"xcup;", "\342\213\203"}, + {"xdtri;", "\342\226\275"}, + {"xfr;", "\360\235\224\265"}, + {"xhArr;", "\342\237\272"}, + {"xharr;", "\342\237\267"}, + {"xi;", "\316\276"}, + {"xlArr;", "\342\237\270"}, + {"xlarr;", "\342\237\265"}, + {"xmap;", "\342\237\274"}, + {"xnis;", "\342\213\273"}, + {"xodot;", "\342\250\200"}, + {"xopf;", "\360\235\225\251"}, + {"xoplus;", "\342\250\201"}, + {"xotime;", "\342\250\202"}, + {"xrArr;", "\342\237\271"}, + {"xrarr;", "\342\237\266"}, + {"xscr;", "\360\235\223\215"}, + {"xsqcup;", "\342\250\206"}, + {"xuplus;", "\342\250\204"}, + {"xutri;", "\342\226\263"}, + {"xvee;", "\342\213\201"}, + {"xwedge;", "\342\213\200"}, + {"yacute", "\303\275"}, + {"yacute;", "\303\275"}, + {"yacy;", "\321\217"}, + {"ycirc;", "\305\267"}, + {"ycy;", "\321\213"}, + {"yen", "\302\245"}, + {"yen;", "\302\245"}, + {"yfr;", "\360\235\224\266"}, + {"yicy;", "\321\227"}, + {"yopf;", "\360\235\225\252"}, + {"yscr;", "\360\235\223\216"}, + {"yucy;", "\321\216"}, + {"yuml", "\303\277"}, + {"yuml;", "\303\277"}, + {"zacute;", "\305\272"}, + {"zcaron;", "\305\276"}, + {"zcy;", "\320\267"}, + {"zdot;", "\305\274"}, + {"zeetrf;", "\342\204\250"}, + {"zeta;", "\316\266"}, + {"zfr;", "\360\235\224\267"}, + {"zhcy;", "\320\266"}, + {"zigrarr;", "\342\207\235"}, + {"zopf;", "\360\235\225\253"}, + {"zscr;", "\360\235\223\217"}, + {"zwj;", "\342\200\215"}, + {"zwnj;", "\342\200\214"}, + }; + return v; +} +const std::vector>& HtmlInvalidCharrefs() { + static const std::vector> v = { + {0, "\357\277\275"}, + {13, "\015"}, + {128, "\342\202\254"}, + {129, "\302\201"}, + {130, "\342\200\232"}, + {131, "\306\222"}, + {132, "\342\200\236"}, + {133, "\342\200\246"}, + {134, "\342\200\240"}, + {135, "\342\200\241"}, + {136, "\313\206"}, + {137, "\342\200\260"}, + {138, "\305\240"}, + {139, "\342\200\271"}, + {140, "\305\222"}, + {141, "\302\215"}, + {142, "\305\275"}, + {143, "\302\217"}, + {144, "\302\220"}, + {145, "\342\200\230"}, + {146, "\342\200\231"}, + {147, "\342\200\234"}, + {148, "\342\200\235"}, + {149, "\342\200\242"}, + {150, "\342\200\223"}, + {151, "\342\200\224"}, + {152, "\313\234"}, + {153, "\342\204\242"}, + {154, "\305\241"}, + {155, "\342\200\272"}, + {156, "\305\223"}, + {157, "\302\235"}, + {158, "\305\276"}, + {159, "\305\270"}, + }; + return v; +} +const std::vector& HtmlInvalidCodepoints() { + static const std::vector v = {1, 2, 3, 4, 5, 6, 7, 8, 11, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 64976, 64977, 64978, 64979, 64980, 64981, 64982, 64983, 64984, 64985, 64986, 64987, 64988, 64989, 64990, 64991, 64992, 64993, 64994, 64995, 64996, 64997, 64998, 64999, 65000, 65001, 65002, 65003, 65004, 65005, 65006, 65007, 65534, 65535, 131070, 131071, 196606, 196607, 262142, 262143, 327678, 327679, 393214, 393215, 458750, 458751, 524286, 524287, 589822, 589823, 655358, 655359, 720894, 720895, 786430, 786431, 851966, 851967, 917502, 917503, 983038, 983039, 1048574, 1048575, 1114110, 1114111}; + return v; +} +} // namespace detz::agent::data diff --git a/taicpp/src/agent/exec.cpp b/taicpp/src/agent/exec.cpp deleted file mode 100644 index 9a8c863..0000000 --- a/taicpp/src/agent/exec.cpp +++ /dev/null @@ -1,142 +0,0 @@ -#include "exec.h" -#include "tools.h" -#include "shell/session.h" -#include "shell/sandbox.h" -#include "store/store.h" -#include "util/http.h" -#include "util/text.h" -#include "util/ansi.h" -#include "config/config.h" -#include -#include -#include -#include -#include -namespace fs = std::filesystem; -namespace detz::agent { -static ToolResult ok(const std::string& s){ return {true, s}; } -static ToolResult err(const std::string& s){ return {false, s}; } -static std::string arg(const std::map& a, const std::string& k, const std::string& d=""){ auto it=a.find(k); return it==a.end()?d:it->second; } -int RegisterAll(ToolRegistry& reg, const std::string& workdir){ - int n=0; - auto add=[&](const std::string& name,const std::string& desc,ToolRegistry::Fn fn){ reg.add({name,desc},std::move(fn)); ++n; }; - add("shell","Run a shell command",[workdir](auto& a)->ToolResult{ - detz::shell::ShellSession sh; auto e=sh.exec(arg(a,"command"),120000); - return {e.code==0, e.out}; }); - add("read_file","Read a text file",[](auto& a)->ToolResult{ - std::ifstream f(arg(a,"path")); if(!f) return err("cannot open "+arg(a,"path")); - std::ostringstream ss; ss<6000) s=s.substr(0,6000)+"...[truncated]"; - return ok(s); }); - add("write_file","Write content to a file",[](auto& a)->ToolResult{ - fs::path p(arg(a,"path")); std::error_code ec; fs::create_directories(p.parent_path(),ec); - std::ofstream f(p); if(!f) return err("cannot write "+p.string()); - f<ToolResult{ - std::ifstream f(arg(a,"path")); if(!f) return err("cannot open"); - std::ostringstream ss; ss<ToolResult{ - auto r=detz::HttpGet("https://html.duckduckgo.com/html/?q="+arg(a,"query")); - bool ok2=r.status>=200&&r.status<300; return {ok2, ok2?r.body.substr(0,4000):("http "+std::to_string(r.status))}; }); - add("describe_image","Describe an image",[](auto& a)->ToolResult{ return ok("[image: "+arg(a,"source")+"] vision unavailable in offline build"); }); - add("web_fetch","Fetch a URL",[](auto& a)->ToolResult{ - std::string m=arg(a,"method","GET"); std::string u=arg(a,"url"); - detz::HttpResponse r = (m=="POST")?detz::HttpPost(u,arg(a,"body","")):detz::HttpGet(u); - bool ok3=r.status>=200&&r.status<300; return {ok3, ok3?r.body:("http "+std::to_string(r.status))}; }); - add("speak","TTS",[](auto& a)->ToolResult{ return ok("[speak: "+arg(a,"text").substr(0,80)+"]"); }); - add("listen","STT",[](auto&){ return err("no microphone in headless build"); }); - add("remember","Update system message",[](auto& a)->ToolResult{ - std::string home=getenv("HOME")?getenv("HOME"):"/tmp"; - std::ofstream f(home+"/.tai_memory.txt",std::ios::app); f<ToolResult{ - std::string home=getenv("HOME")?getenv("HOME"):"/tmp"; - std::ifstream f(home+"/.tai_memory.txt"); std::ostringstream ss; ss<ToolResult{ - for(auto base:{std::string("./.tai/skills/"),std::string(std::string(getenv("HOME")?:"/tmp")+"/.tai/skills/")}){ - std::ifstream f(base+arg(a,"name")+"/SKILL.md"); if(f){ std::ostringstream ss; ss<ToolResult{ return ok("[forked task queued: "+arg(a,"task").substr(0,100)+"]"); }); - add("poll","Collect subagent",[](auto&){ return ok("[no finished subagents]"); }); - add("ask_user","Ask user",[](auto& a)->ToolResult{ return ok("[non-interactive; question logged: "+arg(a,"question")+"]"); }); - add("sysinfo","Host info",[](auto&)->ToolResult{ - detz::shell::ShellSession sh; return ok(sh.exec("uname -a; nproc; free -m | head -3; df -h / | tail -1").out); }); - add("create_skill","Create skill",[](auto& a)->ToolResult{ - fs::path d=fs::path("./.tai/skills")/arg(a,"name"); std::error_code ec; fs::create_directories(d,ec); - std::ofstream f(d/"SKILL.md"); f<<"# "+arg(a,"name")+"\n\n"+arg(a,"brief"); return ok("skill created at "+d.string()); }); - add("install_skill","Install skill",[](auto& a)->ToolResult{ return ok("[installed "+arg(a,"source")+"]"); }); - add("store_secret","Store secret",[](auto& a)->ToolResult{ - std::string home=getenv("HOME")?getenv("HOME"):"/tmp"; - std::ofstream f(home+"/.tai_secrets",std::ios::app); f<ToolResult{ - std::string home=getenv("HOME")?getenv("HOME"):"/tmp"; - std::ifstream f(home+"/.tai_secrets"); std::ostringstream ss; ss<ToolResult{ return ok("deleted "+arg(a,"name")); }); - add("schedule","Schedule prompt",[](auto& a)->ToolResult{ return ok("[scheduled: "+arg(a,"prompt").substr(0,80)+"]"); }); - add("unschedule","Delete schedule",[](auto& a)->ToolResult{ return ok("unscheduled "+arg(a,"id")); }); - add("schedules","List schedules",[](auto&){ return ok("(none)"); }); - add("record_save","Save record",[workdir](auto& a)->ToolResult{ - try{ detz::store::Store st(workdir+"/detz.db"); auto id=st.add_record(arg(a,"kind","note"),arg(a,"title",""),arg(a,"content")); - char b[64]; snprintf(b,sizeof b,"mem:%016llx",(unsigned long long)id); return ok(b); } - catch(std::exception& e){ return err(e.what()); } }); - add("record_read","Read record",[workdir](auto& a)->ToolResult{ - try{ detz::store::Store st(workdir+"/detz.db"); - unsigned long long id=0; sscanf(arg(a,"id").c_str(),"mem:%llx",&id); - auto r=st.get_record((int64_t)id); if(!r) return err("not found"); return ok(r->content); } - catch(std::exception& e){ return err(e.what()); } }); - add("record_search","Search records",[workdir](auto& a)->ToolResult{ - try{ detz::store::Store st(workdir+"/detz.db"); auto hits=st.search(arg(a,"query"),10); - std::string o; for(auto& h:hits) o+=h.item+" "+h.title+" "+h.snippet+"\n"; return ok(o.empty()?"(none)":o); } - catch(std::exception& e){ return err(e.what()); } }); - add("record_delete","Delete record",[workdir](auto& a)->ToolResult{ - try{ detz::store::Store st(workdir+"/detz.db"); - unsigned long long id=0; sscanf(arg(a,"id").c_str(),"mem:%llx",&id); - return ok(st.delete_record((int64_t)id)?"deleted":"not found"); } - catch(std::exception& e){ return err(e.what()); } }); - add("graph_link","Link nodes",[](auto& a)->ToolResult{ return ok("linked "+arg(a,"a")+" -> "+arg(a,"b")); }); - add("graph_query","Query graph",[](auto& a)->ToolResult{ return ok("(no edges for "+arg(a,"node")+")"); }); - add("delete_file","Delete file",[](auto& a)->ToolResult{ - std::error_code ec; bool gone=fs::remove(arg(a,"path"),ec); return {gone&&!ec, gone?"deleted":"failed: "+ec.message()}; }); - add("audit","Audit trail",[](auto&){ return ok("(audit via git log in this build)"); }); - add("restore","Restore file",[](auto& a)->ToolResult{ return ok("[restore "+arg(a,"path")+" @ "+arg(a,"id")+"]"); }); - add("release","Cut release",[](auto& a)->ToolResult{ - detz::shell::ShellSession sh; return ok(sh.exec("git tag v-$(date +%s) 2>&1; echo released").out); }); - add("tags","List tags",[](auto&){ return ok("shell file web memory skill terminal plexer tui record secret schedule"); }); - add("install","Manage install",[](auto& a)->ToolResult{ return ok("[install action: "+arg(a,"action")+"]"); }); - add("create_bot","Create bot",[](auto& a)->ToolResult{ return ok("bot @"+arg(a,"name")+" created"); }); - add("search","Search everything",[workdir](auto& a)->ToolResult{ - try{ detz::store::Store st(workdir+"/detz.db"); auto hits=st.search(arg(a,"query"),10); - std::string o; for(auto& h:hits) o+=h.item+" "+h.title+"\n"; return ok(o.empty()?"(none)":o); } - catch(std::exception& e){ return err(e.what()); } }); - add("mcp_add","Add MCP server",[](auto& a)->ToolResult{ return ok("mcp server "+arg(a,"name")+" registered"); }); - add("mcp_list","List MCP servers",[](auto&){ return ok("(none configured)"); }); - add("mcp_enable","Enable MCP",[](auto& a)->ToolResult{ return ok("enabled "+arg(a,"server")); }); - add("mcp_disable","Disable MCP",[](auto& a)->ToolResult{ return ok("disabled "+arg(a,"server")); }); - add("mcp_remove","Remove MCP",[](auto& a)->ToolResult{ return ok("removed "+arg(a,"server")); }); - add("mcp_tools","List MCP tools",[](auto&){ return ok("(no MCP tools)"); }); - add("mcp_call","Call MCP tool",[](auto& a)->ToolResult{ return err("no MCP transport for "+arg(a,"tool")); }); - add("mcp_login","MCP OAuth login",[](auto& a)->ToolResult{ return err("interactive login required for "+arg(a,"server")); }); - return n; -} -} // namespace detz::agent diff --git a/taicpp/src/agent/exec.h b/taicpp/src/agent/exec.h deleted file mode 100644 index 9f61fe7..0000000 --- a/taicpp/src/agent/exec.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once -#include "agent.h" -#include -namespace detz::agent { -// Registers real executors for all 61 tools backed by shell/store/util/http. -// Returns count registered. Idempotent per registry. -int RegisterAll(ToolRegistry& reg, const std::string& workdir); -} // namespace detz::agent diff --git a/taicpp/src/agent/gen/gen_data.py b/taicpp/src/agent/gen/gen_data.py new file mode 100644 index 0000000..b0fc70b --- /dev/null +++ b/taicpp/src/agent/gen/gen_data.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Regenerate the byte-exact data files of the agent core from ../tai.py: + + python3 -I src/agent/gen/gen_data.py > src/agent/data.cpp + python3 -I src/agent/gen/gen_data.py html > src/agent/data_html.cpp + python3 -I src/agent/gen/gen_data.py unicode > src/llm/unicode_data.cpp + +Tool schemas, tool tags and prompt texts come from tai.py; the HTML entity +tables come from the Python standard library (html.unescape input). The +output is deterministic. tai.py is imported, never modified.""" +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +sys.path.insert(0, ROOT) +sys.dont_write_bytecode = True +import tai # noqa: E402 + + +def lit(text): + """C++ raw string literal, split in chunks below the 64k literal comfort.""" + delim = "TAIRAW" + assert (")" + delim + '"') not in text + chunks = [text[i:i + 8000] for i in range(0, len(text), 8000)] or [""] + return "\n ".join('R"%s(%s)%s"' % (delim, chunk, delim) for chunk in chunks) + + +def cesc(text): + """Plain C string body: printable ASCII verbatim, everything else octal bytes.""" + out = [] + for ch in text: + cp = ord(ch) + if ch == '"' or ch == "\\": + out.append("\\" + ch) + elif 0x20 <= cp < 0x7F: + out.append(ch) + else: + for byte in ch.encode("utf-8", "surrogatepass"): + out.append("\\%03o" % byte) + return "".join(out) + + +def html_table(): + import html + from html import entities + out = [] + out.append("// GENERATED by src/agent/gen/gen_data.py html from Python %s html.entities. Do not edit." % sys.version.split()[0]) + out.append('#include "data.h"') + out.append("namespace detz::agent::data {") + out.append("const std::vector>& Html5Entities() {") + out.append(" static const std::vector> v = {") + for name in sorted(entities.html5): + out.append(' {"%s", "%s"},' % (cesc(name), cesc(entities.html5[name]))) + out.append(" };") + out.append(" return v;") + out.append("}") + out.append("const std::vector>& HtmlInvalidCharrefs() {") + out.append(" static const std::vector> v = {") + for num in sorted(html._invalid_charrefs): + out.append(' {%d, "%s"},' % (num, cesc(html._invalid_charrefs[num]))) + out.append(" };") + out.append(" return v;") + out.append("}") + out.append("const std::vector& HtmlInvalidCodepoints() {") + out.append(" static const std::vector v = {%s};" % ", ".join(str(n) for n in sorted(html._invalid_codepoints))) + out.append(" return v;") + out.append("}") + out.append("} // namespace detz::agent::data") + sys.stdout.write("\n".join(out) + "\n") + + +def ranges(code_points): + """Sorted code points -> [[start, end_inclusive]].""" + out = [] + for cp in code_points: + if out and out[-1][1] + 1 == cp: + out[-1][1] = cp + else: + out.append([cp, cp]) + return out + + +def unicode_tables(): + import unicodedata + lower, upper, spaces = [], [], [] + sig_ignorable, sig_cased, digits = [], [], [] + sigma = "Σ" + final = "ς" + for cp in range(0x110000): + ch = chr(cp) + low = ch.lower() + if low != ch: + lower.append((cp, low)) + up = ch.upper() + if up != ch: + upper.append((cp, up)) + if ch.isspace(): + spaces.append(cp) + if 0xD800 <= cp <= 0xDFFF: + continue + f1 = ("A" + ch + sigma).lower()[-1] == final + f2 = ("A" + sigma + ch).lower()[1] == final + g = (ch + sigma).lower()[-1] == final + if f1 and f2: + sig_ignorable.append(cp) + if g: + sig_cased.append(cp) + if ch.isdecimal(): + digits.append(cp) + out = [] + out.append("// GENERATED by src/agent/gen/gen_data.py unicode from Python %s (unicodedata %s). Do not edit." % ( + sys.version.split()[0], unicodedata.unidata_version)) + out.append('#include "unicode_data.h"') + out.append("namespace detz::py::ucd {") + for name, table in (("LowerMap", lower), ("UpperMap", upper)): + out.append("const std::vector& %s() {" % name) + out.append(" static const std::vector v = {") + for cp, text in table: + out.append(' {0x%X, "%s"},' % (cp, cesc(text))) + out.append(" };") + out.append(" return v;") + out.append("}") + for name, cps in (("SpaceRanges", spaces), ("SigmaIgnorableRanges", sig_ignorable), + ("SigmaCasedRanges", sig_cased), ("DecimalRanges", digits)): + out.append("const std::vector& %s() {" % name) + out.append(" static const std::vector v = {") + for first, last in ranges(cps): + out.append(" {0x%X, 0x%X}," % (first, last)) + out.append(" };") + out.append(" return v;") + out.append("}") + out.append("} // namespace detz::py::ucd") + sys.stdout.write("\n".join(out) + "\n") + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "html": + html_table() + return + if len(sys.argv) > 1 and sys.argv[1] == "unicode": + unicode_tables() + return + out = [] + out.append("// GENERATED by src/agent/gen/gen_data.py from tai.py (version %s). Do not edit." % tai.VERSION) + out.append('#include "data.h"') + out.append("namespace detz::agent::data {") + out.append("const std::vector& ToolSchemaJson() {") + out.append(" static const std::vector v = {") + for schema in tai.TOOL_SCHEMAS: + out.append(" " + lit(json.dumps(schema)) + ",") + out.append(" };") + out.append(" return v;") + out.append("}") + out.append("const std::vector>>& ToolTags() {") + out.append(" static const std::vector>> v = {") + for name, tags in tai.TOOL_TAGS.items(): + out.append(' {"%s", {%s}},' % (name, ", ".join('"%s"' % t for t in tags))) + out.append(" };") + out.append(" return v;") + out.append("}") + out.append("const std::vector& CoreTools() {") + out.append(" static const std::vector v = {%s};" % ", ".join('"%s"' % t for t in tai.CORE_TOOLS)) + out.append(" return v;") + out.append("}") + out.append("const std::vector& LazyNameStop() {") + out.append(" static const std::vector v = {%s};" % ", ".join('"%s"' % t for t in tai.LAZY_NAME_STOP)) + out.append(" return v;") + out.append("}") + texts = [ + ("DefaultSystem", tai.DEFAULT_SYSTEM), + ("MergeSystem", tai.MERGE_SYSTEM), + ("CompactSystem", tai.COMPACT_SYSTEM), + ("AutonomousNote", tai.AUTONOMOUS_NOTE), + ("LegacyPasswordNote", tai.LEGACY_PASSWORD_NOTE), + ("VaultMemoryNote", tai.VAULT_MEMORY_NOTE), + ("McpResolverBrief", tai.MCP_RESOLVER_BRIEF), + ("GoalVerifierTask", tai.GoalVerifier.TASK), + ("CurlHint", tai.CURL_HINT), + ] + for name, text in texts: + out.append("const std::string& %s() {" % name) + out.append(" static const std::string s =") + out.append(" " + lit(text) + ";") + out.append(" return s;") + out.append("}") + out.append("} // namespace detz::agent::data") + sys.stdout.write("\n".join(out) + "\n") + + +main() diff --git a/taicpp/src/agent/goal.cpp b/taicpp/src/agent/goal.cpp new file mode 100644 index 0000000..4eb6627 --- /dev/null +++ b/taicpp/src/agent/goal.cpp @@ -0,0 +1,212 @@ +#include "goal.h" +#include "../llm/py.h" +#include "consts.h" +#include "data.h" +#include +namespace detz::agent { + +std::string Approval::parse_allow(const std::string& answer) { + std::string cleaned = py::strip(answer); + std::string lowered = py::lower(cleaned); + if (cleaned == "Y" || lowered == "yolo" || lowered == "always") { return "always"; } + if (lowered == "y" || lowered == "yes") { return "once"; } + if (lowered == "n" || lowered == "no" || lowered.empty()) { return "deny"; } + return "unknown"; +} + +bool Approval::is_affirmative(const std::string& text) { + static const char* phrases[] = { + "y", "yes", "yeah", "yep", "sure", + "proceed", "go ahead", "do it", "run it", "allow", "approve", "approved", + "ok", "okay", "fine", "good", "great", "perfect", + "all good", "looks good", "sounds good", + "it was perfect", "that was perfect", "everything was fine", "everything is fine"}; + std::string words = py::join(py::split_ws(py::lower(py::strip(text))), " "); + words = py::rstrip_chars(words, ".!?"); + if (words.empty()) { return false; } + for (const char* p : phrases) { if (words == p) { return true; } } + return false; +} + +std::optional GoalFailure::from_dict(const json::Value& raw) { + if (!raw.is_object()) { return std::nullopt; } + std::string item = py::strip(py::str_or_empty(raw.get("item"))); + std::string reason = py::strip(py::str_or_empty(raw.get("reason"))); + std::string proposal = py::strip(py::str_or_empty(raw.get("proposal"))); + if (item.empty() || reason.empty() || proposal.empty()) { return std::nullopt; } + return GoalFailure{py::head(item, VERIFY_MAX_ITEM), py::head(reason, VERIFY_MAX_REASON), py::head(proposal, VERIFY_MAX_PROPOSAL)}; +} +json::Value GoalFailure::to_dict() const { + json::Value d = json::Value::object(); + d.set("item", item); + d.set("reason", reason); + d.set("proposal", proposal); + return d; +} + +GoalVerdict GoalVerdict::from_text(const std::string& text) { + std::string raw = py::strip(text); + auto [data, error] = GoalVerifier::extract_json(raw); + std::vector failures; + std::vector broken; + json::Value success; + if (error.has_value()) { + broken.push_back("verifier returned no usable JSON: " + *error); + } else if (!data.is_object()) { + broken.push_back("verifier returned a JSON value that is not an object"); + } else { + success = data.get("success"); + const json::Value& entries = data.get("failures"); + if (!success.is_bool()) { broken.push_back("field success is not true or false"); } + if (!entries.is_array()) { + broken.push_back("field failures is not a list"); + } else { + size_t pos = 0; + for (const auto& entry : entries.items()) { + if (pos >= static_cast(VERIFY_MAX_FAILURES)) { break; } + auto parsed = GoalFailure::from_dict(entry); + if (!parsed.has_value()) { + broken.push_back("failure " + std::to_string(pos + 1) + " misses item, reason or proposal"); + } else { + failures.push_back(*parsed); + } + ++pos; + } + } + if (success.is_bool() && success.as_bool() && !failures.empty()) { + broken.push_back("verifier claims success but lists failures"); + } + } + bool ok = broken.empty() && success.is_bool() && success.as_bool(); + for (const auto& note : broken) { + failures.push_back(GoalFailure{"goal verification", note, "re-run the request so it can be verified"}); + } + if (!ok && failures.empty()) { + failures.push_back(GoalFailure{"goal", "verifier reported failure without details", "re-run the request with explicit checks"}); + } + if (failures.size() > static_cast(VERIFY_MAX_FAILURES)) { failures.resize(static_cast(VERIFY_MAX_FAILURES)); } + std::string commentary; + if (data.is_object() && data.get("commentary").is_string()) { + commentary = py::head(py::strip(data.get("commentary").str()), VERIFY_MAX_COMMENTARY); + } + GoalVerdict v; + v.ok = ok; + v.failures = failures; + v.commentary = commentary; + v.raw = py::head(raw, VERIFY_MAX_COMMENTARY); + return v; +} + +std::string GoalVerdict::report(const std::string& goal) const { + std::string title = py::head(py::join(py::split_ws(goal), " "), 200); + if (title.empty()) { title = "goal"; } + if (ok) { + std::string note = commentary.empty() ? "" : " " + commentary; + return "goal verified: " + title + note; + } + std::vector lines = {"goal NOT verified: " + title}; + for (const auto& f : failures) { + lines.push_back("- " + f.item + ": " + f.reason); + lines.push_back(" next: " + f.proposal); + } + if (!commentary.empty()) { lines.push_back(commentary); } + return py::join(lines, "\n"); +} + +std::string GoalVerdict::followup() const { + std::vector lines = {"Goal verification FAILED, the goal is not delivered yet. Continue now with the next proposal below."}; + for (const auto& f : failures) { + lines.push_back("- " + f.item + ": " + f.reason + " Next: " + f.proposal); + } + if (!commentary.empty()) { lines.push_back(commentary); } + lines.push_back("Do not repeat denied, skipped or failed actions verbatim; use the proposals and move forward."); + return py::head(py::join(lines, "\n"), VERIFY_MAX_CHARS); +} + +std::pair> GoalVerifier::extract_json(const std::string& text) { + std::string body = py::strip(text); + if (py::startswith(body, "```")) { + body = body.substr(3); + if (py::lower(py::head(body, 4)) == "json") { body = py::slice(body, 4, std::nullopt); } + size_t end = body.find("```"); + if (end != std::string::npos) { body = body.substr(0, end); } + body = py::strip(body); + } + size_t start = body.find('{'); + size_t end = body.rfind('}'); + if (start == std::string::npos || end == std::string::npos || end <= start) { + return {json::Value(), std::string("no JSON object found")}; + } + try { + return {json::Value::loads(body.substr(start, end + 1 - start)), std::nullopt}; + } catch (const json::ParseError& exc) { + return {json::Value(), "invalid JSON: " + std::string(exc.what())}; + } +} + +std::string GoalVerifier::transcript(const json::Value& messages) { + std::vector lines; + for (const auto& item : messages.items()) { + std::string role = item.get("role").truthy() ? py::str(item.get("role")) : "?"; + if (role == "system") { continue; } + std::string text = py::str_or_empty(item.get("content")); + if (role == "assistant") { + std::vector names; + if (item.get("tool_calls").truthy()) { + for (const auto& call : item.get("tool_calls").items()) { + json::Value fn = call.get("function").truthy() ? call.get("function") : json::Value::object(); + std::string nm = fn.has("name") ? py::str(fn.get("name")) : "?"; + names.push_back(nm + " " + py::head(py::str_or_empty(fn.get("arguments")), 80)); + } + } + if (!names.empty()) { text = py::strip(text + " [tools: " + py::join(names, "; ") + "]"); } + } + lines.push_back(py::head("[" + role + "] " + py::join(py::split_ws(text), " "), VERIFY_MAX_LINE)); + } + std::string joined = py::join(lines, "\n"); + return static_cast(py::len(joined)) > VERIFY_MAX_CHARS ? py::tail(joined, VERIFY_MAX_CHARS) : joined; +} + +std::vector GoalVerifier::tried_actions(const json::Value& messages) { + std::vector tried; + for (const auto& item : messages.items()) { + if (!item.get("tool_calls").truthy()) { continue; } + for (const auto& call : item.get("tool_calls").items()) { + json::Value fn = call.get("function").truthy() ? call.get("function") : json::Value::object(); + std::string args = py::head(py::join(py::split_ws(py::str_or_empty(fn.get("arguments"))), " "), 80); + std::string name = fn.get("name").truthy() ? py::str(fn.get("name")) : "?"; + tried.push_back(py::strip(name + " " + args)); + } + } + if (tried.size() > static_cast(VERIFY_MAX_TRIED)) { tried.erase(tried.begin(), tried.end() - VERIFY_MAX_TRIED); } + return tried; +} + +std::string GoalVerifier::build_task(const std::string& goal, const std::string& final_text, + const std::string& transcript, const std::vector& tried) { + std::string goal_text = py::head(py::join(py::split_ws(goal), " "), VERIFY_MAX_GOAL); + if (goal_text.empty()) { goal_text = "(empty goal)"; } + std::string final_clean = py::head(py::join(py::split_ws(final_text), " "), VERIFY_MAX_COMMENTARY); + if (final_clean.empty()) { final_clean = "(empty answer)"; } + std::string transcript_text = transcript.empty() ? "(no transcript)" : transcript; + std::string tried_text = tried.empty() ? "(none)" : py::join(tried, "\n"); + // GoalVerifier.TASK % (goal, final, transcript, tried, VERIFY_MAX_FAILURES) + const std::string& tmpl = data::GoalVerifierTask(); + std::vector args = {goal_text, final_clean, transcript_text, tried_text, std::to_string(VERIFY_MAX_FAILURES)}; + std::string out; + size_t k = 0; + for (size_t i = 0; i < tmpl.size(); ++i) { + if (tmpl[i] == '%' && i + 1 < tmpl.size() && (tmpl[i + 1] == 's' || tmpl[i + 1] == 'd')) { + out += args.at(k++); + ++i; + } else if (tmpl[i] == '%' && i + 1 < tmpl.size() && tmpl[i + 1] == '%') { + out += '%'; + ++i; + } else { + out += tmpl[i]; + } + } + return out; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/goal.h b/taicpp/src/agent/goal.h new file mode 100644 index 0000000..75aeed7 --- /dev/null +++ b/taicpp/src/agent/goal.h @@ -0,0 +1,58 @@ +#pragma once +// Denied, AgentsFull, Approval, GoalFailure, GoalVerdict, GoalVerifier +// (tai.py 19110-19354). +#include "../llm/json.h" +#include +#include +#include +#include +#include +namespace detz::agent { + +// class Denied(Exception): "denied by user" plus the guidance the user typed. +class Denied : public std::exception { + public: + explicit Denied(std::string guidance = "") : guidance_(std::move(guidance)) {} + const char* what() const noexcept override { return "denied by user"; } + const std::string& guidance() const { return guidance_; } + private: + std::string guidance_; +}; + +class AgentsFull : public std::runtime_error { + public: + explicit AgentsFull(const std::string& m) : std::runtime_error(m) {} +}; + +struct Approval { + // "once" | "always" | "deny" | "unknown" + static std::string parse_allow(const std::string& answer); + static bool is_affirmative(const std::string& text); +}; + +struct GoalFailure { + std::string item, reason, proposal; + // nullopt when raw is not a dict or misses item, reason or proposal. + static std::optional from_dict(const json::Value& raw); + json::Value to_dict() const; +}; + +struct GoalVerdict { + bool ok{false}; + std::vector failures; + std::string commentary, raw; + static GoalVerdict from_text(const std::string& text); + std::string report(const std::string& goal) const; + std::string followup() const; +}; + +struct GoalVerifier { + // (data, error): data null Value when error is set. + static std::pair> extract_json(const std::string& text); + static std::string transcript(const json::Value& messages); + static std::vector tried_actions(const json::Value& messages); + static std::string build_task(const std::string& goal, const std::string& final_text, + const std::string& transcript, const std::vector& tried); +}; + +} // namespace detz::agent diff --git a/taicpp/src/agent/loop.cpp b/taicpp/src/agent/loop.cpp deleted file mode 100644 index 4674652..0000000 --- a/taicpp/src/agent/loop.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "loop.h" -#include "tools.h" -#include -namespace detz::agent { -TurnResult RunTurn(ToolRegistry& reg, const std::string& text, int max_steps) { - TurnResult r; - auto names = SelectTools(text); - bool has_path = text.find('/') != std::string::npos || text.find('.') != std::string::npos; - bool has_url = text.find("http") != std::string::npos || text.find("www.") != std::string::npos; - std::string out; - for (auto& n : names) { - if (r.steps >= max_steps || r.steps >= 500) break; - if ((n=="read_file"||n=="write_file"||n=="edit_file"||n=="delete_file") && !has_path) continue; - if ((n=="web_fetch"||n=="web_search") && n=="web_fetch" && !has_url) continue; - if (n=="shell" && (text.size()<3)) continue; - std::map args = {{"query", text}, {"question", text}, - {"instruction", text}, {"task", text}, {"text", text}, {"prompt", text}}; - if (has_path) { args["path"]=text; args["content"]=text; } - if (has_url) { args["url"]=text; } - auto res = reg.call(n, args); - ++r.steps; - if (res) out += "[" + n + "] " + (res->ok ? res->out : ("ERROR: " + res->out)) + "\n"; - } - if (out.empty()) out = text; - r.output = out; - return r; -} -} // namespace detz::agent diff --git a/taicpp/src/agent/loop.h b/taicpp/src/agent/loop.h deleted file mode 100644 index 6e259b9..0000000 --- a/taicpp/src/agent/loop.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once -#include "agent.h" -#include -namespace detz::agent { -// Autonomous turn loop mirroring tai.py Agent.run_turn: select tools, -// dispatch via registry, cap steps at TOTAL_STEP_CAP(500)/max_steps. -struct TurnResult { bool ok{true}; std::string output; int steps{0}; }; -TurnResult RunTurn(ToolRegistry& reg, const std::string& text, int max_steps = 10); -} // namespace detz::agent diff --git a/taicpp/src/agent/ports.h b/taicpp/src/agent/ports.h new file mode 100644 index 0000000..6de35e8 --- /dev/null +++ b/taicpp/src/agent/ports.h @@ -0,0 +1,325 @@ +#pragma once +// Narrow interfaces between the agent core and everything it does not own. +// Every method mirrors the tai.py function or Store method of the same name; +// dict/list results are json::Value with the key names tai.py uses. The +// integrator implements each port on the real C++ module (docs/wiring/agent.md). +#include "../llm/json.h" +#include "../llm/proc.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace detz::agent { + +using Value = json::Value; + +// Raised by a port that is not wired (message names the missing port). +class PortNotWired : public std::runtime_error { + public: + explicit PortNotWired(const std::string& what) : std::runtime_error("port not wired: " + what) {} +}; + +// --------------------------------------------------------------------------- +// Store: the subset of tai.py Store (line 5741) the agent and tools use. +// All profile arguments follow `_scope`: empty string means the store's own. +// --------------------------------------------------------------------------- +class StorePort { + public: + virtual ~StorePort() = default; + virtual std::string profile() const = 0; // Store.profile + virtual void set_profile(const std::string& name) = 0; + virtual bool seal_enabled() const = 0; // Store.seal.enabled + + virtual std::string redact(const std::string& text, const std::string& profile) = 0; + virtual void log_event(const std::string& profile, const std::string& role, const std::string& kind, + const std::string& text) = 0; + + // sessions, systems, bots, turn state + virtual Value load_session(const std::string& name, const std::string& bot, const std::string& ctx) = 0; // array of messages + virtual void save_session(const std::string& name, const Value& messages, const std::string& bot, const std::string& ctx) = 0; + virtual std::optional load_system(const std::string& name) = 0; + virtual void save_system(const std::string& name, const std::string& text) = 0; + virtual std::optional load_bot_system(const std::string& profile, const std::string& bot) = 0; + virtual void save_bot_system(const std::string& profile, const std::string& bot, const std::string& text) = 0; + virtual std::optional resolve_bot(const std::string& profile, const std::string& mention) = 0; + virtual Value list_bots(const std::string& profile) = 0; // [{name, nicknames, ...}] + virtual Value load_bots(const std::string& profile) = 0; // {name: {nicknames, created, updated}} + virtual void save_bots(const std::string& profile, const Value& bots) = 0; + virtual Value load_turn_state(const std::string& profile, const std::string& bot, const std::string& ctx) = 0; // {} or {open, goal, steps, tools, updated} + virtual void save_turn_state(const std::string& profile, const std::string& bot, const Value& state, const std::string& ctx) = 0; + virtual void save_dir_defaults(const std::string& directory, const std::optional& provider, + const std::optional& model, const std::optional& bot, + const std::vector& clear) = 0; + + // records, chunks, tags, graph + virtual std::string add_record(const std::string& kind, const std::string& title, const std::string& content, + const std::vector& tags, const std::string& profile) = 0; + virtual void add_chunks(const std::string& parent_kind, const std::string& parent_id, + const std::vector& texts, const std::string& profile) = 0; + virtual Value get_chunks_by_ids(const std::vector& ids, const std::string& profile) = 0; // {id: {parent_kind, parent_id, idx, text}} + virtual Value read_record(const std::string& record_id, int64_t offset, int64_t limit, const std::string& profile) = 0; // null or {id, slice, start, end, total, tags} + virtual Value search_records(const std::string& query, const std::optional& kind, + const std::vector& tags, int64_t limit, const std::string& profile) = 0; // [{id, kind, title, size, reads, tags}] + virtual bool delete_record(const std::string& record_id, const std::string& profile) = 0; + virtual std::vector item_tags(const std::string& item, const std::string& profile) = 0; + virtual std::vector> tag_counts(const std::string& prefix, int64_t limit, const std::string& profile) = 0; + virtual std::string add_edge(const std::string& src, const std::string& dst, const Value& relation, const std::string& profile) = 0; // throws std::invalid_argument (ValueError) + virtual Value edges_for(const std::string& node, const std::string& profile) = 0; // [{other, direction, relation}] + virtual bool has_node(const std::string& kind, const std::string& key, const std::string& profile) = 0; + virtual std::string node_title(const std::string& node, const std::string& profile) = 0; + virtual Value traverse(const std::string& start, int64_t depth, int64_t limit, const std::string& profile) = 0; // [{node, depth, via, title}] + + // search + virtual Value fts_search(const std::string& query, const std::vector& kinds, const std::string& profile, int64_t limit) = 0; // [{kind, item, sub, title, snippet, rank}] + virtual Value events_like(const std::string& profile, const std::string& query, int64_t limit) = 0; // [[ts, role, kind, text]] + virtual Value search_events(const std::string& profile, const std::string& query, const std::vector& tags) = 0; // [[ts, role, kind, text]] + virtual Value audit_search(const std::string& query, const std::string& profile, int64_t limit) = 0; + + // audit + file records + virtual int64_t audit_event(const std::string& actor, const std::string& action, const std::string& path, + const std::string& message, const std::optional& old_text, + const std::optional& new_text, const std::vector& tags, + const std::optional& old_size, const std::optional& new_size, + const std::string& profile) = 0; // throws on sqlite/OS failure + virtual Value audit_history(const std::optional& path, const std::optional& tag, + int64_t limit, const std::string& profile) = 0; // [{id, ts, actor, action, path, old_size, new_size, message, tags}] + virtual Value audit_get(int64_t row_id, const std::string& profile) = 0; // null or row incl. old/new + virtual void upsert_file_record(const std::string& path, const std::string& content, + const std::optional& true_size, const std::string& profile) = 0; + virtual void delete_file_record(const std::string& path, const std::string& profile) = 0; + + // secrets + virtual std::optional load_secret(const std::string& name, const std::string& profile) = 0; + virtual void save_secret(const std::string& name, const std::string& value, const Value& meta, + const std::vector& tags, const std::string& profile) = 0; + virtual Value list_secret_infos(const std::string& profile) = 0; // [{name, meta, tags, updated}] + virtual void delete_secret(const std::string& name, const std::string& profile) = 0; + + // schedules + virtual int64_t add_schedule(const std::string& name, const std::string& prompt, const std::string& profile, + int64_t every_sec, const std::string& next_run, int64_t timeout, + const std::vector& tags) = 0; + virtual void remove_schedule(int64_t row_id, const std::string& profile) = 0; + virtual Value list_schedules(const std::string& profile) = 0; // [{id, name, prompt, status, every, next_run, tags, last, result}] + + // MCP registry rows + virtual Value mcp_list_servers() = 0; + virtual Value mcp_get_server(const std::string& name) = 0; // null when unknown + virtual void mcp_save_server(const Value& entry) = 0; // throws std::invalid_argument + virtual void mcp_remove_server(const std::string& name) = 0; + virtual void mcp_set_enabled(const std::string& server, const std::string& profile, bool enabled) = 0; + virtual std::vector mcp_enabled_profiles(const std::string& server) = 0; // profiles with enabled = 1 + virtual std::vector mcp_server_profiles(const std::string& server) = 0; // DISTINCT profile rows + virtual void mcp_store_tools(const std::string& server, const Value& tools) = 0; + virtual Value mcp_cached_tools(const std::string& server) = 0; // [{tool, description, ...}] + virtual std::vector mcp_tool_pins(const std::string& server) = 0; + + // raw SQL, used by RagStats / RetrievalEval only (read-only SELECTs). + // Rows come back as arrays of scalars. Throws std::runtime_error on sqlite errors. + virtual Value sql_rows(const std::string& sql, const Value& params) = 0; + virtual bool fts_ok() const = 0; +}; + +// --------------------------------------------------------------------------- +// Runtime: opening a second store for worker agents (default_runner). +// --------------------------------------------------------------------------- +class RuntimePort { + public: + virtual ~RuntimePort() = default; + // Store(config, seal) for a worker, sharing the parent's seal. Caller owns it. + virtual std::unique_ptr open_store() = 0; + // boot_settings(config, store): apply vault settings to the config. + virtual void boot_settings(StorePort& store) = 0; +}; + +// --------------------------------------------------------------------------- +// Shell / sandbox: run_live, LiveStream, LaneBoard, box_* (src/shell). +// --------------------------------------------------------------------------- +struct RunResult { + int code{0}; + std::string output; + double elapsed{0}; + bool timed_out{false}; +}; +class LiveStreamPort { + public: + virtual ~LiveStreamPort() = default; + virtual void close(const std::string& status) = 0; // LiveStream.close(status) +}; +// ask(text, secret) -> answer; nullopt means "no answer" (end-of-file is sent). +using AskFn = std::function(const std::string& text, bool secret)>; +class LaneBoardPort { + public: + virtual ~LaneBoardPort() = default; + virtual void start() = 0; + virtual void stop() = 0; +}; +class ShellPort { + public: + virtual ~ShellPort() = default; + // run_live(command, shell=True, cwd, env, stream, ask). env == nullptr: inherit. + virtual RunResult run_live_shell(const std::string& command, const std::string& cwd, + const std::map& extra_env, LiveStreamPort* stream, + const AskFn& ask) = 0; + // run_live(argv, stream=, ask=) for argv commands (container exec). + virtual RunResult run_live_argv(const std::vector& argv, LiveStreamPort* stream, const AskFn& ask) = 0; + virtual std::unique_ptr make_live_stream() = 0; // LiveStream() + virtual std::vector shell_target_paths(const std::string& command, const std::string& workdir) = 0; + // Sandbox box. ensure_box throws llm::BackendError. + virtual std::string ensure_box() = 0; + virtual llm::ProcResult box_exec(const std::string& engine, const std::vector& argv, + const std::optional& input, double timeout_s) = 0; + // LaneBoard.available(app) ? LaneBoard(profile) : nullptr + virtual std::unique_ptr make_lane_board(const std::string& profile) = 0; +}; + +// --------------------------------------------------------------------------- +// Display: Spinner, Glow, render_markdown, render_diff (src/shell, src/util). +// --------------------------------------------------------------------------- +struct GlowCall { std::string verb, target, extra, pretty; }; +class UiPort { + public: + virtual ~UiPort() = default; + virtual void spinner_start(const std::string& label) = 0; + virtual void spinner_stop() = 0; + virtual std::string render_markdown(const std::string& text) = 0; + virtual std::string render_diff(const std::string& path, const std::optional& old_text, + const std::optional& new_text, bool colors) = 0; + virtual bool color_enabled() = 0; + // Glow + virtual bool glow_enabled() = 0; + virtual GlowCall glow_describe_call(const std::string& name, const std::string& args_text) = 0; + virtual std::string glow_bar(const std::string& name) = 0; + virtual std::string glow_event_row(const std::string& icon, const std::string& verb, const std::string& target, + const std::string& extra, int inner, const std::string& accent) = 0; + virtual std::string glow_preview_row(const std::string& preview, int inner) = 0; + virtual std::string glow_done_row(const std::string& verb, const std::string& target, size_t chars, + double elapsed_ms, bool failed, int inner) = 0; + virtual std::string glow_icon_for(const std::string& name) = 0; + virtual std::string glow_accent_for(const std::string& name) = 0; + virtual int glow_width() = 0; + virtual std::string glow_hairline() = 0; + virtual std::vector glow_lanes(const Value& records) = 0; +}; + +// --------------------------------------------------------------------------- +// MCP bridge (src/mcp): McpBridge / McpOAuth / mcp_* helpers. +// --------------------------------------------------------------------------- +class McpClientPort { + public: + virtual ~McpClientPort() = default; + virtual Value list_tools() = 0; // throws McpError / McpHttpError + virtual void close() = 0; +}; +struct McpError : public std::runtime_error { using std::runtime_error::runtime_error; }; +struct McpHttpError : public McpError { + McpHttpError(const std::string& m, int c) : McpError(m), code(c) {} + int code; +}; +class McpPort { + public: + virtual ~McpPort() = default; + virtual Value schemas_for(const std::string& profile) = 0; // MCP tool schemas offered to the model + virtual std::optional> resolve(const std::string& tool_name) = 0; // McpBridge.resolve + virtual std::string call(const std::string& server, const std::string& tool, const Value& args, const std::string& profile) = 0; + virtual std::unique_ptr connect(const Value& entry, const std::string& profile) = 0; + virtual std::pair> normalize(const std::string& text, const std::optional& name) = 0; // mcp_normalize + virtual std::optional> github_target(const std::string& source) = 0; // mcp_github_target + virtual std::string sanitize_name(const std::string& raw) = 0; // mcp_sanitize_name + virtual std::pair> rewrite_placeholders(const std::string& source) = 0; + // McpOAuth + virtual std::pair oauth_load_tokens(const std::string& server, const std::string& profile) = 0; + virtual void oauth_drop_tokens(const std::string& server, const std::string& profile) = 0; + virtual std::string oauth_login(const Value& server_row, const std::string& profile) = 0; // returns note +}; + +// --------------------------------------------------------------------------- +// Skills (src/skills / install): discover_skills & friends. +// --------------------------------------------------------------------------- +class SkillsPort { + public: + virtual ~SkillsPort() = default; + virtual Value discover_skills(const std::string& home_dir, const std::string& project_dir) = 0; // {name: {description, body, root, install?, managed?}} + virtual std::string skill_catalog(const Value& skills) = 0; + virtual std::string skill_provenance_note(const Value& skill) = 0; + virtual std::vector skill_extra_files(const std::string& root) = 0; + virtual std::optional skill_blueprint(const std::string& name) = 0; // {brief, scope, hint} + virtual std::string skill_scope_dir(const std::string& scope, const std::string& home_dir, + const std::string& project_dir, const std::string& name) = 0; + virtual std::string default_skill_scope(const std::string& project_dir) = 0; + virtual Value parse_skill_file(const std::string& path) = 0; // null when invalid; throws on IO error + virtual bool is_external_skill_root(const std::string& path) = 0; + virtual void save_skill_install_meta(const std::string& scope_dir, const std::string& entry, const Value& meta) = 0; +}; + +// --------------------------------------------------------------------------- +// Plexer / TUI control (src/plexer, src/tui). +// --------------------------------------------------------------------------- +class PlexerControllerPort { + public: + virtual ~PlexerControllerPort() = default; + virtual std::string kind() const = 0; + virtual void send_keys(const std::string& text, bool enter) = 0; // throws PlexerFailure + virtual std::string read_pane(int lines) = 0; + virtual void type_text(const std::string& text, double delay, bool enter) = 0; + virtual Value status() = 0; +}; +struct PlexerFailure : public std::runtime_error { using std::runtime_error::runtime_error; }; +class PlexerPort { + public: + virtual ~PlexerPort() = default; + virtual std::unique_ptr detect() = 0; // PlexerControllerFactory.detect(); nullptr when none + virtual std::string native_read_pane(const std::string& socket_path, const std::string& pane, int lines) = 0; // NativePlexerController(...).read_pane + // validate_flow_spec(spec) -> (app, error); error empty when valid. + virtual std::pair validate_flow_spec(const Value& spec) = 0; + virtual Value run_flow(const Value& app, PlexerControllerPort& controller) = 0; // PlexerFlow(app).run(controller) +}; +class TuiControlPort { + public: + virtual ~TuiControlPort() = default; + virtual bool control_exists() = 0; // os.path.exists(grr_control_path()) + virtual Value control_call(const Value& payload) = 0; // grr_control_call +}; + +// --------------------------------------------------------------------------- +// Ops: install_report, release, sysinfo, speech, scheduler helpers. +// --------------------------------------------------------------------------- +class OpsPort { + public: + virtual ~OpsPort() = default; + virtual std::string install_report(const std::string& action, const std::vector& targets, + const std::string& home) = 0; + virtual std::vector install_targets() = 0; // INSTALL_TARGETS + // release: returns the current version of the running script or throws std::runtime_error. + virtual std::string current_version() = 0; + virtual std::string next_version(const std::string& current, const std::string& part) = 0; + // do_release -> (old, new, backup_basename, audit_row_id) + virtual std::tuple do_release(const std::string& part, const std::string& message, + const std::string& actor) = 0; + virtual std::string collect_sysinfo(const std::optional>& checks) = 0; + virtual std::vector sysinfo_check_names() = 0; + virtual std::string edge_synthesize(const std::string& text, const std::string& voice) = 0; // audio bytes; throws llm::BackendError or std::runtime_error + virtual std::string play_audio(const std::string& path) = 0; + virtual std::string listen_audio(int seconds, const std::string& audio_dir) = 0; +}; + +// Process-wide port registry; unset ports are null and the code that needs +// them returns/throws the "port not wired" error instead of pretending. +struct Ports { + RuntimePort* runtime{nullptr}; + ShellPort* shell{nullptr}; + UiPort* ui{nullptr}; + McpPort* mcp{nullptr}; + SkillsPort* skills{nullptr}; + PlexerPort* plexer{nullptr}; + TuiControlPort* tui{nullptr}; + OpsPort* ops{nullptr}; +}; +Ports& ports(); + +} // namespace detz::agent diff --git a/taicpp/src/agent/rag.cpp b/taicpp/src/agent/rag.cpp new file mode 100644 index 0000000..5962581 --- /dev/null +++ b/taicpp/src/agent/rag.cpp @@ -0,0 +1,803 @@ +#include "rag.h" +#include "../llm/helpers.h" +#include "../llm/py.h" +#include "consts.h" +#include "text.h" +#include +#include +#include +#include +#include +namespace detz::agent { + +namespace { +std::string sv(const json::Value& v) { return py::str(v); } +std::string fmt(const char* f, double a) { char b[64]; std::snprintf(b, sizeof b, f, a); return b; } +double round4(double x) { return std::strtod(fmt("%.4f", x).c_str(), nullptr); } + +json::Value rows_of(StorePort& st, const std::string& sql, const json::Value& params) { + try { + return st.sql_rows(sql, params); + } catch (const std::exception&) { + return json::Value::array(); + } +} +// _one: first column of the first row, null when no row or error +json::Value one_of(StorePort& st, const std::string& sql, const json::Value& params) { + json::Value rows = rows_of(st, sql, params); + if (rows.size() == 0) { return json::Value(); } + return rows.at(0).at(0); +} +int64_t int_or0(const json::Value& v) { return v.truthy() ? v.as_int() : 0; } +} // namespace + +std::optional RetrievalEval::rank_of(const std::string& expected_id, const std::vector& ranked_ids) { + auto it = std::find(ranked_ids.begin(), ranked_ids.end(), expected_id); + if (it == ranked_ids.end()) { return std::nullopt; } + return static_cast(it - ranked_ids.begin()) + 1; +} +std::optional RetrievalEval::best_rank(const std::vector& expected_ids, const std::vector& ranked_ids) { + std::optional best; + for (const auto& e : expected_ids) { + auto r = rank_of(e, ranked_ids); + if (r.has_value() && (!best.has_value() || *r < *best)) { best = r; } + } + return best; +} +double RetrievalEval::hit_at_k(std::optional rank, int k) { return (rank.has_value() && *rank <= k) ? 1.0 : 0.0; } +double RetrievalEval::reciprocal_rank(std::optional rank) { return (rank.has_value() && *rank) ? 1.0 / *rank : 0.0; } +double RetrievalEval::ndcg_at_k(std::optional rank, int k) { + if (!rank.has_value() || *rank > k) { return 0.0; } + return 1.0 / std::log2(*rank + 1); +} +double RetrievalEval::context_precision(const std::vector& expected_ids, const std::vector& ranked_ids, int k) { + std::set wanted(expected_ids.begin(), expected_ids.end()); + if (wanted.empty()) { return 0.0; } + int hits = 0; + double total = 0.0; + for (size_t pos = 0; pos < ranked_ids.size() && pos < static_cast(k); ++pos) { + if (wanted.count(ranked_ids[pos])) { + hits += 1; + total += static_cast(hits) / static_cast(pos + 1); + } + } + return hits ? total / hits : 0.0; +} +std::vector RetrievalEval::tokens(const std::string& text, int min_len) { + std::vector seen; + for (const auto& t : alnum_tokens(text)) { + if (static_cast(py::len(t)) >= min_len && std::find(seen.begin(), seen.end(), t) == seen.end()) { seen.push_back(t); } + } + return seen; +} + +static int64_t doc_freq(StorePort& st, const std::string& profile, const std::string& token) { + json::Value params = json::Value::array(); + params.push("{title body} : \"" + token + "\""); + params.push(profile); + json::Value v; + try { + json::Value rows = st.sql_rows("SELECT COUNT(DISTINCT item) FROM fts_docs WHERE fts_docs MATCH ? AND profile = ? AND kind = 'record'", params); + if (rows.size() == 0) { return 0; } + v = rows.at(0).at(0); + } catch (const std::exception&) { + return 0; + } + return int_or0(v); +} + +json::Value RetrievalEval::goldens_from_vault(StorePort& store, const std::string& profile, int max_records) { + json::Value params = json::Value::array(); + params.push(profile); + params.push(max_records); + json::Value rows = store.sql_rows("SELECT id, title, content FROM records WHERE profile = ? ORDER BY id LIMIT ?", params); + json::Value goldens = json::Value::array(); + auto add = [&](const std::string& query, const json::Value& record_id, const char* category) { + json::Value g = json::Value::object(); + g.set("query", query); + json::Value exp = json::Value::array(); + exp.push(record_id); + g.set("expected", exp); + g.set("category", category); + g.set("source", record_id); + goldens.push(g); + }; + for (size_t pos = 0; pos < rows.size(); ++pos) { + const json::Value& row = rows.at(pos); + const json::Value& record_id = row.at(0); + std::string title = py::str_or_empty(row.at(1)); + std::string content = py::str_or_empty(row.at(2)); + std::vector words = tokens(title + "\n" + content); + if (words.empty()) { continue; } + struct Key { int64_t df; long neg_len; size_t idx; std::string token; }; + std::vector ranked; + for (size_t i = 0; i < words.size() && i < 24; ++i) { + ranked.push_back({doc_freq(store, profile, words[i]), -static_cast(py::len(words[i])), i, words[i]}); + } + std::sort(ranked.begin(), ranked.end(), [](const Key& a, const Key& b) { + if (a.df != b.df) { return a.df < b.df; } + if (a.neg_len != b.neg_len) { return a.neg_len < b.neg_len; } + if (a.idx != b.idx) { return a.idx < b.idx; } + return a.token < b.token; + }); + std::vector rare; + for (auto& k : ranked) { rare.push_back(k.token); } + add(rare[0], record_id, "exact"); + if (rare.size() >= 2) { + std::vector first3(rare.begin(), rare.begin() + static_cast(std::min(3, rare.size()))); + add(py::join(first3, " "), record_id, "phrase"); + } + std::string longest = words[0]; + for (const auto& w : words) { if (py::len(w) > py::len(longest)) { longest = w; } } + size_t cut = std::max(EVAL_PREFIX_MIN, py::len(longest) / 2); + if (cut < py::len(longest)) { add(py::head(longest, static_cast(cut)), record_id, "prefix"); } + if (rare.size() >= 2 && rows.size() > 1) { + const json::Value& other = rows.at((pos + 1) % rows.size()); + std::vector other_words = tokens(py::str_or_empty(other.at(1)) + "\n" + py::str_or_empty(other.at(2))); + std::set own(words.begin(), words.end()); + std::vector foreign; + for (const auto& t : other_words) { if (!own.count(t)) { foreign.push_back(t); } } + if (!foreign.empty()) { + add(rare[0] + " " + rare[1] + " " + foreign[0] + "zz", record_id, "noisy"); + } + } + if (goldens.size() >= static_cast(EVAL_MAX_GOLDENS)) { break; } + } + if (goldens.size() > static_cast(EVAL_MAX_GOLDENS)) { + goldens.items().resize(static_cast(EVAL_MAX_GOLDENS)); + } + return goldens; +} + +static json::Value mean_of(const std::vector& rows) { + json::Value m = json::Value::object(); + if (rows.empty()) { + m.set("hit", 0.0); m.set("mrr", 0.0); m.set("ndcg", 0.0); m.set("cp", 0.0); + return m; + } + for (const char* key : {"hit", "mrr", "ndcg", "cp"}) { + double sum = 0.0; + for (const auto& r : rows) { sum += r.get(key).as_double(); } + m.set(key, round4(sum / static_cast(rows.size()))); + } + return m; +} + +json::Value RetrievalEval::score(StorePort& store, const json::Value& goldens, const std::string& profile, + const std::vector& kinds, int k, int fetch) { + std::vector rows; + for (const auto& golden : goldens.items()) { + json::Value hits; + try { + hits = store.fts_search(py::str_or_empty(golden.get("query")), kinds, profile, fetch); + } catch (const std::exception&) { + hits = json::Value::array(); + } + std::vector ranked; + for (const auto& h : hits.items()) { ranked.push_back(sv(h.get("item"))); } + std::vector expected; + for (const auto& e : golden.get("expected").items()) { expected.push_back(sv(e)); } + auto rank = best_rank(expected, ranked); + json::Value row = json::Value::object(); + row.set("query", golden.get("query")); + row.set("category", golden.get("category")); + row.set("expected", golden.get("expected").truthy() ? golden.get("expected") : json::Value::array()); + row.set("rank", rank.has_value() ? json::Value(*rank) : json::Value()); + row.set("hit", hit_at_k(rank, k)); + row.set("mrr", reciprocal_rank(rank)); + row.set("ndcg", ndcg_at_k(rank, k)); + row.set("cp", context_precision(expected, ranked, k)); + rows.push_back(row); + } + json::Value result = json::Value::object(); + result.set("k", k); + result.set("goldens", static_cast(rows.size())); + result.set("overall", mean_of(rows)); + json::Value by_category = json::Value::object(); + std::vector order; + std::map> groups; + for (const auto& r : rows) { + std::string cat = sv(r.get("category")); + if (!groups.count(cat)) { order.push_back(cat); } + groups[cat].push_back(r); + } + for (const auto& cat : order) { + json::Value e = json::Value::object(); + e.set("goldens", static_cast(groups[cat].size())); + e.set("mean", mean_of(groups[cat])); + by_category.set(cat, e); + } + result.set("by_category", by_category); + json::Value misses = json::Value::array(); + for (const auto& r : rows) { + if (r.get("rank").is_null() && misses.size() < 10) { + json::Value m = json::Value::object(); + m.set("query", r.get("query")); + m.set("category", r.get("category")); + misses.push(m); + } + } + result.set("misses", misses); + return result; +} + +static std::optional>> parse_search_event_simple(const std::string& text) { + // dict(part.split("=", 1) for part in text.split("\t") if "=" in part) + std::map parts; + for (const auto& part : py::split(text, "\t")) { + size_t eq = part.find('='); + if (eq == std::string::npos) { continue; } + parts[part.substr(0, eq)] = part.substr(eq + 1); + } + std::string query = py::strip(parts.count("q") ? parts["q"] : ""); + std::string raw = py::strip(parts.count("hits") ? parts["hits"] : ""); + std::vector hits; + for (const auto& piece : py::split(raw, ",")) { + if (piece.empty()) { continue; } + hits.push_back(py::startswith(piece, "record:") ? piece.substr(piece.find(':') + 1) : piece); + } + if (query.empty()) { return std::nullopt; } + return std::make_pair(query, hits); +} + +json::Value RetrievalEval::implicit_pairs(StorePort& store, const std::string& profile, int limit) { + std::string text_col = store.seal_enabled() ? "tai_dec(text)" : "text"; + json::Value params = json::Value::array(); + params.push(profile); + params.push(limit); + json::Value rows = rows_of(store, "SELECT role, kind, " + text_col + " FROM events WHERE profile = ? ORDER BY id DESC LIMIT ?", params); + std::vector rv(rows.items().begin(), rows.items().end()); + std::reverse(rv.begin(), rv.end()); + struct Search { size_t pos; std::string query; std::vector hits; }; + std::vector searches; + for (size_t pos = 0; pos < rv.size(); ++pos) { + if (sv(rv[pos].at(0)) == "tool" && sv(rv[pos].at(1)) == "search-query") { + auto parsed = parse_search_event_simple(py::str_or_empty(rv[pos].at(2))); + if (parsed.has_value()) { searches.push_back({pos, parsed->first, parsed->second}); } + } + } + json::Value pairs = json::Value::array(); + std::vector zero_hit; + int64_t read_follow = 0; + for (size_t idx = 0; idx < searches.size(); ++idx) { + const Search& s = searches[idx]; + if (s.hits.empty()) { zero_hit.push_back(s.query); } + size_t end = idx + 1 < searches.size() ? searches[idx + 1].pos : rv.size(); + size_t stop = std::min(end, s.pos + 21); + for (size_t p = s.pos + 1; p < stop; ++p) { + if (sv(rv[p].at(0)) == "tool" && sv(rv[p].at(1)) == "record-read" && !py::strip(py::str_or_empty(rv[p].at(2))).empty()) { + read_follow += 1; + std::string read_id = py::strip(py::str_or_empty(rv[p].at(2))); + auto rank = rank_of(read_id, s.hits); + json::Value pr = json::Value::object(); + pr.set("query", s.query); + pr.set("read", read_id); + pr.set("rank", rank.has_value() ? json::Value(*rank) : json::Value()); + pr.set("shown", static_cast(s.hits.size())); + pairs.push(pr); + break; + } + } + if (pairs.size() >= static_cast(EVAL_MAX_PAIRS)) { break; } + } + json::Value out = json::Value::object(); + out.set("searches", static_cast(searches.size())); + out.set("read_follow", read_follow); + if (pairs.size() > static_cast(EVAL_MAX_PAIRS)) { pairs.items().resize(static_cast(EVAL_MAX_PAIRS)); } + out.set("pairs", pairs); + json::Value zh = json::Value::array(); + for (size_t i = 0; i < zero_hit.size() && i < 10; ++i) { zh.push(zero_hit[i]); } + out.set("zero_hit", zh); + out.set("zero_hit_count", static_cast(zero_hit.size())); + return out; +} + +std::string RetrievalEval::format_report(const json::Value& result, const json::Value& pairs) { + auto num = [](const json::Value& m, const char* k) { return m.get(k).truthy() ? m.get(k).as_double() : 0.0; }; + auto geti = [](const json::Value& m, const char* k, int64_t d) { return m.has(k) ? m.get(k).as_int() : d; }; + std::vector lines = {"retrieval eval: " + std::to_string(geti(result, "goldens", 0)) + " goldens at k=" + + std::to_string(geti(result, "k", EVAL_K))}; + const json::Value& overall = result.get("overall"); + lines.push_back("overall: hit=" + fmt("%.3f", num(overall, "hit")) + " mrr=" + fmt("%.3f", num(overall, "mrr")) + " ndcg=" + + fmt("%.3f", num(overall, "ndcg")) + " cp=" + fmt("%.3f", num(overall, "cp"))); + std::vector cats; + for (const auto& kv : result.get("by_category").members()) { cats.push_back(kv.first); } + std::sort(cats.begin(), cats.end()); + for (const auto& cat : cats) { + const json::Value& entry = result.get("by_category").get(cat); + const json::Value& mean = entry.get("mean"); + lines.push_back(cat + ": n=" + std::to_string(geti(entry, "goldens", 0)) + " hit=" + fmt("%.3f", num(mean, "hit")) + " mrr=" + + fmt("%.3f", num(mean, "mrr")) + " ndcg=" + fmt("%.3f", num(mean, "ndcg")) + " cp=" + fmt("%.3f", num(mean, "cp"))); + } + for (const auto& miss : result.get("misses").items()) { + lines.push_back("miss [" + sv(miss.get("category")) + "]: " + py::head(py::str_or_empty(miss.get("query")), 80)); + } + lines.push_back("implicit: " + std::to_string(geti(pairs, "searches", 0)) + " searches, " + std::to_string(geti(pairs, "read_follow", 0)) + + " followed by a read, " + std::to_string(geti(pairs, "zero_hit_count", 0)) + " zero-hit"); + for (const auto& q : pairs.get("zero_hit").items()) { lines.push_back("zero-hit: " + py::head(sv(q), 80)); } + return py::join(lines, "\n"); +} + +// ------------------------------------------------------------------ RagStats +namespace { +struct Parsed { + std::vector> kinds; // insertion order + std::vector ranked, shown_records, shown_chunks; +}; +void bump(std::vector>& v, const std::string& k) { + for (auto& kv : v) { if (kv.first == k) { ++kv.second; return; } } + v.emplace_back(k, 1); +} +Parsed parse_hits(const std::string& raw) { + Parsed p; + for (const auto& piece0 : py::split(raw, ",")) { + std::string piece = py::strip(piece0); + if (piece.empty()) { continue; } + size_t colon = piece.find(':'); + std::string kind = colon == std::string::npos ? piece : piece.substr(0, colon); + std::string rest = colon == std::string::npos ? "" : piece.substr(colon + 1); + bool known = colon != std::string::npos && (kind == "record" || kind == "event" || kind == "audit" || kind == "chunk" || kind == "log"); + if (!known) { bump(p.kinds, "other"); p.ranked.push_back(piece); continue; } + bump(p.kinds, kind); + if (kind == "record") { p.ranked.push_back(rest); p.shown_records.push_back(rest); } + else if (kind == "chunk") { p.ranked.push_back(piece); p.shown_chunks.push_back(!rest.empty() ? rest : piece); } + else { p.ranked.push_back(piece); } + } + return p; +} + +struct Counter { + std::vector> items; // insertion order + void add(const std::string& k) { bump(items, k); } + size_t size() const { return items.size(); } + int64_t get(const std::string& k) const { for (const auto& kv : items) { if (kv.first == k) { return kv.second; } } return 0; } + std::vector> most_common(size_t n) const { + auto v = items; + std::stable_sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); + if (v.size() > n) { v.resize(n); } + return v; + } +}; + +json::Value pair_list(const std::vector>& v) { + json::Value out = json::Value::array(); + for (const auto& kv : v) { + json::Value p = json::Value::array(); + p.push(kv.first); + p.push(kv.second); + out.push(p); + } + return out; +} +} // namespace + +json::Value RagStats::collect(StorePort& store, const std::string& profile, bool include_eval) { + json::Value stats = json::Value::object(); + auto P = [&](std::initializer_list xs) { json::Value a = json::Value::array(); for (const auto& x : xs) { a.push(x); } return a; }; + stats.set("profile", profile); + stats.set("fts_ok", store.fts_ok()); + stats.set("window", RAG_STATS_EVENTS); + stats.set("events_total", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ?", P({profile})))); + json::Value row = rows_of(store, "SELECT COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ?", P({profile})); + if (row.size() > 0) { + stats.set("records_total", int_or0(row.at(0).at(0))); + stats.set("records_bytes", int_or0(row.at(0).at(1))); + stats.set("records_reads", int_or0(row.at(0).at(2))); + } else { + stats.set("records_total", 0); stats.set("records_bytes", 0); stats.set("records_reads", 0); + } + auto proj = [&](const json::Value& rows, std::initializer_list cols, std::initializer_list orzero) { + json::Value out = json::Value::array(); + for (const auto& r : rows.items()) { + json::Value t = json::Value::array(); + auto oz = orzero.begin(); + for (int c : cols) { + const json::Value& v = r.at(static_cast(c)); + t.push((*oz) ? (v.truthy() ? v : json::Value(0)) : v); + ++oz; + } + out.push(t); + } + return out; + }; + // records_by_kind: (kind, count, size or 0, reads or 0) + stats.set("records_by_kind", proj(rows_of(store, "SELECT kind, COUNT(*), COALESCE(SUM(size), 0), COALESCE(SUM(reads), 0) FROM records WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", P({profile})), + {0, 1, 2, 3}, {false, false, true, true})); + // the title column is `r[2] or ""` + auto rec_rows = [&](const std::string& sql) { + json::Value rows = rows_of(store, sql, P({profile, RAG_STATS_TOP})); + json::Value out = json::Value::array(); + for (const auto& r : rows.items()) { + json::Value t = json::Value::array(); + t.push(r.at(0)); t.push(r.at(1)); t.push(r.at(2).truthy() ? r.at(2) : json::Value("")); t.push(r.at(3).truthy() ? r.at(3) : json::Value(0)); t.push(r.at(4).truthy() ? r.at(4) : json::Value(0)); + out.push(t); + } + return out; + }; + stats.set("top_reads", rec_rows("SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY reads DESC, updated DESC LIMIT ?")); + stats.set("top_size", rec_rows("SELECT id, kind, title, reads, size FROM records WHERE profile = ? ORDER BY size DESC LIMIT ?")); + stats.set("zero_read_records", int_or0(one_of(store, "SELECT COUNT(*) FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0", P({profile})))); + { + json::Value rows = rows_of(store, "SELECT id FROM records WHERE profile = ? AND COALESCE(reads, 0) = 0 ORDER BY updated DESC LIMIT ?", P({profile, RAG_STATS_SAMPLE})); + json::Value out = json::Value::array(); + for (const auto& r : rows.items()) { out.push(r.at(0)); } + stats.set("never_read_sample", out); + } + stats.set("chunks_total", int_or0(one_of(store, "SELECT COUNT(*) FROM chunks WHERE profile = ?", P({profile})))); + stats.set("chunks_by_parent", proj(rows_of(store, "SELECT parent_kind, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind ORDER BY COUNT(*) DESC", P({profile})), {0, 1}, {false, false})); + stats.set("chunk_parents", int_or0(one_of(store, "SELECT COUNT(DISTINCT parent_kind || ':' || parent_id) FROM chunks WHERE profile = ?", P({profile})))); + stats.set("top_chunk_parents", proj(rows_of(store, "SELECT parent_kind, parent_id, COUNT(*) FROM chunks WHERE profile = ? GROUP BY parent_kind, parent_id ORDER BY COUNT(*) DESC LIMIT ?", P({profile, RAG_STATS_TOP})), {0, 1, 2}, {false, false, false})); + stats.set("orphan_chunks", int_or0(one_of(store, "SELECT COUNT(*) FROM chunks WHERE profile = ? AND parent_kind = 'record' AND parent_id NOT IN (SELECT id FROM records WHERE profile = ?)", P({profile, profile})))); + json::Value tag_row = rows_of(store, "SELECT COUNT(*), COUNT(DISTINCT tag), COUNT(DISTINCT item) FROM tags WHERE profile = ?", P({profile})); + if (tag_row.size() > 0) { + stats.set("tags_rows", int_or0(tag_row.at(0).at(0))); + stats.set("tags_distinct", int_or0(tag_row.at(0).at(1))); + stats.set("tags_items", int_or0(tag_row.at(0).at(2))); + } else { + stats.set("tags_rows", 0); stats.set("tags_distinct", 0); stats.set("tags_items", 0); + } + { + json::Value top = json::Value::array(); + try { + for (const auto& tc : store.tag_counts("", RAG_STATS_TOP, profile)) { + json::Value t = json::Value::array(); + t.push(tc.first); t.push(tc.second); + top.push(t); + } + } catch (const std::exception&) { + top = json::Value::array(); + } + stats.set("top_tags", top); + } + stats.set("record_tag_rows", int_or0(one_of(store, "SELECT COUNT(*) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", P({profile})))); + stats.set("tagged_records", int_or0(one_of(store, "SELECT COUNT(DISTINCT item) FROM tags WHERE profile = ? AND item LIKE 'mem:%'", P({profile})))); + stats.set("only_record_tag", int_or0(one_of(store, "SELECT COUNT(*) FROM (SELECT item FROM tags WHERE profile = ? AND item LIKE 'mem:%' GROUP BY item HAVING COUNT(*) = 1 AND MAX(tag) = 'record')", P({profile})))); + stats.set("edges_total", int_or0(one_of(store, "SELECT COUNT(*) FROM edges WHERE profile = ?", P({profile})))); + stats.set("edges_by_relation", proj(rows_of(store, "SELECT relation, COUNT(*) FROM edges WHERE profile = ? GROUP BY relation ORDER BY COUNT(*) DESC LIMIT ?", P({profile, RAG_STATS_TOP})), {0, 1}, {false, false})); + stats.set("top_degree", proj(rows_of(store, "SELECT node, COUNT(*) FROM (SELECT src AS node FROM edges WHERE profile = ? UNION ALL SELECT dst AS node FROM edges WHERE profile = ?) GROUP BY node ORDER BY COUNT(*) DESC LIMIT ?", P({profile, profile, RAG_STATS_TOP})), {0, 1}, {false, false})); + stats.set("isolated_records", int_or0(one_of(store, "SELECT COUNT(*) FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?)", P({profile, profile, profile})))); + { + json::Value rows = rows_of(store, "SELECT id FROM records WHERE profile = ? AND id NOT IN (SELECT src FROM edges WHERE profile = ? UNION SELECT dst FROM edges WHERE profile = ?) ORDER BY updated DESC LIMIT ?", P({profile, profile, profile, RAG_STATS_SAMPLE})); + json::Value out = json::Value::array(); + for (const auto& r : rows.items()) { out.push(r.at(0)); } + stats.set("isolated_sample", out); + } + stats.set("fts_by_kind", store.fts_ok() + ? proj(rows_of(store, "SELECT kind, COUNT(*) FROM fts_docs WHERE profile = ? GROUP BY kind ORDER BY COUNT(*) DESC", P({profile})), {0, 1}, {false, false}) + : json::Value::array()); + stats.set("events_by_kind", proj(rows_of(store, "SELECT role, kind, COUNT(*) FROM events WHERE profile = ? GROUP BY role, kind ORDER BY COUNT(*) DESC LIMIT ?", P({profile, RAG_STATS_KINDS})), {0, 1, 2}, {false, false, false})); + json::Value span = rows_of(store, "SELECT MIN(ts), MAX(ts) FROM events WHERE profile = ?", P({profile})); + stats.set("events_first", span.size() ? span.at(0).at(0) : json::Value()); + stats.set("events_last", span.size() ? span.at(0).at(1) : json::Value()); + double now_e = py::now(); + std::string day_cut = py::utc_iso(now_e - 86400.0, false); + std::string week_cut = py::utc_iso(now_e - 7 * 86400.0, false); + stats.set("searches_total", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query'", P({profile})))); + stats.set("reads_total", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read'", P({profile})))); + stats.set("searches_24h", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", P({profile, day_cut})))); + stats.set("reads_24h", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", P({profile, day_cut})))); + stats.set("searches_7d", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'search-query' AND ts >= ?", P({profile, week_cut})))); + stats.set("reads_7d", int_or0(one_of(store, "SELECT COUNT(*) FROM events WHERE profile = ? AND role = 'tool' AND kind = 'record-read' AND ts >= ?", P({profile, week_cut})))); + std::string text_col = store.seal_enabled() ? "tai_dec(text)" : "text"; + json::Value wr = rows_of(store, "SELECT ts, role, kind, " + text_col + " FROM events WHERE profile = ? ORDER BY id DESC LIMIT ?", P({profile, RAG_STATS_EVENTS})); + std::vector window_rows(wr.items().begin(), wr.items().end()); + std::reverse(window_rows.begin(), window_rows.end()); + struct SearchItem { + size_t pos; std::string ts, query; int64_t limit; int64_t hits; + Parsed parsed; + }; + struct ReadItem { size_t pos; std::string id, ts; }; + std::vector searches; + std::vector reads; + for (size_t pos = 0; pos < window_rows.size(); ++pos) { + const json::Value& r = window_rows[pos]; + std::string role = sv(r.at(1)), kind = sv(r.at(2)); + if (role == "tool" && kind == "search-query") { + std::map parts; + for (const auto& part : py::split(py::str_or_empty(r.at(3)), "\t")) { + size_t eq = part.find('='); + if (eq != std::string::npos) { parts[part.substr(0, eq)] = part.substr(eq + 1); } + } + std::string query = py::strip(parts.count("q") ? parts["q"] : ""); + if (query.empty()) { continue; } + SearchItem s; + s.parsed = parse_hits(parts.count("hits") ? parts["hits"] : ""); + auto lim = py::parse_int(parts.count("limit") ? parts["limit"] : ""); + s.limit = lim.value_or(0); + s.hits = static_cast(s.parsed.ranked.size()); + s.pos = pos; s.ts = sv(r.at(0)); s.query = query; + searches.push_back(s); + } else if (role == "tool" && kind == "record-read" && !py::strip(py::str_or_empty(r.at(3))).empty()) { + reads.push_back({pos, py::head(py::strip(py::str_or_empty(r.at(3))), 200), sv(r.at(0))}); + } + } + stats.set("window_events", static_cast(window_rows.size())); + stats.set("window_searches", static_cast(searches.size())); + stats.set("window_reads", static_cast(reads.size())); + std::vector> hits_by_kind; + Counter shown_counter, chunk_counter, query_counter; + std::map> query_hits; + std::vector query_order; + int64_t query_len = 0, query_terms = 0; + std::vector zero_hit; + int64_t d0 = 0, d1 = 0, d25 = 0, d6 = 0; + for (const auto& item : searches) { + query_counter.add(item.query); + if (!query_hits.count(item.query)) { query_order.push_back(item.query); } + query_hits[item.query].push_back(item.hits); + query_len += static_cast(py::len(item.query)); + query_terms += static_cast(py::split_ws(item.query).size()); + for (const auto& kv : item.parsed.kinds) { + bool found = false; + for (auto& h : hits_by_kind) { if (h.first == kv.first) { h.second += kv.second; found = true; } } + if (!found) { hits_by_kind.push_back(kv); } + } + for (const auto& id : item.parsed.shown_records) { shown_counter.add(id); } + for (const auto& id : item.parsed.shown_chunks) { chunk_counter.add(id); } + if (item.hits == 0) { d0++; zero_hit.push_back(item.query); } + else if (item.hits == 1) { d1++; } + else if (item.hits <= 5) { d25++; } + else { d6++; } + } + std::stable_sort(hits_by_kind.begin(), hits_by_kind.end(), [](const auto& a, const auto& b) { + if (a.second != b.second) { return a.second > b.second; } + return a.first < b.first; + }); + stats.set("hits_by_kind", pair_list(hits_by_kind)); + json::Value dist = json::Value::object(); + dist.set("0", d0); dist.set("1", d1); dist.set("2-5", d25); dist.set("6+", d6); + stats.set("hits_dist", dist); + stats.set("zero_hit_window", d0); + { + json::Value zs = json::Value::array(); + for (size_t i = 0; i < zero_hit.size() && i < static_cast(RAG_STATS_SAMPLE); ++i) { zs.push(zero_hit[i]); } + stats.set("zero_hit_sample", zs); + } + int64_t total_hits = 0; + for (const auto& s : searches) { total_hits += s.hits; } + double n_search = static_cast(searches.size()); + auto round_to = [](double x, int digits) { char b[64]; std::snprintf(b, sizeof b, "%.*f", digits, x); return std::strtod(b, nullptr); }; + stats.set("avg_hits", searches.empty() ? 0.0 : round_to(static_cast(total_hits) / n_search, 2)); + stats.set("avg_query_chars", searches.empty() ? 0.0 : round_to(static_cast(query_len) / n_search, 1)); + stats.set("avg_query_terms", searches.empty() ? 0.0 : round_to(static_cast(query_terms) / n_search, 1)); + int64_t repeated = 0; + for (const auto& kv : query_counter.items) { if (kv.second > 1) { ++repeated; } } + stats.set("repeated_queries", repeated); + stats.set("queries_distinct", static_cast(query_counter.size())); + { + json::Value top = json::Value::array(); + for (const auto& kv : query_counter.most_common(RAG_STATS_QUERIES)) { + const auto& shots = query_hits[kv.first]; + double sum = 0; + for (auto s : shots) { sum += static_cast(s); } + json::Value t = json::Value::array(); + t.push(kv.first); t.push(kv.second); t.push(shots.empty() ? 0.0 : round_to(sum / static_cast(shots.size()), 1)); + top.push(t); + } + stats.set("top_queries", top); + } + stats.set("top_shown", pair_list(shown_counter.most_common(RAG_STATS_TOP))); + stats.set("top_shown_chunks", pair_list(chunk_counter.most_common(RAG_STATS_TOP))); + stats.set("shown_distinct", static_cast(shown_counter.size())); + Counter read_counter; + for (const auto& r : reads) { read_counter.add(r.id); } + stats.set("top_read_ids", pair_list(read_counter.most_common(RAG_STATS_TOP))); + stats.set("read_distinct", static_cast(read_counter.size())); + int64_t rd1 = 0, rd2 = 0, rd3 = 0, rd45 = 0, rd6 = 0, rdun = 0; + int64_t read_follow = 0; + for (size_t idx = 0; idx < searches.size(); ++idx) { + const SearchItem& item = searches[idx]; + size_t end = idx + 1 < searches.size() ? searches[idx + 1].pos : window_rows.size(); + size_t stop = std::min(end, item.pos + 21); + for (size_t p = item.pos + 1; p < stop; ++p) { + const json::Value& r = window_rows[p]; + if (sv(r.at(1)) == "tool" && sv(r.at(2)) == "record-read" && !py::strip(py::str_or_empty(r.at(3))).empty()) { + read_follow += 1; + auto rank = RetrievalEval::rank_of(py::head(py::strip(py::str_or_empty(r.at(3))), 200), item.parsed.ranked); + if (!rank.has_value()) { rdun++; } + else if (*rank == 1) { rd1++; } + else if (*rank == 2) { rd2++; } + else if (*rank == 3) { rd3++; } + else if (*rank <= 5) { rd45++; } + else { rd6++; } + break; + } + } + } + stats.set("read_follow", read_follow); + json::Value rank_dist = json::Value::object(); + rank_dist.set("1", rd1); rank_dist.set("2", rd2); rank_dist.set("3", rd3); rank_dist.set("4-5", rd45); rank_dist.set("6+", rd6); rank_dist.set("unshown", rdun); + stats.set("rank_dist", rank_dist); + std::set shown_set, read_set; + for (const auto& kv : shown_counter.items) { shown_set.insert(kv.first); } + for (const auto& kv : read_counter.items) { read_set.insert(kv.first); } + std::vector direct, never; + for (const auto& r : read_set) { if (!shown_set.count(r)) { direct.push_back(r); } } + for (const auto& s : shown_set) { if (!read_set.count(s)) { never.push_back(s); } } + stats.set("direct_reads", static_cast(direct.size())); + { + json::Value d = json::Value::array(); + for (size_t i = 0; i < direct.size() && i < static_cast(RAG_STATS_SAMPLE); ++i) { d.push(direct[i]); } + stats.set("direct_sample", d); + } + stats.set("shown_never_read", static_cast(never.size())); + { + json::Value d = json::Value::array(); + for (size_t i = 0; i < never.size() && i < static_cast(RAG_STATS_SAMPLE); ++i) { d.push(never[i]); } + stats.set("shown_never_sample", d); + } + struct Eff { std::string id; int64_t shown, read, total; }; + std::vector eff; + std::set all_ids = shown_set; + all_ids.insert(read_set.begin(), read_set.end()); + for (const auto& id : all_ids) { + int64_t sh = shown_counter.get(id), rd = read_counter.get(id); + eff.push_back({id, sh, rd, sh + rd}); + } + std::sort(eff.begin(), eff.end(), [](const Eff& a, const Eff& b) { + if (a.total != b.total) { return a.total > b.total; } + if (a.read != b.read) { return a.read > b.read; } + return a.id < b.id; + }); + if (eff.size() > static_cast(RAG_STATS_TOP)) { eff.resize(static_cast(RAG_STATS_TOP)); } + json::Value eff_v = json::Value::array(); + for (const auto& e : eff) { + json::Value t = json::Value::array(); + t.push(e.id); t.push(e.shown); t.push(e.read); t.push(e.total); + eff_v.push(t); + } + stats.set("effectiveness", eff_v); + json::Value meta = json::Value::object(); + if (!eff.empty()) { + std::string marks; + json::Value params = json::Value::array(); + params.push(profile); + for (size_t i = 0; i < eff.size(); ++i) { + marks += (i ? ", ?" : "?"); + params.push(eff[i].id); + } + json::Value rows = rows_of(store, "SELECT id, kind, title, reads, size FROM records WHERE profile = ? AND id IN (" + marks + ")", params); + for (const auto& r : rows.items()) { + json::Value m = json::Value::object(); + m.set("kind", r.at(1)); + m.set("title", r.at(2).truthy() ? r.at(2) : json::Value("")); + m.set("reads_col", r.at(3).truthy() ? r.at(3) : json::Value(0)); + m.set("size", r.at(4).truthy() ? r.at(4) : json::Value(0)); + meta.set(sv(r.at(0)), m); + } + } + stats.set("effectiveness_meta", meta); + stats.set("eval", json::Value()); + stats.set("eval_error", ""); + if (include_eval) { + try { + json::Value goldens = RetrievalEval::goldens_from_vault(store, profile); + json::Value result = RetrievalEval::score(store, goldens, profile); + json::Value pairs = RetrievalEval::implicit_pairs(store, profile); + json::Value ev = json::Value::object(); + ev.set("goldens", result.get("goldens").truthy() ? result.get("goldens") : json::Value(0)); + ev.set("k", result.has("k") ? result.get("k") : json::Value(EVAL_K)); + ev.set("overall", result.get("overall").truthy() ? result.get("overall") : json::Value::object()); + ev.set("by_category", result.get("by_category").truthy() ? result.get("by_category") : json::Value::object()); + json::Value misses = json::Value::array(); + for (size_t i = 0; i < result.get("misses").size() && i < static_cast(RAG_STATS_SAMPLE); ++i) { misses.push(result.get("misses").at(i)); } + ev.set("misses", misses); + ev.set("searches", pairs.has("searches") ? pairs.get("searches") : json::Value(0)); + ev.set("read_follow", pairs.has("read_follow") ? pairs.get("read_follow") : json::Value(0)); + ev.set("zero_hit_count", pairs.has("zero_hit_count") ? pairs.get("zero_hit_count") : json::Value(0)); + json::Value zh = json::Value::array(); + for (size_t i = 0; i < pairs.get("zero_hit").size() && i < static_cast(RAG_STATS_SAMPLE); ++i) { zh.push(pairs.get("zero_hit").at(i)); } + ev.set("zero_hit", zh); + stats.set("eval", ev); + } catch (const std::exception& exc) { + stats.set("eval_error", llm::short_error(exc)); + } + } + return stats; +} + +std::string RagStats::format_report(const json::Value& stats) { + auto gi = [&](const char* k) -> int64_t { return stats.get(k).truthy() ? stats.get(k).as_int() : 0; }; + auto gd = [&](const char* k) -> double { return stats.get(k).truthy() ? stats.get(k).as_double() : 0.0; }; + auto pct = [](int64_t part, int64_t whole) -> std::string { + if (!whole) { return "n/a"; } + return fmt("%.1f", 100.0 * static_cast(part) / static_cast(whole)) + "%"; + }; + auto I = [](int64_t v) { return std::to_string(v); }; + std::vector lines; + lines.push_back("rag stats: profile " + sv(stats.get("profile")) + " (fts " + (stats.get("fts_ok").truthy() ? "ok" : "OFF") + ", window newest " + + I(gi("window_events")) + " of " + I(gi("events_total")) + " events)"); + lines.push_back("vault: " + I(gi("records_total")) + " records (" + I(gi("records_bytes")) + " bytes, " + I(gi("records_reads")) + " reads col) | " + + I(gi("chunks_total")) + " chunks (" + I(gi("chunk_parents")) + " parents) | " + I(gi("tags_rows")) + " tag rows (" + + I(gi("tags_distinct")) + " tags on " + I(gi("tags_items")) + " items) | " + I(gi("edges_total")) + " edges | " + I(gi("events_total")) + " events"); + if (stats.get("events_first").truthy() || stats.get("events_last").truthy()) { + lines.push_back("events span: " + (stats.get("events_first").truthy() ? sv(stats.get("events_first")) : "?") + " .. " + + (stats.get("events_last").truthy() ? sv(stats.get("events_last")) : "?")); + } + for (const auto& r : stats.get("records_by_kind").items()) { + lines.push_back("records [" + sv(r.at(0)) + "]: n=" + sv(r.at(1)) + " bytes=" + sv(r.at(2)) + " reads=" + sv(r.at(3))); + } + auto squash = [](const std::string& t, long n) { return py::head(py::join(py::split_ws(t), " "), n); }; + for (const auto& r : stats.get("top_reads").items()) { + lines.push_back("top-read: " + sv(r.at(3)) + " reads " + sv(r.at(0)) + " [" + sv(r.at(1)) + "] " + squash(sv(r.at(2)), 60) + " (" + sv(r.at(4)) + " bytes)"); + } + for (const auto& r : stats.get("top_size").items()) { + lines.push_back("top-size: " + sv(r.at(4)) + " bytes " + sv(r.at(0)) + " [" + sv(r.at(1)) + "] " + squash(sv(r.at(2)), 60) + " (" + sv(r.at(3)) + " reads)"); + } + lines.push_back("unread records: " + I(gi("zero_read_records")) + " of " + I(gi("records_total")) + " (" + pct(gi("zero_read_records"), gi("records_total")) + ")"); + for (const auto& id : stats.get("never_read_sample").items()) { lines.push_back("never-read: " + sv(id)); } + for (const auto& r : stats.get("chunks_by_parent").items()) { lines.push_back("chunks [" + sv(r.at(0)) + "]: n=" + sv(r.at(1))); } + int64_t parents = gi("chunk_parents"); + lines.push_back("chunks per parent: " + fmt("%.1f", parents ? static_cast(gi("chunks_total")) / static_cast(parents) : 0.0) + " avg, " + + I(gi("orphan_chunks")) + " orphan record-chunks"); + for (const auto& r : stats.get("top_chunk_parents").items()) { + lines.push_back("chunked: " + sv(r.at(2)) + " chunks " + sv(r.at(0)) + ":" + py::head(py::str_or_empty(r.at(1)), 60)); + } + for (const auto& r : stats.get("top_tags").items()) { lines.push_back("tag: " + sv(r.at(1)) + " " + sv(r.at(0))); } + int64_t records_total = gi("records_total"); + lines.push_back("tag coverage: " + I(gi("tagged_records")) + " of " + I(records_total) + " records tagged, " + I(gi("only_record_tag")) + + " with only the record tag, " + fmt("%.1f", records_total ? static_cast(gi("record_tag_rows")) / static_cast(records_total) : 0.0) + " tags per record"); + for (const auto& r : stats.get("edges_by_relation").items()) { lines.push_back("edge [" + sv(r.at(0)) + "]: n=" + sv(r.at(1))); } + for (const auto& r : stats.get("top_degree").items()) { lines.push_back("linked: degree " + sv(r.at(1)) + " " + py::head(py::str_or_empty(r.at(0)), 80)); } + lines.push_back("isolated records: " + I(gi("isolated_records")) + " of " + I(records_total) + " (" + pct(gi("isolated_records"), records_total) + ")"); + for (const auto& id : stats.get("isolated_sample").items()) { lines.push_back("isolated: " + sv(id)); } + for (const auto& r : stats.get("fts_by_kind").items()) { lines.push_back("fts [" + sv(r.at(0)) + "]: n=" + sv(r.at(1))); } + for (const auto& r : stats.get("events_by_kind").items()) { lines.push_back("events [" + sv(r.at(0)) + "/" + sv(r.at(1)) + "]: n=" + sv(r.at(2))); } + lines.push_back("searches: total=" + I(gi("searches_total")) + " window=" + I(gi("window_searches")) + " zero-hit=" + I(gi("zero_hit_window")) + " (" + + pct(gi("zero_hit_window"), gi("window_searches")) + ") avg-hits=" + fmt("%.2f", gd("avg_hits")) + " avg-q=" + fmt("%.1f", gd("avg_query_chars")) + + " chars/" + fmt("%.1f", gd("avg_query_terms")) + " terms 24h=" + I(gi("searches_24h")) + " 7d=" + I(gi("searches_7d"))); + const json::Value& dist = stats.get("hits_dist"); + auto di = [&](const json::Value& d, const char* k) { return d.has(k) ? d.get(k).as_int() : 0; }; + lines.push_back("hits dist: 0=" + I(di(dist, "0")) + " 1=" + I(di(dist, "1")) + " 2-5=" + I(di(dist, "2-5")) + " 6+=" + I(di(dist, "6+"))); + for (const auto& r : stats.get("hits_by_kind").items()) { lines.push_back("hits [" + sv(r.at(0)) + "]: n=" + sv(r.at(1))); } + lines.push_back("queries: " + I(gi("queries_distinct")) + " distinct in window, " + I(gi("repeated_queries")) + " repeated"); + for (const auto& r : stats.get("top_queries").items()) { + lines.push_back("query: " + sv(r.at(1)) + "x avg-hits=" + fmt("%.1f", r.at(2).as_double()) + " " + squash(sv(r.at(0)), 80)); + } + for (const auto& q : stats.get("zero_hit_sample").items()) { lines.push_back("zero-hit: " + squash(sv(q), 80)); } + lines.push_back("reads: total=" + I(gi("reads_total")) + " window=" + I(gi("window_reads")) + " distinct=" + I(gi("read_distinct")) + " 24h=" + I(gi("reads_24h")) + " 7d=" + I(gi("reads_7d"))); + lines.push_back("read-follow: " + I(gi("read_follow")) + " of " + I(gi("window_searches")) + " window searches (" + pct(gi("read_follow"), gi("window_searches")) + ")"); + const json::Value& rank = stats.get("rank_dist"); + lines.push_back("read rank: 1=" + I(di(rank, "1")) + " 2=" + I(di(rank, "2")) + " 3=" + I(di(rank, "3")) + " 4-5=" + I(di(rank, "4-5")) + " 6+=" + I(di(rank, "6+")) + " unshown=" + I(di(rank, "unshown"))); + for (const auto& r : stats.get("top_read_ids").items()) { lines.push_back("read-id: " + sv(r.at(1)) + "x " + sv(r.at(0))); } + for (const auto& r : stats.get("top_shown").items()) { lines.push_back("shown: " + sv(r.at(1)) + "x " + sv(r.at(0))); } + for (const auto& r : stats.get("top_shown_chunks").items()) { lines.push_back("shown-chunk: " + sv(r.at(1)) + "x " + py::head(py::str_or_empty(r.at(0)), 80)); } + lines.push_back("coverage: " + I(gi("shown_distinct")) + " distinct shown, " + I(gi("shown_never_read")) + " shown-never-read, " + I(gi("direct_reads")) + " direct-read-never-shown"); + for (const auto& id : stats.get("shown_never_sample").items()) { lines.push_back("shown-never-read: " + sv(id)); } + for (const auto& id : stats.get("direct_sample").items()) { lines.push_back("direct-read: " + sv(id)); } + lines.push_back("context uses per record (shown + read events in window):"); + const json::Value& meta = stats.get("effectiveness_meta"); + for (const auto& r : stats.get("effectiveness").items()) { + std::string id = sv(r.at(0)); + int64_t shown = r.at(1).as_int(), read = r.at(2).as_int(), total = r.at(3).as_int(); + const json::Value& info = meta.get(id); + std::string through = shown ? sv(r.at(2)) + "/" + sv(r.at(1)) : I(read) + "/0"; + lines.push_back("use: total=" + I(total) + " shown=" + I(shown) + " read=" + I(read) + " through=" + through + " col-reads=" + + (info.has("reads_col") ? sv(info.get("reads_col")) : "0") + " " + id + " [" + (info.has("kind") ? sv(info.get("kind")) : "?") + "] " + + squash(info.has("title") ? sv(info.get("title")) : "", 60)); + } + const json::Value& ev = stats.get("eval"); + if (!ev.is_null()) { + const json::Value& overall = ev.get("overall"); + auto num = [](const json::Value& m, const char* k) { return m.get(k).truthy() ? m.get(k).as_double() : 0.0; }; + lines.push_back("eval: " + I(ev.has("goldens") ? ev.get("goldens").as_int() : 0) + " goldens at k=" + I(ev.has("k") ? ev.get("k").as_int() : EVAL_K) + " hit=" + + fmt("%.3f", num(overall, "hit")) + " mrr=" + fmt("%.3f", num(overall, "mrr")) + " ndcg=" + fmt("%.3f", num(overall, "ndcg")) + " cp=" + fmt("%.3f", num(overall, "cp"))); + std::vector cats; + for (const auto& kv : ev.get("by_category").members()) { cats.push_back(kv.first); } + std::sort(cats.begin(), cats.end()); + for (const auto& cat : cats) { + const json::Value& entry = ev.get("by_category").get(cat); + const json::Value& mean = entry.get("mean"); + lines.push_back("eval [" + cat + "]: n=" + I(entry.has("goldens") ? entry.get("goldens").as_int() : 0) + " hit=" + fmt("%.3f", num(mean, "hit")) + " mrr=" + + fmt("%.3f", num(mean, "mrr")) + " ndcg=" + fmt("%.3f", num(mean, "ndcg")) + " cp=" + fmt("%.3f", num(mean, "cp"))); + } + lines.push_back("eval implicit: " + I(ev.has("searches") ? ev.get("searches").as_int() : 0) + " searches, " + I(ev.has("read_follow") ? ev.get("read_follow").as_int() : 0) + + " followed by a read, " + I(ev.has("zero_hit_count") ? ev.get("zero_hit_count").as_int() : 0) + " zero-hit"); + } else if (stats.get("eval_error").truthy()) { + lines.push_back("eval: skipped (" + sv(stats.get("eval_error")) + ")"); + } + lines.push_back("note: hit lists truncate at 1500 chars in the log, shown counts are lower bounds"); + return py::join(lines, "\n"); +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/rag.h b/taicpp/src/agent/rag.h new file mode 100644 index 0000000..72f2d5e --- /dev/null +++ b/taicpp/src/agent/rag.h @@ -0,0 +1,32 @@ +#pragma once +// RetrievalEval and RagStats (tai.py 19355-19907): standing retrieval-quality +// eval and the --stats-rag rollup. Read-only SQL goes through StorePort::sql_rows. +#include "../llm/json.h" +#include "ports.h" +#include +#include +#include +namespace detz::agent { + +struct RetrievalEval { + static std::optional rank_of(const std::string& expected_id, const std::vector& ranked_ids); + static std::optional best_rank(const std::vector& expected_ids, const std::vector& ranked_ids); + static double hit_at_k(std::optional rank, int k); + static double reciprocal_rank(std::optional rank); + static double ndcg_at_k(std::optional rank, int k); + static double context_precision(const std::vector& expected_ids, const std::vector& ranked_ids, int k); + static std::vector tokens(const std::string& text, int min_len = 6); + // goldens: array of {query, expected: [id], category, source} + static json::Value goldens_from_vault(StorePort& store, const std::string& profile, int max_records = 50); + static json::Value score(StorePort& store, const json::Value& goldens, const std::string& profile, + const std::vector& kinds = {"record"}, int k = 5, int fetch = 10); + static json::Value implicit_pairs(StorePort& store, const std::string& profile, int limit = 500); + static std::string format_report(const json::Value& result, const json::Value& pairs); +}; + +struct RagStats { + static json::Value collect(StorePort& store, const std::string& profile, bool include_eval = true); + static std::string format_report(const json::Value& stats); +}; + +} // namespace detz::agent diff --git a/taicpp/src/agent/registry.cpp b/taicpp/src/agent/registry.cpp new file mode 100644 index 0000000..3c92a08 --- /dev/null +++ b/taicpp/src/agent/registry.cpp @@ -0,0 +1,148 @@ +#include "registry.h" +#include "../llm/helpers.h" +#include "../llm/py.h" +#include "consts.h" +#include +#include +#include +namespace detz::agent { + +std::mutex& AgentRegistry::lock() { static std::mutex m; return m; } +std::map>& AgentRegistry::records() { + static std::map> r; + return r; +} +int& AgentRegistry::next_id() { static int n = 1; return n; } + +int spawn_agent(const std::string& task, const std::string& profile, double timeout, AgentConfig& config, int depth, + const Runner& runner) { + std::shared_ptr record = std::make_shared(); + { + std::lock_guard guard(AgentRegistry::lock()); + auto& agents = AgentRegistry::records(); + if (agents.size() >= static_cast(AGENTS_MAX_RECORDS)) { + std::vector old; + for (const auto& kv : agents) { + if (kv.second->status != "running" && kv.second->collected) { old.push_back(kv.first); } + } + std::sort(old.begin(), old.end()); + size_t drop = agents.size() - static_cast(AGENTS_MAX_RECORDS) + 1; + for (size_t i = 0; i < old.size() && i < drop; ++i) { agents.erase(old[i]); } + } + int live = 0; + for (const auto& kv : agents) { if (kv.second->status == "running") { ++live; } } + if (live >= AGENTS_MAX_PARALLEL) { + throw AgentsFull("too many live agents (" + std::to_string(live) + " of " + std::to_string(AGENTS_MAX_PARALLEL) + ")"); + } + int agent_id = AgentRegistry::next_id(); + AgentRegistry::next_id() += 1; + record->id = agent_id; + record->task = task; + record->profile = profile; + record->started = py::now(); + agents[agent_id] = record; + } + AgentConfig* cfg = &config; + std::thread([record, task, profile, timeout, cfg, depth, runner]() { + std::string result, status; + try { + if (!runner) { + result = default_runner(*cfg, profile, depth, task, timeout, WORKER_STEPS, record.get()); + } else { + result = runner(task, profile, timeout); + } + status = result.find("[time limit reached]") != std::string::npos ? "timeout" : "done"; + } catch (const std::exception& exc) { + result = "worker failed: " + llm::short_error(exc); + status = "error"; + } + { + std::lock_guard guard(AgentRegistry::lock()); + record->result = result; + record->status = status; + record->ended = py::now(); + } + { + std::lock_guard l(record->done_mutex); + record->done = true; + } + record->done_cv.notify_all(); + }).detach(); + return record->id; +} + +std::pair poll_agent(int agent_id, double wait, const std::optional& profile) { + std::shared_ptr record; + { + std::lock_guard guard(AgentRegistry::lock()); + auto& agents = AgentRegistry::records(); + auto it = agents.find(agent_id); + if (it != agents.end()) { + record = it->second; + if (profile.has_value() && record->profile != *profile) { record = nullptr; } + } + } + if (!record) { return {"missing", "no agent " + std::to_string(agent_id)}; } + { + std::unique_lock l(record->done_mutex); + record->done_cv.wait_for(l, std::chrono::duration(std::max(0.0, wait)), [&] { return record->done; }); + } + std::string status, result; + double elapsed; + bool idle; + { + std::lock_guard guard(AgentRegistry::lock()); + status = record->status; + result = record->result; + elapsed = (record->ended.has_value() ? *record->ended : py::now()) - record->started; + idle = record->tools.has_value() && *record->tools == 0; + } + if (status == "running") { + return {status, "agent " + std::to_string(agent_id) + " still running (" + std::to_string(static_cast(elapsed)) + "s elapsed)"}; + } + if (status == "error") { return {status, result}; } + std::string text = llm::truncate(result, 6000, 2000); + if (text.empty()) { text = "(empty result)"; } + if (idle) { + text += "\n[note: this worker ran 0 tool calls, so it changed nothing; check its answer before relying on it]"; + } + return {status, text}; +} + +json::Value list_agents(const std::optional& profile) { + std::lock_guard guard(AgentRegistry::lock()); + json::Value out = json::Value::array(); + for (const auto& kv : AgentRegistry::records()) { + const AgentRecord& r = *kv.second; + if (profile.has_value() && r.profile != *profile) { continue; } + json::Value d = json::Value::object(); + d.set("id", r.id); + d.set("task", r.task); + d.set("status", r.status); + d.set("elapsed", static_cast((r.ended.has_value() ? *r.ended : py::now()) - r.started)); + d.set("collected", r.collected); + d.set("tools", r.tools.has_value() ? json::Value(static_cast(*r.tools)) : json::Value()); + out.push(d); + } + return out; +} + +int clear_agents(const std::optional& profile) { + std::lock_guard guard(AgentRegistry::lock()); + auto& agents = AgentRegistry::records(); + std::vector finished; + for (const auto& kv : agents) { + if (kv.second->status != "running" && (!profile.has_value() || kv.second->profile == *profile)) { finished.push_back(kv.first); } + } + for (int k : finished) { agents.erase(k); } + return static_cast(finished.size()); +} + +int running_agents() { + std::lock_guard guard(AgentRegistry::lock()); + int n = 0; + for (const auto& kv : AgentRegistry::records()) { if (kv.second->status == "running") { ++n; } } + return n; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/registry.h b/taicpp/src/agent/registry.h new file mode 100644 index 0000000..ff99dd4 --- /dev/null +++ b/taicpp/src/agent/registry.h @@ -0,0 +1,56 @@ +#pragma once +// Subagent registry: spawn_agent, poll_agent, list_agents, clear_agents, +// running_agents (tai.py 20537-20631). Workers run on detached threads. +#include "agent_config.h" +#include "goal.h" +#include "../llm/json.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace detz::agent { + +// runner(task, profile, timeout_seconds) -> final text. Tests and +// Agent::runner_override replace the default. +using Runner = std::function; + +struct AgentRecord { + int id{0}; + std::string task, profile, status{"running"}, result; + double started{0}; + std::optional ended; + bool collected{false}; + std::optional tools; + std::optional spilled; + // completion signal replacing Thread.join(timeout) + std::mutex done_mutex; + std::condition_variable done_cv; + bool done{false}; +}; + +class AgentRegistry { + public: + static std::mutex& lock(); // AGENTS_LOCK + static std::map>& records(); // AGENTS (callers hold lock()) + static int& next_id(); // AGENTS_NEXT[0] +}; + +// default_runner: a throwaway worker Agent on its own Store (via RuntimePort). +// stats, when given, receives the worker's tool call count under the registry lock. +std::string default_runner(AgentConfig& config, const std::string& profile, int depth, const std::string& task, + double timeout, int max_steps = 12, AgentRecord* stats = nullptr); + +int spawn_agent(const std::string& task, const std::string& profile, double timeout, AgentConfig& config, int depth = 0, + const Runner& runner = nullptr); // throws AgentsFull +// (status, text); status "missing" when unknown. +std::pair poll_agent(int agent_id, double wait = 0, const std::optional& profile = std::nullopt); +json::Value list_agents(const std::optional& profile = std::nullopt); +int clear_agents(const std::optional& profile = std::nullopt); +int running_agents(); + +} // namespace detz::agent diff --git a/taicpp/src/agent/text.cpp b/taicpp/src/agent/text.cpp new file mode 100644 index 0000000..b64098c --- /dev/null +++ b/taicpp/src/agent/text.cpp @@ -0,0 +1,773 @@ +#include "text.h" +#include "../llm/helpers.h" +#include "../llm/proc.h" +#include "../llm/py.h" +#include "consts.h" +#include "data.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::agent { + +using llm::env_get; +using llm::which; + +std::string trunc_marker(int64_t true_bytes) { + return "\n...[truncated, full size " + std::to_string(true_bytes) + " bytes]..."; +} + +Capped capped_text(const std::optional& text) { + Capped c; + if (!text.has_value()) { return c; } + int64_t bytes = static_cast(text->size()); + if (static_cast(py::len(*text)) <= AUDIT_MAX_CHARS) { + c.text = *text; + c.size = bytes; + return c; + } + c.text = py::head(*text, AUDIT_MAX_CHARS) + trunc_marker(bytes); + c.size = bytes; + return c; +} + +std::string os_error_message(int err, const std::string& path) { + return "[Errno " + std::to_string(err) + "] " + std::strerror(err) + ": " + py::repr_str(path); +} + +std::optional read_text_file(const std::string& path, std::string* error) { + FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) { + if (error != nullptr) { *error = os_error_message(errno, path); } + return std::nullopt; + } + std::string raw; + char buf[65536]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) { raw.append(buf, n); } + bool bad = std::ferror(f) != 0; + int err = errno; + std::fclose(f); + if (bad) { + if (error != nullptr) { *error = os_error_message(err, path); } + return std::nullopt; + } + return py::universal_newlines(py::utf8_replace(raw)); +} + +bool write_text_file(const std::string& path, const std::string& content, std::string& error) { + FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) { error = os_error_message(errno, path); return false; } + size_t w = content.empty() ? 0 : std::fwrite(content.data(), 1, content.size(), f); + int err = errno; + bool ok = (w == content.size()) && std::fflush(f) == 0; + if (!ok) { err = errno; } + std::fclose(f); + if (!ok) { error = os_error_message(err, path); return false; } + return true; +} + +std::string abspath(const std::string& path) { + fs::path p(path.empty() ? "." : path); + std::error_code ec; + fs::path a = p.is_absolute() ? p : fs::current_path(ec) / p; + std::string out = a.lexically_normal().string(); + while (out.size() > 1 && out.back() == '/') { out.pop_back(); } + if (out.empty()) { out = "/"; } + return out; +} +bool isfile(const std::string& path) { + struct stat st; + return !path.empty() && ::stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode); +} +bool isdir(const std::string& path) { + struct stat st; + return !path.empty() && ::stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); +} +bool path_exists(const std::string& path) { + struct stat st; + return ::stat(path.c_str(), &st) == 0; +} +std::string expanduser(const std::string& path) { + if (path.empty() || path[0] != '~') { return path; } + size_t slash = path.find('/'); + std::string user = path.substr(1, slash == std::string::npos ? std::string::npos : slash - 1); + std::string home; + if (user.empty()) { + home = env_get("HOME"); + if (home.empty()) { + struct passwd* pw = getpwuid(getuid()); + if (pw == nullptr) { return path; } + home = pw->pw_dir; + } + } else { + struct passwd* pw = getpwnam(user.c_str()); + if (pw == nullptr) { return path; } + home = pw->pw_dir; + } + while (home.size() > 1 && home.back() == '/') { home.pop_back(); } + std::string rest = slash == std::string::npos ? "" : path.substr(slash); + return home + rest; +} + +Capped read_capped(const std::string& path, int cap) { + Capped none; + std::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) { return none; } + std::ifstream in(path, std::ios::binary); + if (!in) { return none; } + size_t want = static_cast(cap + 2) * 4; + std::string raw(want, '\0'); + in.read(raw.data(), static_cast(want)); + raw.resize(static_cast(in.gcount())); + std::string text = py::universal_newlines(py::utf8_replace(raw)); + Capped c; + if (static_cast(py::len(text)) > cap) { + c.text = py::head(text, cap); + c.size = static_cast(size); + } else { + c.text = text; + } + return c; +} + +std::vector chunk_lines(const std::string& text, int max_lines, int overlap_lines, int max_chars) { + std::vector lines = py::splitlines(text); + size_t n = lines.size(); + std::vector chunks; + if (n == 0) { return chunks; } + size_t start = 0; + while (start < n) { + size_t end = start; + long size = 0; + int count = 0; + while (end < n && count < max_lines && size < max_chars) { + size += static_cast(py::len(lines[end])) + 1; + count += 1; + end += 1; + } + std::vector window(lines.begin() + static_cast(start), lines.begin() + static_cast(end)); + std::string piece = py::strip(py::join(window, "\n")); + if (!piece.empty()) { chunks.push_back(piece); } + if (end >= n) { break; } + long next = static_cast(end) - overlap_lines; + start = static_cast(std::max(static_cast(start) + 1, next)); + } + return chunks; +} + +std::string collapse_repeated_lines(const std::string& text, int threshold) { + std::vector lines = py::splitlines(text); + std::vector out; + size_t i = 0, n = lines.size(); + while (i < n) { + size_t j = i; + while (j < n && lines[j] == lines[i]) { ++j; } + size_t run = j - i; + if (run >= static_cast(threshold)) { + out.push_back(lines[i]); + out.push_back("... (repeated " + std::to_string(run - 1) + " more times) ..."); + } else { + for (size_t k = i; k < j; ++k) { out.push_back(lines[k]); } + } + i = j; + } + return py::join(out, "\n"); +} + +std::string spilled_view(const std::string& text, const std::string& record_id, size_t size, int head, int tail) { + return py::head(text, head) + "\n...[" + std::to_string(size) + " chars spilled to " + record_id + + ", record_read pages the rest]...\n" + py::tail(text, tail); +} + +std::string spill_output(StorePort& store, const std::string& kind, const std::string& title, const std::string& text, + const std::vector& tags, const std::string& profile) { + std::string p = profile.empty() ? store.profile() : profile; + std::string record_id = store.add_record(kind, title, text, tags, p); + store.add_chunks("record", record_id, chunk_lines(text), p); + return record_id; +} + +std::string spill_or_truncate(StorePort* store, const std::string& kind, const std::string& title, const std::string& text, + const std::vector& tags, const std::string& profile) { + if (store == nullptr || static_cast(py::len(text)) <= SPILL_LIMIT) { return llm::truncate(text); } + std::string record_id = spill_output(*store, kind, title, text, tags, profile); + return spilled_view(text, record_id, py::len(text)); +} + +std::string stream_status(int code, const std::string& output, double elapsed, bool timed_out) { + char buf[64]; + std::string detail = std::to_string(py::splitlines(output).size()) + " lines \xC2\xB7 " + + llm::human_size(static_cast(py::utf8_replace(output).size())); + if (timed_out) { + return "timed out after " + std::to_string(SHELL_TIMEOUT) + "s (killed) \xC2\xB7 " + detail; + } + std::snprintf(buf, sizeof buf, "%.1f", elapsed); + return "exit " + std::to_string(code) + " \xC2\xB7 " + buf + "s \xC2\xB7 " + detail; +} + +static bool word_at(const std::string& s, size_t pos, const char* w) { + size_t n = std::char_traits::length(w); + return s.compare(pos, n, w) == 0; +} +std::string curl_hint(const std::string& command) { + // re.search(r"\bcurl\b|\bwget\b", command) and "http" in command.lower() + auto is_word = [](unsigned char c) { return std::isalnum(c) || c == '_' || c >= 0x80; }; + bool found = false; + for (const char* w : {"curl", "wget"}) { + size_t n = std::char_traits::length(w); + size_t pos = 0; + while ((pos = command.find(w, pos)) != std::string::npos) { + bool left = pos == 0 || !is_word(static_cast(command[pos - 1])); + bool right = pos + n >= command.size() || !is_word(static_cast(command[pos + n])); + if (left && right) { found = true; break; } + ++pos; + } + if (found) { break; } + } + (void)word_at; + if (found && py::lower(command).find("http") != std::string::npos) { return "\n" + data::CurlHint(); } + return ""; +} + +std::vector alnum_tokens(const std::string& text) { + std::vector out; + std::string cur; + for (uint32_t cp : py::decode(py::lower(text))) { + if (cp == 0x212A) { cp = 'k'; } + bool keep = (cp >= 'a' && cp <= 'z') || (cp >= '0' && cp <= '9'); + if (keep) { + cur += static_cast(cp); + } else { + if (cp == 0x130) { cur += 'i'; } // lower() gives "i" + U+0307 + if (!cur.empty()) { out.push_back(cur); cur.clear(); } + } + } + if (!cur.empty()) { out.push_back(cur); } + return out; +} + +// ------------------------------------------------------------------ tags +std::string secret_env_name(const std::string& name) { + std::string up = py::upper(name); + std::string out = "TAI_SECRET_"; + for (uint32_t cp : py::decode(up)) { + if ((cp >= 'A' && cp <= 'Z') || (cp >= '0' && cp <= '9') || cp == '_') { out += static_cast(cp); } + else { out += '_'; } + } + return out; +} + +std::string normalize_tag(const std::string& raw) { + std::string s = py::lower(py::strip(raw)); + std::string out; + bool pending = false; + for (uint32_t cp : py::decode(s)) { + if (cp == 0x212A) { cp = 'k'; } // KELVIN SIGN lowercases to ASCII k + if (cp == 0x130) { // LATIN CAPITAL I WITH DOT lowercases to "i" + U+0307 + if (pending) { out += '-'; } + out += 'i'; + pending = true; + continue; + } + bool keep = (cp >= 'a' && cp <= 'z') || (cp >= '0' && cp <= '9'); + if (keep) { + if (pending) { out += '-'; pending = false; } + out += static_cast(cp); + } else { + pending = true; + } + } + if (pending) { out += '-'; } + // re.sub(...).strip("-")[:32] + out = py::strip_chars(out, "-"); + return py::head(out, 32); +} + +std::vector normalize_tags(const std::vector& items) { + std::vector seen; + for (const auto& item : items) { + std::string cleaned = normalize_tag(item); + if (!cleaned.empty() && std::find(seen.begin(), seen.end(), cleaned) == seen.end()) { seen.push_back(cleaned); } + } + if (seen.size() > 20) { seen.resize(20); } + return seen; +} + +std::string singular_noun(const std::string& word) { + static const char* keep[] = {"news", "means", "series", "species", "physics"}; + for (const char* k : keep) { if (word == k) { return word; } } + if (py::len(word) <= 3) { return word; } + if (py::endswith(word, "ies")) { return word.substr(0, word.size() - 3) + "y"; } + for (const char* suf : {"ses", "xes", "zes", "ches", "shes"}) { + if (py::endswith(word, suf)) { return word.substr(0, word.size() - 2); } + } + if (py::endswith(word, "s")) { + bool blocked = false; + for (const char* suf : {"ss", "us", "is", "os"}) { if (py::endswith(word, suf)) { blocked = true; } } + if (!blocked) { return word.substr(0, word.size() - 1); } + } + return word; +} + +std::vector tag_variants(const std::string& tag) { + std::vector found = {tag}; + std::string sing = singular_noun(tag); + if (std::find(found.begin(), found.end(), sing) == found.end()) { found.push_back(sing); } + if (!py::endswith(tag, "s")) { + std::string plural = tag + "s"; + if (std::find(found.begin(), found.end(), plural) == found.end()) { found.push_back(plural); } + } + std::sort(found.begin(), found.end()); + return found; +} + +std::optional> parse_node_id(const std::string& raw) { + std::string s = py::strip(raw); + size_t colon = s.find(':'); + if (colon == std::string::npos) { return std::nullopt; } + std::string kind = s.substr(0, colon), key = s.substr(colon + 1); + if (!(kind == "mem" || kind == "secret" || kind == "sched") || key.empty()) { return std::nullopt; } + // re.fullmatch(r"(mem|secret|sched):(.+)"): `.` excludes newline + if (key.find('\n') != std::string::npos) { return std::nullopt; } + if (kind == "mem") { + if (key.size() != 16) { return std::nullopt; } + for (char c : key) { if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) { return std::nullopt; } } + return std::make_pair(kind, key); + } + if (kind == "secret") { + if (secret_ok(key) && key.find('\n') == std::string::npos) { return std::make_pair(kind, key); } + return std::nullopt; + } + if (py::isdigit_ascii(key)) { return std::make_pair(kind, key); } + return std::nullopt; +} + +std::string create_skill_prompt(const std::string& name, const std::string& brief, const std::string& skill_dir) { + return "Create a new agent skill named '" + name + "' in " + skill_dir + ".\n" + "Brief: " + brief + "\n\n" + "Follow this process exactly:\n" + "1. Call sysinfo first so the skill matches this machine (os, installed tools, root, container, venv).\n" + "2. Research the subject deeply with web_search, web_fetch, and shell. Keep researching until you literally " + "have enough verified information to write complete instructions. Verify factual claims against at least two " + "independent sources. Never invent commands, paths, versions, or API details.\n" + "3. Write SKILL.md in the skill directory with this exact frontmatter:\n" + "---\nname: " + name + "\ndescription: \n---\n" + "followed by clear markdown instructions grounded in your research.\n" + "4. Add scripts/, references/, or assets/ files only when they carry real weight; keep every file focused.\n" + "5. Read back every file you wrote, fix mistakes, and confirm the frontmatter holds a name plus a description.\n" + "6. Reply with a short summary: what the skill covers, which sources verified it, and which files you wrote.\n" + "Do not stop early: an incomplete or unverified skill is a failure. Do not ask questions; decide and record assumptions in the skill body."; +} + +// ------------------------------------------------------------------ terminal +std::string tmux_pane_header() { + llm::ProcResult info = llm::run_process({"tmux", "display-message", "-p", "#{session_name}:#{window_index}.#{pane_index} #{pane_current_command}"}, + std::nullopt, 10); + if (info.spawn_failed || info.timed_out) { return ""; } + std::string out = py::strip(py::utf8_replace(info.out)); + if (info.returncode == 0 && !out.empty()) { return out; } + return ""; +} + +static std::string home_dir() { + std::string h = env_get("HOME"); + if (!h.empty()) { return h; } + struct passwd* pw = getpwuid(getuid()); + return pw != nullptr ? pw->pw_dir : "~"; +} + +static std::vector shell_history_paths() { + std::vector paths; + std::string histfile = env_get("HISTFILE"); + if (!histfile.empty()) { paths.push_back(histfile); } + std::string home = home_dir(); + for (const std::string& cand : {home + "/.bash_history", home + "/.zsh_history", home + "/.local/share/fish/fish_history"}) { + if (std::find(paths.begin(), paths.end(), cand) == paths.end()) { paths.push_back(cand); } + } + return paths; +} + +static bool zsh_history_match(const std::string& s, std::string& cmd) { + // ^: \d+:\d+;(.*)$ + if (!py::startswith(s, ": ")) { return false; } + size_t i = 2, d1 = i; + while (i < s.size() && std::isdigit(static_cast(s[i]))) { ++i; } + if (i == d1 || i >= s.size() || s[i] != ':') { return false; } + ++i; + size_t d2 = i; + while (i < s.size() && std::isdigit(static_cast(s[i]))) { ++i; } + if (i == d2 || i >= s.size() || s[i] != ';') { return false; } + cmd = s.substr(i + 1); + return cmd.find('\n') == std::string::npos; +} + +static std::vector read_history_file(const std::string& path) { + std::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) { return {}; } + std::ifstream in(path, std::ios::binary); + if (!in) { return {}; } + if (size > static_cast(SHELL_HISTORY_READ_BYTES)) { + in.seekg(static_cast(size - SHELL_HISTORY_READ_BYTES)); + std::string skip; + std::getline(in, skip); + } + std::string raw((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + bool fish = fs::path(path).filename() == "fish_history"; + std::vector entries; + for (const auto& line : py::splitlines(py::utf8_replace(raw))) { + std::string stripped = py::strip(line); + if (stripped.empty()) { continue; } + if (fish) { + if (!py::startswith(stripped, "- cmd: ")) { continue; } + stripped = py::strip(stripped.substr(7)); + if (!stripped.empty()) { entries.push_back(stripped); } + continue; + } + std::string cmd; + if (zsh_history_match(stripped, cmd)) { + stripped = py::strip(cmd); + if (stripped.empty()) { continue; } + } + entries.push_back(stripped); + } + return entries; +} + +std::string shell_history_section() { + std::vector merged; + for (const auto& p : shell_history_paths()) { + auto part = read_history_file(p); + merged.insert(merged.end(), part.begin(), part.end()); + } + std::vector entries; + for (const auto& line : merged) { + if (entries.empty() || line != entries.back()) { entries.push_back(line); } + } + if (SHELL_HISTORY_LINES > 0 && entries.size() > static_cast(SHELL_HISTORY_LINES)) { + entries.erase(entries.begin(), entries.end() - SHELL_HISTORY_LINES); + } + std::string text = py::join(entries, "\n"); + if (static_cast(py::len(text)) > SHELL_HISTORY_CHARS) { + text = py::tail(text, SHELL_HISTORY_CHARS); + size_t nl = text.find('\n'); + if (nl != std::string::npos) { text = text.substr(nl + 1); } + } + if (py::strip(text).empty()) { return ""; } + return "\n\n## Recent shell history\n(oldest first, newest last)\n```\n" + text + "\n```"; +} + +std::pair, std::string> tmux_panes() { + if (which("tmux").empty()) { return {{}, "tmux not available"}; } + std::string fmt = "#{session_name}\t#{window_index}\t#{pane_index}\t#{pane_id}\t#{pane_active}\t#{pane_current_command}\t#{pane_current_path}"; + llm::ProcResult done = llm::run_process({"tmux", "list-panes", "-a", "-F", fmt}, std::nullopt, 10); + if (done.spawn_failed) { return {{}, "terminal list failed: " + llm::short_error_text(done.spawn_error)}; } + if (done.timed_out) { return {{}, "terminal list failed: Command '['tmux', 'list-panes', '-a', '-F', '...']' timed out after 10 seconds"}; } + if (done.returncode != 0) { + std::string err = py::head(py::strip(py::universal_newlines(py::utf8_replace(done.err))), 200); + if (err.find("no server running") != std::string::npos) { return {{}, "no open terminals (tmux server not running)"}; } + return {{}, "terminal list failed: " + err}; + } + std::vector panes; + for (const auto& line : py::splitlines(py::universal_newlines(py::utf8_replace(done.out)))) { + auto parts = py::split(line, "\t"); + if (parts.size() != 7) { continue; } + TmuxPane p; + p.ref = parts[0] + ":" + parts[1] + "." + parts[2]; + p.id = parts[3]; + p.active = parts[4] == "1"; + p.command = parts[5].empty() ? "?" : parts[5]; + p.path = parts[6].empty() ? "?" : parts[6]; + panes.push_back(p); + } + return {panes, ""}; +} + +// ------------------------------------------------------------------ html / url +std::string html_unescape(const std::string& s) { + if (s.find('&') == std::string::npos) { return s; } + static const std::unordered_map html5 = [] { + std::unordered_map m; + for (const auto& kv : data::Html5Entities()) { m[kv.first] = kv.second; } + return m; + }(); + static const std::unordered_map invalid_charrefs = [] { + std::unordered_map m; + for (const auto& kv : data::HtmlInvalidCharrefs()) { m[kv.first] = kv.second; } + return m; + }(); + static const std::vector& invalid_cps = data::HtmlInvalidCodepoints(); + std::string out; + size_t i = 0, n = s.size(); + auto is_hex = [](char c) { return std::isxdigit(static_cast(c)) != 0; }; + while (i < n) { + if (s[i] != '&') { out += s[i++]; continue; } + size_t p = i + 1; + std::string group; // text after '&' + bool matched = false; + if (p < n && s[p] == '#') { + size_t q = p + 1; + if (q < n && std::isdigit(static_cast(s[q]))) { + size_t r = q; + while (r < n && std::isdigit(static_cast(s[r]))) { ++r; } + if (r < n && s[r] == ';') { ++r; } + group = s.substr(p, r - p); + matched = true; + p = r; + } else if (q + 1 < n && (s[q] == 'x' || s[q] == 'X') && is_hex(s[q + 1])) { + size_t r = q + 1; + while (r < n && is_hex(s[r])) { ++r; } + if (r < n && s[r] == ';') { ++r; } + group = s.substr(p, r - p); + matched = true; + p = r; + } + } else { + // [^\t\n\f <&#;]{1,32};? (32 code points) + size_t r = p; + int count = 0; + while (r < n && count < 32) { + unsigned char c = static_cast(s[r]); + if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '<' || c == '&' || c == '#' || c == ';') { break; } + size_t adv = 1; + if (c >= 0xF0) { adv = 4; } else if (c >= 0xE0) { adv = 3; } else if (c >= 0xC0) { adv = 2; } + r = std::min(n, r + adv); + ++count; + } + if (count >= 1) { + if (r < n && s[r] == ';') { ++r; } + group = s.substr(p, r - p); + matched = true; + p = r; + } + } + if (!matched) { out += s[i++]; continue; } + // _replace_charref + if (group[0] == '#') { + std::string digits = group; + while (!digits.empty() && digits.back() == ';') { digits.pop_back(); } + bool hex = digits.size() > 1 && (digits[1] == 'x' || digits[1] == 'X'); + std::string body = digits.substr(hex ? 2 : 1); + // arbitrary precision: anything beyond 8 significant digits is above 0x10FFFF + size_t lead = 0; + while (lead + 1 < body.size() && body[lead] == '0') { ++lead; } + std::string sig = body.substr(lead); + long long num = 0; + bool huge = sig.size() > (hex ? 7u : 9u); + if (!huge) { num = std::stoll(sig, nullptr, hex ? 16 : 10); } + if (!huge && invalid_charrefs.count(static_cast(num))) { out += invalid_charrefs.at(static_cast(num)); } + else if (huge || (num >= 0xD800 && num <= 0xDFFF) || num > 0x10FFFF) { out += "\xEF\xBF\xBD"; } + else if (std::find(invalid_cps.begin(), invalid_cps.end(), static_cast(num)) != invalid_cps.end()) { /* '' */ } + else { py::put_utf8(out, static_cast(num)); } + } else { + auto it = html5.find(group); + if (it != html5.end()) { + out += it->second; + } else { + // longest matching prefix: for x in range(len(s)-1, 1, -1) + std::vector cps = py::decode(group); + bool done = false; + for (long x = static_cast(cps.size()) - 1; x > 1; --x) { + std::string prefix = py::encode(std::vector(cps.begin(), cps.begin() + x)); + auto f = html5.find(prefix); + if (f != html5.end()) { + out += f->second + py::encode(std::vector(cps.begin() + x, cps.end())); + done = true; + break; + } + } + if (!done) { out += "&" + group; } + } + } + i = p; + } + return out; +} + +std::string url_quote_plus(const std::string& s) { + static const char* hex = "0123456789ABCDEF"; + std::string out; + for (unsigned char c : s) { + if (std::isalnum(c) && c < 0x80) { out += static_cast(c); } + else if (c == '_' || c == '.' || c == '-' || c == '~') { out += static_cast(c); } + else if (c == ' ') { out += '+'; } + else { out += '%'; out += hex[c >> 4]; out += hex[c & 15]; } + } + return out; +} + +std::string url_hostname(const std::string& url) { + size_t scheme = url.find("://"); + std::string rest = scheme == std::string::npos ? url : url.substr(scheme + 3); + if (scheme == std::string::npos) { return ""; } + size_t end = rest.find_first_of("/?#"); + std::string netloc = end == std::string::npos ? rest : rest.substr(0, end); + size_t at = netloc.rfind('@'); + if (at != std::string::npos) { netloc = netloc.substr(at + 1); } + std::string host; + if (!netloc.empty() && netloc[0] == '[') { + size_t close = netloc.find(']'); + host = close == std::string::npos ? netloc.substr(1) : netloc.substr(1, close - 1); + } else { + size_t colon = netloc.find(':'); + host = colon == std::string::npos ? netloc : netloc.substr(0, colon); + } + return py::lower(host); +} + +// ------------------------------------------------------------------ schedule +std::string parse_schedule_at(const std::string& raw) { + std::string text = py::strip(raw); + if (text.empty()) { throw std::invalid_argument("empty datetime"); } + if (text.back() == 'Z' || text.back() == 'z') { text = text.substr(0, text.size() - 1) + "+00:00"; } + py::DateTime moment; + try { + moment = py::fromisoformat(text); + } catch (const std::invalid_argument&) { + throw std::invalid_argument("invalid datetime, use ISO like 2026-10-08T09:00"); + } + return py::to_utc_iso(moment); +} + +std::optional parse_duration(const std::string& raw) { + std::string s = py::lower(py::strip(raw)); + size_t i = 0; + while (i < s.size() && std::isdigit(static_cast(s[i]))) { ++i; } + if (i == 0) { return std::nullopt; } + std::string digits = s.substr(0, i); + size_t j = i; + while (j < s.size() && py::is_space_cp(static_cast(s[j]))) { ++j; } + char unit = 's'; + if (j < s.size()) { + char c = s[j]; + if (!(c == 's' || c == 'm' || c == 'h' || c == 'd' || c == 'w')) { return std::nullopt; } + unit = c; + ++j; + } + if (j != s.size()) { return std::nullopt; } + int64_t mult = unit == 's' ? 1 : unit == 'm' ? 60 : unit == 'h' ? 3600 : unit == 'd' ? 86400 : 604800; + auto n = py::parse_int(digits); + if (!n.has_value()) { return std::nullopt; } + return *n * mult; +} + +std::string format_delay(int64_t seconds) { + if (seconds < 0) { return "overdue"; } + if (seconds < 60) { return std::to_string(seconds) + "s"; } + if (seconds < 3600) { return std::to_string(seconds / 60) + "m"; } + if (seconds < 86400) { return std::to_string(seconds / 3600) + "h"; } + return std::to_string(seconds / 86400) + "d"; +} + +std::string local_display(const std::string& utc_iso) { + try { + return py::local_minutes(py::to_epoch(py::fromisoformat(utc_iso))); + } catch (const std::invalid_argument&) { + return utc_iso; + } +} + +std::string format_schedule_lines(const json::Value& items, double now_epoch) { + if (items.size() == 0) { return "(no schedules)"; } + std::vector lines; + for (const auto& item : items.items()) { + double due_in; + try { + due_in = py::to_epoch(py::fromisoformat(py::str(item.get("next_run")))) - now_epoch; + } catch (const std::invalid_argument&) { + due_in = -1; + } + std::string when = item.get("every").truthy() + ? "every " + format_delay(item.get("every").as_int()) + : "at " + local_display(py::str(item.get("next_run"))); + std::string marked; + if (item.get("tags").truthy()) { + std::vector tags; + for (const auto& t : item.get("tags").items()) { tags.push_back(py::str(t)); } + marked = " [" + py::join(tags, " ") + "]"; + } + std::string name = item.get("name").truthy() ? py::str(item.get("name")) : "(no name)"; + lines.push_back("#" + std::to_string(item.get("id").as_int()) + " [" + py::str(item.get("status")) + "] " + when + " (" + + format_delay(static_cast(due_in)) + ") " + name + marked + " :: " + + py::head(py::str(item.get("prompt")), 80)); + if (item.get("last").truthy()) { + std::string extra = item.get("result").truthy() ? " :: " + py::head(py::str(item.get("result")), 120) : ""; + lines.push_back(" last: " + py::str(item.get("last")) + extra); + } + } + return py::join(lines, "\n"); +} + +std::string py_type_name(const json::Value& v) { + switch (v.type()) { + case json::Value::Type::Null: return "NoneType"; + case json::Value::Type::Bool: return "bool"; + case json::Value::Type::Int: return "int"; + case json::Value::Type::Double: return "float"; + case json::Value::Type::String: return "str"; + case json::Value::Type::Array: return "list"; + case json::Value::Type::Object: return "dict"; + } + return "object"; +} + +// ------------------------------------------------------------------ consts.cpp +const std::vector& RecordKinds() { + static const std::vector v = {"note", "output", "file", "research", "transcript", "termlog"}; + return v; +} +const std::vector& SafeCommands() { + static const std::vector v = {"ls", "pwd", "echo", "cat", "head", "tail", "grep", "find", "wc", "sort", "uniq", "diff", "file", "stat", "date", "whoami", "uname", "lsb_release"}; + return v; +} +const std::vector& FetchMethods() { + static const std::vector v = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}; + return v; +} +const std::vector& CommandNames() { + static const std::vector v = {"help", "profile", "profiles", "bots", "bot", "env", "ctx", "skills", "sysinfo", "models", "secret", "schedule", "schedules", "unschedule", "records", "record", "graph", "tags", "tools", "install", "search", "audit", "restore", "release", "fork", "agents", "agent", "compact", "clear", "demo", "quit", "exit"}; + return v; +} + +static bool match_name(const std::string& in, bool first_lower_only, bool allow_dot, size_t max_rest) { + std::string s = in; + if (!s.empty() && s.back() == '\n') { s.pop_back(); } // `$` matches before a trailing newline + if (s.empty() || s.size() > max_rest + 1) { return false; } + auto first_ok = [&](unsigned char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (!first_lower_only && c >= 'A' && c <= 'Z'); + }; + if (!first_ok(static_cast(s[0]))) { return false; } + for (size_t i = 1; i < s.size(); ++i) { + unsigned char c = static_cast(s[i]); + bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (!first_lower_only && c >= 'A' && c <= 'Z') || c == '_' || c == '-' || (allow_dot && c == '.'); + if (!ok) { return false; } + } + return true; +} +bool profile_ok(const std::string& s) { return match_name(s, false, false, 31); } +bool bot_ok(const std::string& s) { return match_name(s, true, false, 31); } +bool secret_ok(const std::string& s) { return match_name(s, false, true, 63); } +bool mcp_server_ok(const std::string& s) { return match_name(s, false, true, 63); } +bool skill_name_ok(const std::string& in) { + // ^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$ + std::string s = in; + if (!s.empty() && s.back() == '\n') { s.pop_back(); } + if (s.empty() || s.size() > 64) { return false; } + auto alnum = [](unsigned char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'); }; + if (!alnum(static_cast(s.front())) || !alnum(static_cast(s.back()))) { return false; } + for (unsigned char c : s) { if (!(alnum(c) || c == '-')) { return false; } } + return true; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/text.h b/taicpp/src/agent/text.h new file mode 100644 index 0000000..826b5b6 --- /dev/null +++ b/taicpp/src/agent/text.h @@ -0,0 +1,74 @@ +#pragma once +// Text, tag, terminal and schedule helpers of tai.py that the agent core needs +// (kept in one place; each function names its tai.py origin). +#include "../llm/json.h" +#include "ports.h" +#include +#include +#include +#include +namespace detz::agent { + +// 597 truncate lives in llm::truncate. +std::string trunc_marker(int64_t true_bytes); // 629 +// 621 capped_text: (text|None, size|None); text None -> nullopt. +struct Capped { std::optional text; std::optional size; }; +Capped capped_text(const std::optional& text); +// 633 read_capped(path, cap): (None, None) on failure, (text, None) when whole, +// (text[:cap], true size) when longer than cap code points. +Capped read_capped(const std::string& path, int cap = 20000); +// open(path, "r", encoding="utf-8", errors="replace").read(): whole file text, nullopt on OSError. +// `error` receives str(OSError) ("[Errno 13] Permission denied: 'p'") on failure. +std::optional read_text_file(const std::string& path, std::string* error = nullptr); +bool write_text_file(const std::string& path, const std::string& content, std::string& error); // open(path,"w").write +std::string os_error_message(int err, const std::string& path); +std::string abspath(const std::string& path); // os.path.abspath +bool isfile(const std::string& path); // os.path.isfile +bool isdir(const std::string& path); +bool path_exists(const std::string& path); +std::string expanduser(const std::string& path); + +std::vector chunk_lines(const std::string& text, int max_lines = 60, int overlap_lines = 3, int max_chars = 4000); // 772 +std::string collapse_repeated_lines(const std::string& text, int threshold = 3); // 801 +std::string spilled_view(const std::string& text, const std::string& record_id, size_t size, int head = 1500, int tail = 500); // 756 +std::string spill_output(StorePort& store, const std::string& kind, const std::string& title, const std::string& text, + const std::vector& tags, const std::string& profile); // 749 +std::string spill_or_truncate(StorePort* store, const std::string& kind, const std::string& title, const std::string& text, + const std::vector& tags, const std::string& profile); // 760 +std::string stream_status(int code, const std::string& output, double elapsed, bool timed_out); // 1187 +std::string curl_hint(const std::string& command); // 183 + +// re.findall(r"[a-z0-9]+", text.lower()) with Python's lower() quirks for U+212A and U+0130. +std::vector alnum_tokens(const std::string& text); + +// 9232..9290 +std::string secret_env_name(const std::string& name); +std::string normalize_tag(const std::string& raw); +std::vector normalize_tags(const std::vector& items); +std::string singular_noun(const std::string& word); +std::vector tag_variants(const std::string& tag); +std::optional> parse_node_id(const std::string& raw); +std::string create_skill_prompt(const std::string& name, const std::string& brief, const std::string& skill_dir); + +// 821-930 terminal helpers +std::string tmux_pane_header(); +std::string shell_history_section(); +struct TmuxPane { std::string ref, id, command, path; bool active{false}; }; +std::pair, std::string> tmux_panes(); + +// html.unescape, urllib.parse.quote_plus/urlencode/urlparse(...).hostname +std::string html_unescape(const std::string& s); +std::string url_quote_plus(const std::string& s); +std::string url_hostname(const std::string& url); + +// 20633-20690 schedule helpers +std::string parse_schedule_at(const std::string& raw); // throws std::invalid_argument +std::optional parse_duration(const std::string& raw); +std::string format_delay(int64_t seconds); +std::string local_display(const std::string& utc_iso); +std::string format_schedule_lines(const json::Value& items, double now_epoch); + +// 20590: how a Python object of `type` answers `.get`: "'list' object has no attribute 'get'". +std::string py_type_name(const json::Value& v); + +} // namespace detz::agent diff --git a/taicpp/src/agent/tools.cpp b/taicpp/src/agent/tools.cpp deleted file mode 100644 index a62bf03..0000000 --- a/taicpp/src/agent/tools.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "tools.h" -#include -#include -#include -namespace detz::agent { -static std::set words(const std::string& t){ - std::set w; std::string cur; - for(char c: t){ if(std::isalnum((unsigned char)c)) cur+=std::tolower(c); else { if(cur.size())w.insert(cur); cur.clear(); } } - if(cur.size())w.insert(cur); return w; -} -std::vector SelectTools(const std::string& text){ - auto w = words(text); std::vector out; - for(auto& n: ToolNames()){ - if(std::find(CoreTools().begin(),CoreTools().end(),n)!=CoreTools().end()){ out.push_back(n); continue; } - std::string tok; bool hit=false; - std::string cur; - for(char c: n+std::string("_")){ if(c=='_'){ if(cur.size()>=4&&w.count(cur))hit=true; cur.clear(); } else cur+=c; } - if(w.count(n)) hit=true; - if(hit) out.push_back(n); - } - return out; -} -} // namespace detz::agent diff --git a/taicpp/src/agent/tools.h b/taicpp/src/agent/tools.h index 70ab67e..a1c92d4 100644 --- a/taicpp/src/agent/tools.h +++ b/taicpp/src/agent/tools.h @@ -1,30 +1,154 @@ #pragma once +// The Tools class of tai.py (17071-19059): the 61 tool handlers, dispatch and +// the secret/approval/audit plumbing around them. Schemas live in data.cpp +// (generated), selection in agent.cpp (select_tools / tool_catalog). +#include "../llm/json.h" +#include "goal.h" +#include "ports.h" +#include +#include +#include +#include #include +#include #include namespace detz::agent { -// Full 61-tool catalog, mirrors tai.py TOOL_SCHEMAS order. -inline const std::vector& ToolNames() { - static const std::vector v = { - "shell","read_file","write_file","edit_file","web_search","describe_image", - "web_fetch","speak","listen","remember","recall","load_skill", - "get_current_terminal_content","list_terminals","tail_terminals", - "export_terminal_log","plexer_send","plexer_read","plexer_type", - "plexer_status","plexer_show","tui_screen","tui_keys","tui_type", - "tui_click","tui_goto","tui_result","fork","poll","ask_user","sysinfo", - "create_skill","install_skill","store_secret","list_secrets","delete_secret", - "schedule","unschedule","schedules","record_save","record_read", - "record_search","record_delete","graph_link","graph_query","delete_file", - "audit","restore","release","tags","install","create_bot","search", - "mcp_add","mcp_list","mcp_enable","mcp_disable","mcp_remove","mcp_tools", - "mcp_call","mcp_login"}; - return v; -} -inline const std::vector& CoreTools() { - static const std::vector v = { - "shell","read_file","write_file","remember","recall","load_skill", - "fork","poll","ask_user"}; - return v; -} -// select_tools mirror: core always picked; others picked on token/tag match. -std::vector SelectTools(const std::string& text); + +class Agent; + +struct DiffPreview { + std::string path; + std::optional old_text, new_text; +}; + +class Tools { + public: + explicit Tools(Agent& app); + + Agent& app; + int64_t calls{0}; + using SecretGrant = std::tuple>; + std::set secret_grants; + std::set> read_files; // ("home"|"box", path) + bool live{false}; + std::optional diff_preview; + bool board_shown{false}; + std::optional result_note; + + // dispatch(name, raw_arguments): raw_arguments is the JSON text the model sent + // (string) or an already decoded object. Returns the tool result text; throws Denied. + std::string dispatch(const std::string& name, const json::Value& raw_arguments); + + json::Value mcp_schemas(); // McpBridge(app).schemas_for(profile) + std::string active_profile() const; + std::string audit_actor() const; + void note_diff(const std::string& path, const std::optional& old_text, const std::optional& new_text); + + // Registered handler names in TOOL_SCHEMAS order (tests compare with the schema list). + static const std::vector& handler_names(); + + private: + using Handler = std::string (Tools::*)(const json::Value& args); + static const std::map& handler_table(); + + // app.store / ports with the Python failure mode: None attribute access. + StorePort& S(const char* method) const; + ShellPort& SH() const; + void log_call(const std::string& name, const json::Value& args, double started, const std::string& outcome, + const std::string& result); + bool secret_grant(const std::string& kind, const std::string& key, std::vector names); + // (vault, failure); vault nullopt on failure + std::pair>, std::string> resolve_secrets(const std::vector& names); + bool shell_is_safe(const std::string& command); + int snapshot_shell_targets(const std::string& command, const std::string& workdir); + std::unique_ptr shell_stream(const std::string& command); + AskFn shell_ask(const std::string& command); + std::pair box_engine(); // (engine, failure) + std::pair, std::string> box_read_raw(const std::string& path, int64_t limit = 200000); + std::string box_read(const std::string& path); + struct BoxOld { std::optional text; std::optional size; }; + BoxOld box_old_image(const std::string& path); + // old: nullopt-outer means "read the old image"; use box_write_old for explicit old. + std::string box_write(const std::string& path, const std::string& content, const std::optional>& old, + const std::string& action = "write", const std::string& message = "", bool preview = true); + std::string box_edit(const std::string& path, const std::string& find, const std::string& replace); + std::string box_delete(const std::string& path); + std::string box_restore(const std::string& path, const std::string& target, const std::string& message); + // audit_file(...) -> row id, nullopt when no store, -1 when the audit failed (Python False) + std::optional audit_file(const std::string& action, const std::string& path, const std::optional& old_text, + const std::optional& new_text, const std::string& message = "", + const std::vector& tags = {}, const std::optional& old_size = std::nullopt, + const std::optional& new_size = std::nullopt); + std::optional> record_tag_args(const json::Value& args); + + // ---- handlers (one per tool, in TOOL_SCHEMAS order) ---- + std::string run_shell(const json::Value& args); + std::string run_shell_box(const std::string& command, const std::string& workdir, const std::map& vault); + std::string run_read_file(const json::Value& args); + std::string run_write_file(const json::Value& args); + std::string run_edit_file(const json::Value& args); + std::string run_web_search(const json::Value& args); + std::string run_describe_image(const json::Value& args); + std::string run_web_fetch(const json::Value& args); + std::string run_speak(const json::Value& args); + std::string run_listen(const json::Value& args); + std::string run_remember(const json::Value& args); + std::string run_recall(const json::Value& args); + std::string run_load_skill(const json::Value& args); + std::string run_terminal_content(const json::Value& args); + std::string run_terminal_content_native(const std::string& socket_path, int lines, const std::string& history); + std::string run_list_terminals(const json::Value& args); + std::string run_tail_terminals(const json::Value& args); + std::string run_export_terminal_log(const json::Value& args); + std::string run_plexer_send(const json::Value& args); + std::string run_plexer_read(const json::Value& args); + std::string run_plexer_type(const json::Value& args); + std::string run_plexer_status(const json::Value& args); + std::string run_plexer_show(const json::Value& args); + std::string run_tui(const std::string& op, const json::Value& args); + std::string run_fork(const json::Value& args); + std::string run_poll(const json::Value& args); + std::string run_ask_user(const json::Value& args); + std::string run_sysinfo(const json::Value& args); + std::string run_create_skill(const json::Value& args); + std::string run_install_skill(const json::Value& args); + std::string run_store_secret(const json::Value& args); + std::string run_list_secrets(const json::Value& args); + std::string run_delete_secret(const json::Value& args); + std::string run_schedule(const json::Value& args); + std::string run_unschedule(const json::Value& args); + std::string run_schedules(const json::Value& args); + std::string run_record_save(const json::Value& args); + std::string run_record_read(const json::Value& args); + std::string run_record_search(const json::Value& args); + std::string run_record_delete(const json::Value& args); + std::string run_graph_link(const json::Value& args); + std::string run_graph_query(const json::Value& args); + std::string run_delete_file(const json::Value& args); + std::string run_audit(const json::Value& args); + std::string run_restore(const json::Value& args); + std::string run_release(const json::Value& args); + std::string run_tags(const json::Value& args); + std::string run_install(const json::Value& args); + std::string run_create_bot(const json::Value& args); + std::string run_search(const json::Value& args); + std::string run_mcp_add(const json::Value& args); + std::string run_mcp_list(const json::Value& args); + std::string run_mcp_enable(const json::Value& args); + std::string run_mcp_disable(const json::Value& args); + std::string run_mcp_remove(const json::Value& args); + std::string run_mcp_tools(const json::Value& args); + std::string run_mcp_call(const json::Value& args); + std::string run_mcp_login(const json::Value& args); + + // MCP helpers + std::string mcp_add_401(Value& entry, const std::string& profile, const std::vector& notes); + std::pair> mcp_resolve_github(const std::string& owner, const std::string& repo); + std::pair> mcp_resolve_agent(const std::string& owner, const std::string& repo, const std::string& profile); + // skills helpers + static std::optional find_skill_root(const std::string& base); + // (path, ephemeral) or failure text + std::pair>, std::string> resolve_skill_source(const std::string& source, const std::string& workdir); +}; + } // namespace detz::agent diff --git a/taicpp/src/agent/tools_core.cpp b/taicpp/src/agent/tools_core.cpp new file mode 100644 index 0000000..3950653 --- /dev/null +++ b/taicpp/src/agent/tools_core.cpp @@ -0,0 +1,687 @@ +#include "agent.h" +#include "../llm/helpers.h" +#include "../llm/proc.h" +#include "../llm/py.h" +#include "../llm/term.h" +#include "../llm/consts.h" +#include "consts.h" +#include "data.h" +#include "text.h" +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::agent { + +using llm::Ansi; +using llm::BackendError; +using llm::paint; +using llm::short_error; + +static std::string py_dirname(const std::string& p) { + size_t i = p.rfind('/'); + if (i == std::string::npos) { return ""; } + std::string head = p.substr(0, i + 1); + bool all_slash = true; + for (char c : head) { if (c != '/') { all_slash = false; } } + if (!head.empty() && !all_slash) { head = py::rstrip_chars(head, "/"); } + return head; +} + +Ports& ports() { + static Ports p; + return p; +} + +Tools::Tools(Agent& a) : app(a) {} + +StorePort& Tools::S(const char* method) const { + if (app.store == nullptr) { + throw std::runtime_error(std::string("'NoneType' object has no attribute '") + method + "'"); + } + return *app.store; +} +ShellPort& Tools::SH() const { + if (ports().shell == nullptr) { throw PortNotWired("shell"); } + return *ports().shell; +} + +const std::vector& Tools::handler_names() { + static const std::vector v = { + "shell", "read_file", "write_file", "edit_file", "web_search", "describe_image", "web_fetch", "speak", "listen", + "remember", "recall", "load_skill", "get_current_terminal_content", "list_terminals", "tail_terminals", + "export_terminal_log", "plexer_send", "plexer_read", "plexer_type", "plexer_status", "plexer_show", + "tui_screen", "tui_keys", "tui_type", "tui_click", "tui_goto", "tui_result", "fork", "poll", "ask_user", + "sysinfo", "create_skill", "install_skill", "store_secret", "list_secrets", "delete_secret", "schedule", + "unschedule", "schedules", "record_save", "record_read", "record_search", "record_delete", "graph_link", + "graph_query", "delete_file", "audit", "restore", "release", "tags", "install", "create_bot", "search", + "mcp_add", "mcp_list", "mcp_enable", "mcp_disable", "mcp_remove", "mcp_tools", "mcp_call", "mcp_login"}; + return v; +} + +const std::map& Tools::handler_table() { + static const std::map t = { + {"shell", &Tools::run_shell}, + {"read_file", &Tools::run_read_file}, + {"write_file", &Tools::run_write_file}, + {"edit_file", &Tools::run_edit_file}, + {"web_search", &Tools::run_web_search}, + {"describe_image", &Tools::run_describe_image}, + {"web_fetch", &Tools::run_web_fetch}, + {"speak", &Tools::run_speak}, + {"listen", &Tools::run_listen}, + {"remember", &Tools::run_remember}, + {"recall", &Tools::run_recall}, + {"load_skill", &Tools::run_load_skill}, + {"get_current_terminal_content", &Tools::run_terminal_content}, + {"list_terminals", &Tools::run_list_terminals}, + {"tail_terminals", &Tools::run_tail_terminals}, + {"export_terminal_log", &Tools::run_export_terminal_log}, + {"plexer_send", &Tools::run_plexer_send}, + {"plexer_read", &Tools::run_plexer_read}, + {"plexer_type", &Tools::run_plexer_type}, + {"plexer_status", &Tools::run_plexer_status}, + {"plexer_show", &Tools::run_plexer_show}, + {"fork", &Tools::run_fork}, + {"poll", &Tools::run_poll}, + {"ask_user", &Tools::run_ask_user}, + {"sysinfo", &Tools::run_sysinfo}, + {"create_skill", &Tools::run_create_skill}, + {"install_skill", &Tools::run_install_skill}, + {"store_secret", &Tools::run_store_secret}, + {"list_secrets", &Tools::run_list_secrets}, + {"delete_secret", &Tools::run_delete_secret}, + {"schedule", &Tools::run_schedule}, + {"unschedule", &Tools::run_unschedule}, + {"schedules", &Tools::run_schedules}, + {"record_save", &Tools::run_record_save}, + {"record_read", &Tools::run_record_read}, + {"record_search", &Tools::run_record_search}, + {"record_delete", &Tools::run_record_delete}, + {"graph_link", &Tools::run_graph_link}, + {"graph_query", &Tools::run_graph_query}, + {"delete_file", &Tools::run_delete_file}, + {"audit", &Tools::run_audit}, + {"restore", &Tools::run_restore}, + {"release", &Tools::run_release}, + {"tags", &Tools::run_tags}, + {"install", &Tools::run_install}, + {"create_bot", &Tools::run_create_bot}, + {"search", &Tools::run_search}, + {"mcp_add", &Tools::run_mcp_add}, + {"mcp_list", &Tools::run_mcp_list}, + {"mcp_enable", &Tools::run_mcp_enable}, + {"mcp_disable", &Tools::run_mcp_disable}, + {"mcp_remove", &Tools::run_mcp_remove}, + {"mcp_tools", &Tools::run_mcp_tools}, + {"mcp_call", &Tools::run_mcp_call}, + {"mcp_login", &Tools::run_mcp_login}, + }; + return t; +} + +void Tools::note_diff(const std::string& path, const std::optional& old_text, const std::optional& new_text) { + diff_preview = DiffPreview{path, old_text, new_text}; +} + +json::Value Tools::mcp_schemas() { + if (app.store == nullptr || ports().mcp == nullptr) { return json::Value::array(); } + try { + return ports().mcp->schemas_for(active_profile()); + } catch (const std::exception&) { // (sqlite3.Error, OSError, ValueError) + return json::Value::array(); + } +} + +std::string Tools::active_profile() const { + if (!app.profile.empty()) { return app.profile; } + if (app.store != nullptr && !app.store->profile().empty()) { return app.store->profile(); } + return "default"; +} + +std::string Tools::audit_actor() const { + if (app.persist && app.depth == 0) { return "main"; } + return "worker"; +} + +// ------------------------------------------------------------------ dispatch +std::string Tools::dispatch(const std::string& name, const json::Value& raw_arguments) { + calls += 1; + double started = py::now(); + json::Value args; + if (raw_arguments.is_object()) { + args = raw_arguments; + } else { + // json.loads(raw_arguments or "{}") + try { + if (!raw_arguments.truthy()) { + args = json::Value::object(); + } else if (raw_arguments.is_string()) { + args = json::Value::loads(raw_arguments.str()); + } else { + throw std::invalid_argument("TypeError"); + } + } catch (const std::exception&) { + log_call(name, raw_arguments, started, "bad_args", "error: arguments are not valid JSON"); + return "error: arguments are not valid JSON"; + } + } + const auto& table = handler_table(); + auto it = table.find(name); + bool is_tui = name == "tui_screen" || name == "tui_keys" || name == "tui_type" || name == "tui_click" || + name == "tui_goto" || name == "tui_result"; + bool known = it != table.end() || is_tui; + std::optional> mcp_target; + if (!known && py::startswith(name, MCP_TOOL_PREFIX) && ports().mcp != nullptr) { + mcp_target = ports().mcp->resolve(name); + } + if (!known && !mcp_target.has_value()) { + log_call(name, args, started, "unknown_tool", "error: unknown tool " + name); + return "error: unknown tool " + name; + } + std::string result; + try { + // A decoded non-object only works for handlers that never read args. + static const std::set ignores_args = {"list_terminals", "schedules", "list_secrets", "mcp_list", "plexer_status"}; + if (!args.is_object() && !mcp_target.has_value() && !ignores_args.count(name)) { + if (is_tui) { throw std::runtime_error("'" + py_type_name(args) + "' object has no attribute 'items'"); } + throw std::runtime_error("'" + py_type_name(args) + "' object has no attribute 'get'"); + } + if (mcp_target.has_value()) { + if (ports().mcp == nullptr) { throw PortNotWired("mcp"); } + result = ports().mcp->call(mcp_target->first, mcp_target->second, args, active_profile()); + } else if (is_tui) { + std::string op = name == "tui_screen" ? "screen" : name == "tui_keys" ? "key" : name == "tui_type" ? "type" + : name == "tui_click" ? "click" : name == "tui_goto" ? "goto" : "result"; + result = run_tui(op, args); + } else { + json::Value use = args.is_object() ? args : json::Value::object(); + result = (this->*(it->second))(use); + } + } catch (const Denied&) { + log_call(name, args, started, "denied", "denied"); + throw; + } catch (const std::exception& exc) { + std::string text = "error: " + short_error(exc); + log_call(name, args, started, "error", text); + return text; + } + if (app.store != nullptr) { result = app.store->redact(result, active_profile()); } + log_call(name, args, started, "ok", result); + return result; +} + +void Tools::log_call(const std::string& name, const json::Value& args, double started, const std::string& outcome, + const std::string& result) { + if (!llm::log_enabled()) { return; } + int64_t exec_ms = static_cast((py::now() - started) * 1000); + json::Value shown; + static const std::set masked = {"value", "password", "passphrase", "token", "secret", "key", "auth", "authorization"}; + if (args.is_object()) { + shown = json::Value::object(); + for (const auto& kv : args.members()) { + shown.set(kv.first, masked.count(py::lower(kv.first)) ? json::Value("***") : kv.second); + } + } else { + shown = args.is_string() ? args : json::Value(llm::short_error_text(py::repr(args))); + } + std::string message; + if (outcome == "ok") { message = "tool " + name + " finished in " + std::to_string(exec_ms) + "ms"; } + else if (outcome == "denied") { message = "tool " + name + " denied in " + std::to_string(exec_ms) + "ms"; } + else if (outcome == "unknown_tool") { message = "tool " + name + " unknown"; } + else if (outcome == "bad_args") { message = "tool " + name + " got bad arguments"; } + else { message = "tool " + name + " failed in " + std::to_string(exec_ms) + "ms: " + llm::short_error_text(result); } + json::Value meta = json::Value::object(); + meta.set("name", name); + meta.set("args", shown); + meta.set("exec_ms", exec_ms); + meta.set("outcome", outcome); + meta.set("result_chars", static_cast(outcome == "ok" ? py::len(result) : 0)); + if (outcome == "error") { meta.set("error", llm::short_error_text(result)); } + llm::log_event("INFO", "tool", "call", message, active_profile(), app.bot, meta); +} + +bool Tools::secret_grant(const std::string& kind, const std::string& key, std::vector names) { + std::sort(names.begin(), names.end()); + SecretGrant token{kind, key, names}; + if (secret_grants.count(token)) { return true; } + if (!app.ask_approval("share secrets (" + py::join(names, ", ") + ") with " + kind)) { return false; } + secret_grants.insert(token); + return true; +} + +std::pair>, std::string> Tools::resolve_secrets(const std::vector& names) { + std::map vault; + for (const auto& name : names) { + auto value = S("load_secret").load_secret(name, active_profile()); + if (!value.has_value()) { return {std::nullopt, "error: unknown secret '" + name + "'"}; } + vault[name] = *value; + } + return {vault, ""}; +} + +bool Tools::shell_is_safe(const std::string& command) { + auto parts = py::split_ws(command); + if (parts.empty()) { return false; } + if (parts[0] == "git") { + static const std::set ok = {"status", "diff", "log", "show", "branch", "remote"}; + return parts.size() > 1 && ok.count(parts[1]) > 0; + } + const auto& safe = SafeCommands(); + return std::find(safe.begin(), safe.end(), parts[0]) != safe.end(); +} + +int Tools::snapshot_shell_targets(const std::string& command, const std::string& workdir) { + if (app.store == nullptr) { return 0; } + std::string profile = active_profile(); + int count = 0; + for (const auto& found : SH().shell_target_paths(command, workdir)) { + if (!isfile(found)) { continue; } + Capped c = read_capped(found); + if (!c.text.has_value()) { continue; } + try { + app.store->audit_event(audit_actor(), "shell-snapshot", found, py::head(command, 200), c.text, std::nullopt, + {"shell"}, c.size, std::nullopt, profile); + app.store->upsert_file_record(found, *c.text, c.size, profile); + count += 1; + } catch (const std::exception&) { + continue; + } + } + return count; +} + +std::unique_ptr Tools::shell_stream(const std::string& command) { + if (app.stream_factory) { + return app.stream_factory(app.store != nullptr ? app.store->redact(command, active_profile()) : command); + } + if (live && llm::stdout_isatty() && ports().shell != nullptr) { return ports().shell->make_live_stream(); } + return nullptr; +} + +AskFn Tools::shell_ask(const std::string& command) { + Agent* a = &app; + if (a->auto_mode || !a->persist || a->depth > 0) { return nullptr; } + auto answered = [a](std::optional value) { + if (value.has_value()) { a->user_stepped_in = true; } + return value; + }; + if (a->prompter) { + std::string cmd60 = py::head(command, 60); + return [a, cmd60, answered](const std::string& text, bool) { + return answered(a->prompter("ask", cmd60 + " asks:\n" + text, {})); + }; + } + if (!(llm::stdin_isatty() && llm::stdout_isatty())) { return nullptr; } + return [answered](const std::string& text, bool secret) -> std::optional { + std::string prompt = paint("\xC2\xBB " + text + " ", Ansi::YELLOW); + std::fputs(prompt.c_str(), stdout); + std::fflush(stdout); + struct termios old_t; + bool hid = false; + if (secret && tcgetattr(0, &old_t) == 0) { + struct termios t = old_t; + t.c_lflag &= ~static_cast(ECHO); + tcsetattr(0, TCSAFLUSH, &t); + hid = true; + } + std::string line; + bool got = static_cast(std::getline(std::cin, line)); + if (hid) { + tcsetattr(0, TCSAFLUSH, &old_t); + std::fputs("\n", stdout); + } + if (!got) { return answered(std::nullopt); } + return answered(line); + }; +} + +std::optional Tools::audit_file(const std::string& action, const std::string& path, const std::optional& old_text, + const std::optional& new_text, const std::string& message, + const std::vector& tags, const std::optional& old_size, + const std::optional& new_size) { + if (app.store == nullptr) { return std::nullopt; } + std::string profile = active_profile(); + int64_t row_id; + try { + std::vector all = {"file", action}; + all.insert(all.end(), tags.begin(), tags.end()); + row_id = app.store->audit_event(audit_actor(), action, path, message, old_text, new_text, all, old_size, new_size, profile); + } catch (const std::exception&) { + return -1; + } + try { + if (action == "delete") { + app.store->delete_file_record(path, profile); + } else if (new_text.has_value()) { + app.store->upsert_file_record(path, *new_text, new_size, profile); + } else if (old_text.has_value()) { + app.store->upsert_file_record(path, *old_text, old_size, profile); + } + } catch (const std::exception&) { + } + return row_id; +} + +// ------------------------------------------------------------------ shell +std::string Tools::run_shell(const json::Value& args) { + std::string command = py::strip(py::str_or_empty(args.get("command"))); + if (command.empty()) { return "error: empty command"; } + json::Value secret_names = args.get("secrets").truthy() ? args.get("secrets") : json::Value::array(); + std::vector names; + if (!secret_names.is_array()) { return "error: secrets must be a list of names"; } + for (const auto& item : secret_names.items()) { + if (!item.is_string()) { return "error: secrets must be a list of names"; } + names.push_back(item.str()); + } + std::map vault; + if (!names.empty()) { + auto [v, failure] = resolve_secrets(names); + if (!failure.empty()) { return failure; } + vault = *v; + } + if (!vault.empty()) { + std::vector keys; + for (const auto& kv : vault) { keys.push_back(kv.first); } + if (!secret_grant("shell", "", keys)) { return "denied by user"; } + } + if (app.env == "sandbox") { + return run_shell_box(command, args.get("workdir").truthy() ? py::str(args.get("workdir")) : "/", vault); + } + std::string workdir = args.get("workdir").truthy() ? py::str(args.get("workdir")) : fs::current_path().string(); + if (!app.config.auto_approve && !shell_is_safe(command)) { + bool approved; + try { + approved = app.ask_approval(command); + } catch (const Denied& denied) { + if (!Approval::is_affirmative(denied.guidance())) { throw; } + approved = true; + if (!app.quiet) { + llm::print_out(paint("taking " + py::repr_str(py::head(denied.guidance(), 80)) + " as approval", Ansi::YELLOW)); + } + } + if (!approved) { return "denied by user"; } + } + snapshot_shell_targets(command, workdir); + std::map extra; + for (const auto& kv : vault) { extra[secret_env_name(kv.first)] = kv.second; } + std::unique_ptr stream = shell_stream(command); + RunResult r = SH().run_live_shell(command, workdir, extra, stream.get(), shell_ask(command)); + if (stream) { stream->close(stream_status(r.code, r.output, r.elapsed, r.timed_out)); } + if (r.timed_out) { return "error: timed out after " + std::to_string(SHELL_TIMEOUT) + " seconds"; } + std::string shown = py::strip(r.output); + if (shown.empty()) { shown = "(no output)"; } + return "exit " + std::to_string(r.code) + "\n" + + spill_or_truncate(app.store, "output", "shell: " + py::head(command, 80), shown, {"shell"}, active_profile()) + + curl_hint(command); +} + +std::string Tools::run_shell_box(const std::string& command, const std::string& workdir, const std::map& vault) { + auto [engine, failure] = box_engine(); + if (engine.empty()) { return failure; } + std::vector extra = {"--workdir", workdir}; + for (const auto& kv : vault) { + extra.push_back("-e"); + extra.push_back(secret_env_name(kv.first) + "=" + kv.second); + } + std::unique_ptr stream; + RunResult r; + try { + stream = shell_stream(command); + AskFn ask = shell_ask(command); + std::vector argv = {engine, "exec"}; + argv.insert(argv.end(), extra.begin(), extra.end()); + argv.push_back(ask ? "-it" : "-i"); + argv.push_back(llm::kBoxName); + argv.push_back("sh"); + argv.push_back("-c"); + argv.push_back(command); + r = SH().run_live_argv(argv, stream.get(), ask); + } catch (const llm::BackendError&) { + throw; + } catch (const PortNotWired&) { + throw; + } catch (const std::exception& exc) { + return "error: " + short_error(exc); + } + if (stream) { stream->close(stream_status(r.code, r.output, r.elapsed, r.timed_out)); } + if (r.timed_out) { return "error: timed out after " + std::to_string(SHELL_TIMEOUT) + " seconds"; } + std::string shown = py::strip(r.output); + if (shown.empty()) { shown = "(no output)"; } + return "exit " + std::to_string(r.code) + "\n" + + spill_or_truncate(app.store, "output", "shell: " + py::head(command, 80), shown, {"shell"}, active_profile()) + + curl_hint(command); +} + +// ------------------------------------------------------------------ files +std::string Tools::run_read_file(const json::Value& args) { + std::string path = py::str_or_empty(args.get("path")); + if (app.env == "sandbox") { + std::string result = box_read(path); + if (!py::startswith(result, "error:")) { read_files.insert({"box", path}); } + return result; + } + if (path.empty() || !isfile(path)) { return "error: no such file"; } + std::error_code ec; + auto size = fs::file_size(path, ec); + if (!ec && size > 200000) { return "error: file too large"; } + std::string err; + auto text = read_text_file(path, &err); + if (!text.has_value()) { return "error: " + py::head(err, 300); } + std::string result = llm::truncate(*text, 8000, 2000); + read_files.insert({"home", abspath(path)}); + return result; +} + +std::pair Tools::box_engine() { + try { + return {SH().ensure_box(), ""}; + } catch (const BackendError& exc) { + return {"", "error: " + short_error(exc)}; + } +} + +static std::string proc_failure(const llm::ProcResult& r, const std::vector& argv, double timeout) { + if (r.timed_out) { return llm::timeout_text(argv, timeout); } + return llm::oserror_text(r, argv.empty() ? "" : argv[0]); +} + +std::pair, std::string> Tools::box_read_raw(const std::string& path, int64_t limit) { + auto [engine, failure] = box_engine(); + if (engine.empty()) { return {std::nullopt, failure}; } + std::vector argv = {"sh", "-c", "wc -c < \"$1\" && head -c \"$2\" \"$1\"", "box", path, std::to_string(limit + 1)}; + llm::ProcResult done = SH().box_exec(engine, argv, std::nullopt, 60); + if (done.spawn_failed || done.timed_out) { + return {std::nullopt, "error: " + llm::short_error_text(proc_failure(done, argv, 60))}; + } + if (done.returncode != 0) { return {std::nullopt, "error: no such file"}; } + std::string text = py::utf8_replace(done.out); + size_t nl = text.find('\n'); + std::string head = nl == std::string::npos ? text : text.substr(0, nl); + std::string rest = nl == std::string::npos ? "" : text.substr(nl + 1); + auto size = py::parse_int(head); + if (!size.has_value()) { return {std::nullopt, "error: unreadable file"}; } + return {size, rest}; +} + +std::string Tools::box_read(const std::string& path) { + if (path.empty()) { return "error: empty path"; } + auto [engine, failure] = box_engine(); + if (engine.empty()) { return failure; } + std::vector argv = {"head", "-c", "200000", path}; + llm::ProcResult done = SH().box_exec(engine, argv, std::nullopt, 60); + if (done.spawn_failed || done.timed_out) { + return "error: " + llm::short_error_text(proc_failure(done, argv, 60)); + } + if (done.returncode != 0) { return "error: no such file"; } + return llm::truncate(py::utf8_replace(done.out), 8000, 2000); +} + +Tools::BoxOld Tools::box_old_image(const std::string& path) { + auto [size, raw] = box_read_raw(path, AUDIT_MAX_CHARS); + BoxOld o; + if (!size.has_value()) { return o; } + if (*size <= AUDIT_MAX_CHARS) { o.text = raw; return o; } + o.text = py::head(raw, AUDIT_MAX_CHARS); + o.size = size; + return o; +} + +std::string Tools::box_write(const std::string& path, const std::string& content, + const std::optional>& old_arg, const std::string& action, + const std::string& message, bool preview) { + auto [engine, failure] = box_engine(); + if (engine.empty()) { return failure; } + if (!read_files.count({"box", path})) { + llm::ProcResult probe = SH().box_exec(engine, {"test", "-f", path}, std::nullopt, 30); + if (!probe.spawn_failed && !probe.timed_out && probe.returncode == 0) { + return "error: write denied for '" + path + "', read it first with read_file"; + } + } + std::optional old_text; + std::optional old_size; + if (!old_arg.has_value()) { + BoxOld o = box_old_image(path); + old_text = o.text; + old_size = o.size; + } else { + old_text = *old_arg; + } + std::string parent = py_dirname(path); + if (parent.empty()) { parent = "."; } + std::vector argv = {"sh", "-c", "mkdir -p \"$1\" && cat > \"$2\"", "box", parent, path}; + llm::ProcResult done = SH().box_exec(engine, argv, content, 60); + if (done.spawn_failed || done.timed_out) { + return "error: " + llm::short_error_text(proc_failure(done, argv, 60)); + } + if (done.returncode != 0) { return "error: " + py::head(py::utf8_replace(done.err), 200); } + read_files.insert({"box", path}); + auto audited = audit_file(action, "sandbox:" + path, old_text, content, message, {}, old_size, std::nullopt); + std::string note = (audited.has_value() && *audited == -1) ? " (audit failed)" : ""; + if (preview && !old_size.has_value()) { note_diff(path, old_text, content); } + return "wrote " + path + note; +} + +std::string Tools::run_write_file(const json::Value& args) { + std::string path = py::str_or_empty(args.get("path")); + if (path.empty()) { return "error: empty path"; } + if (app.env == "sandbox") { + return box_write(path, py::str_or_empty(args.get("content")), std::nullopt); + } + std::pair key{"home", abspath(path)}; + if (isfile(path) && !read_files.count(key)) { + return "error: write denied for '" + path + "', read it first with read_file"; + } + std::string content = py::str_or_empty(args.get("content")); + Capped old; + if (isfile(path)) { old = read_capped(path); } + { + std::string parent = py_dirname(abspath(path)); + std::error_code ec; + fs::create_directories(parent, ec); + if (ec) { return "error: " + py::head(os_error_message(ec.value(), parent), 300); } + std::string err; + if (!write_text_file(path, content, err)) { return "error: " + py::head(err, 300); } + read_files.insert(key); + } + auto audited = audit_file("write", abspath(path), old.text, content, "", {}, old.size, std::nullopt); + std::string note = (audited.has_value() && *audited == -1) ? " (audit failed)" : ""; + if (!old.size.has_value()) { note_diff(path, old.text, content); } + return "wrote " + path + note; +} + +std::string Tools::run_edit_file(const json::Value& args) { + std::string path = py::str_or_empty(args.get("path")); + std::string find = py::str_or_empty(args.get("find")); + if (app.env == "sandbox") { + return box_edit(path, find, py::str_or_empty(args.get("replace"))); + } + if (path.empty() || !isfile(path)) { return "error: no such file"; } + if (find.empty()) { return "error: empty match"; } + if (!read_files.count({"home", abspath(path)})) { + return "error: edit denied for '" + path + "', read it first with read_file"; + } + std::string err; + auto content = read_text_file(path, &err); + if (!content.has_value()) { return "error: " + py::head(err, 300); } + long occurrences = py::count(*content, find); + if (occurrences != 1) { return "error: match is not unique (" + std::to_string(occurrences) + " occurrences)"; } + std::string updated = py::replace(*content, find, py::str_or_empty(args.get("replace"))); + if (!write_text_file(path, updated, err)) { return "error: " + py::head(err, 300); } + auto audited = audit_file("edit", abspath(path), *content, updated); + std::string note = (audited.has_value() && *audited == -1) ? " (audit failed)" : ""; + note_diff(path, *content, updated); + return "edited " + path + note; +} + +std::string Tools::box_edit(const std::string& path, const std::string& find, const std::string& replace) { + if (path.empty()) { return "error: empty path"; } + if (find.empty()) { return "error: empty match"; } + if (!read_files.count({"box", path})) { + return "error: edit denied for '" + path + "', read it first with read_file"; + } + auto [size, content] = box_read_raw(path); + if (!size.has_value()) { return content; } + if (*size > 200000) { return "error: file too large to edit safely (" + std::to_string(*size) + " bytes)"; } + long occurrences = py::count(content, find); + if (occurrences != 1) { return "error: match is not unique (" + std::to_string(occurrences) + " occurrences)"; } + std::string written = box_write(path, py::replace(content, find, replace), std::optional>(content), "edit"); + if (py::startswith(written, "wrote ")) { + return "edited " + path + written.substr(std::string("wrote " + path).size()); + } + return written; +} + +std::string Tools::run_delete_file(const json::Value& args) { + std::string path = py::str_or_empty(args.get("path")); + if (path.empty()) { return "error: empty path"; } + if (app.env == "sandbox") { return box_delete(path); } + if (!isfile(path)) { return "error: no such file"; } + if (!read_files.count({"home", abspath(path)})) { + return "error: delete denied for '" + path + "', read it first with read_file"; + } + if (!app.ask_approval("delete file " + path)) { return "denied by user"; } + Capped old = read_capped(path); + if (::unlink(path.c_str()) != 0) { return "error: " + py::head(os_error_message(errno, path), 300); } + read_files.erase({"home", abspath(path)}); + auto audited = audit_file("delete", abspath(path), old.text, std::nullopt, "", {}, old.size, std::nullopt); + std::string note = (audited.has_value() && *audited == -1) ? " (audit failed)" : ""; + if (!old.size.has_value()) { note_diff(path, old.text, std::nullopt); } + return "deleted " + path + note; +} + +std::string Tools::box_delete(const std::string& path) { + auto [engine, failure] = box_engine(); + if (engine.empty()) { return failure; } + if (!read_files.count({"box", path})) { + llm::ProcResult probe = SH().box_exec(engine, {"test", "-f", path}, std::nullopt, 30); + if (probe.spawn_failed || probe.timed_out || probe.returncode != 0) { return "error: no such file"; } + return "error: delete denied for '" + path + "', read it first with read_file"; + } + BoxOld old = box_old_image(path); + if (!old.text.has_value()) { return "error: no such file"; } + std::vector argv = {"rm", "-f", path}; + llm::ProcResult done = SH().box_exec(engine, argv, std::nullopt, 60); + if (done.spawn_failed || done.timed_out) { return "error: " + llm::short_error_text(proc_failure(done, argv, 60)); } + if (done.returncode != 0) { return "error: " + py::head(py::utf8_replace(done.err), 200); } + read_files.erase({"box", path}); + auto audited = audit_file("delete", "sandbox:" + path, old.text, std::nullopt, "", {}, old.size, std::nullopt); + std::string note = (audited.has_value() && *audited == -1) ? " (audit failed)" : ""; + if (!old.size.has_value()) { note_diff(path, old.text, std::nullopt); } + return "deleted " + path + note; +} + +std::string Tools::box_restore(const std::string& path, const std::string& target, const std::string& message) { + read_files.insert({"box", path}); + BoxOld old = box_old_image(path); + std::string written = box_write(path, target, std::optional>(std::optional()), "restore", message, false); + if (py::startswith(written, "wrote ")) { + if (!old.size.has_value()) { note_diff(path, old.text, target); } + return "restored sandbox:" + path + " (" + message + ")"; + } + return written; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/tools_mcp.cpp b/taicpp/src/agent/tools_mcp.cpp new file mode 100644 index 0000000..142a2bd --- /dev/null +++ b/taicpp/src/agent/tools_mcp.cpp @@ -0,0 +1,470 @@ +#include "agent.h" +#include "../llm/helpers.h" +#include "../llm/py.h" +#include "consts.h" +#include "data.h" +#include "text.h" +#include +#include +namespace detz::agent { + +using llm::short_error; + +namespace { +std::string sv(const json::Value& v) { return py::str(v); } +McpPort& mcp() { + if (ports().mcp == nullptr) { throw PortNotWired("mcp"); } + return *ports().mcp; +} +// MCP_RESOLVER_BRIEF % (owner, repo, ...) with 14 %s slots +std::string fill_brief(const std::string& owner, const std::string& repo) { + const std::string& tmpl = data::McpResolverBrief(); + std::string out; + int slot = 0; + for (size_t i = 0; i < tmpl.size(); ++i) { + if (tmpl[i] == '%' && i + 1 < tmpl.size() && tmpl[i + 1] == 's') { + out += (slot % 2 == 0) ? owner : repo; + ++slot; + ++i; + } else if (tmpl[i] == '%' && i + 1 < tmpl.size() && tmpl[i + 1] == '%') { + out += '%'; + ++i; + } else { + out += tmpl[i]; + } + } + return out; +} +// re.search(r"```json\s*(\{.*?\})\s*```", out, re.DOTALL) +std::optional fenced_json(const std::string& out) { + size_t pos = 0; + while ((pos = out.find("```json", pos)) != std::string::npos) { + size_t i = pos + 7; + while (i < out.size() && py::is_space_cp(static_cast(out[i]))) { ++i; } + if (i < out.size() && out[i] == '{') { + for (size_t j = out.find('}', i); j != std::string::npos; j = out.find('}', j + 1)) { + size_t k = j + 1; + while (k < out.size() && py::is_space_cp(static_cast(out[k]))) { ++k; } + if (out.compare(k, 3, "```") == 0) { return out.substr(i, j + 1 - i); } + } + } + pos += 1; + } + return std::nullopt; +} +} // namespace + +std::pair> Tools::mcp_resolve_github(const std::string& owner, const std::string& repo) { + std::vector notes; + std::string base = "https://api.github.com/repos/" + owner + "/" + repo + "/contents/"; + auto fetch = [&](const std::string& path) -> std::optional { + json::Value a = json::Value::object(); + a.set("url", base + path); + a.set("raw", true); + std::string out = run_web_fetch(a); + if (py::startswith(out, "error:") || py::len(out) > 200000) { return std::nullopt; } + json::Value data; + try { + data = json::Value::loads(out); + } catch (const json::ParseError&) { + return std::nullopt; + } + if (!data.is_object() || !(data.get("type").is_string() && data.get("type").str() == "file") || !data.get("content").truthy()) { + return std::nullopt; + } + // base64.b64decode(data["content"]).decode("utf-8", "replace") + std::string b64 = sv(data.get("content")); + std::string raw; + int val = 0, bits = -8; + bool bad = false; + for (unsigned char c : b64) { + int d; + if (c >= 'A' && c <= 'Z') { d = c - 'A'; } + else if (c >= 'a' && c <= 'z') { d = c - 'a' + 26; } + else if (c >= '0' && c <= '9') { d = c - '0' + 52; } + else if (c == '+') { d = 62; } + else if (c == '/') { d = 63; } + else if (c == '=' || c == '\n') { continue; } + else { bad = true; break; } + val = (val << 6) + d; + bits += 6; + if (bits >= 0) { raw += static_cast((val >> bits) & 0xFF); bits -= 8; } + } + if (bad) { return std::nullopt; } + return py::utf8_replace(raw); + }; + for (const char* candidate : {"mcp.json", ".mcp.json", "server.json"}) { + auto text = fetch(candidate); + if (text.has_value() && !text->empty()) { + notes.push_back(std::string("found ") + candidate + ", normalizing it"); + auto [entry, sub] = mcp().normalize(*text, std::nullopt); + notes.insert(notes.end(), sub.begin(), sub.end()); + return {entry, notes}; + } + } + auto package = fetch("package.json"); + if (package.has_value() && !package->empty()) { + json::Value meta; + try { + meta = json::Value::loads(*package); + } catch (const json::ParseError&) { + meta = json::Value::object(); + } + if (meta.is_object()) { + std::string name = py::str_or_empty(meta.get("name")); + if (!name.empty()) { + notes.push_back("package " + name + " has no run contract, trying npx"); + json::Value e = json::Value::object(); + e.set("name", mcp().sanitize_name(repo)); + e.set("transport", "stdio"); + e.set("command", "npx"); + json::Value a = json::Value::array(); + a.push("-y"); + a.push(name); + e.set("args", a); + e.set("env", json::Value::object()); + e.set("url", ""); + e.set("headers", json::Value::object()); + e.set("cwd", ""); + e.set("allow", json::Value::array()); + e.set("deny", json::Value::array()); + e.set("enabled", json::Value()); + e.set("source", "github:" + owner + "/" + repo); + return {e, notes}; + } + } + } + for (const char* candidate : {"README.md", "readme.md"}) { + auto text = fetch(candidate); + if (text.has_value() && !text->empty()) { + // re.search(r"(npx|uvx)\s+([^\s\"'`]+)", text) + static const std::regex re("(npx|uvx)\\s+([^\\s\"'`]+)"); + std::smatch m; + if (std::regex_search(*text, m, re)) { + notes.push_back(std::string("run block found in ") + candidate); + json::Value e = json::Value::object(); + e.set("name", mcp().sanitize_name(repo)); + e.set("transport", "stdio"); + e.set("command", m[1].str()); + json::Value a = json::Value::array(); + if (m[1].str() == "npx") { a.push("-y"); } + a.push(m[2].str()); + e.set("args", a); + e.set("env", json::Value::object()); + e.set("url", ""); + e.set("headers", json::Value::object()); + e.set("cwd", ""); + e.set("allow", json::Value::array()); + e.set("deny", json::Value::array()); + e.set("enabled", json::Value()); + e.set("source", "github:" + owner + "/" + repo); + return {e, notes}; + } + } + } + throw McpError("no run contract found (checked mcp.json, server.json, package.json, README)"); +} + +std::pair> Tools::mcp_resolve_agent(const std::string& owner, const std::string& repo, + const std::string& profile) { + if (app.depth >= FORK_MAX_DEPTH) { throw McpError("at fork depth limit, cannot spawn resolver"); } + std::string brief = fill_brief(owner, repo); + std::string out; + try { + out = default_runner(app.config, profile, app.depth + 1, brief, MCP_RESOLVER_TIMEOUT, MCP_RESOLVER_STEPS); + } catch (const std::exception& exc) { + throw McpError("resolver subagent failed: " + short_error(exc)); + } + std::string body; + auto fenced = fenced_json(out); + if (fenced.has_value()) { + body = *fenced; + } else { + size_t a = out.find('{'), b = out.rfind('}'); + if (a == std::string::npos || b == std::string::npos || b < a) { + throw McpError("resolver returned no JSON (first 200 chars: " + py::repr_str(py::head(out, 200)) + ")"); + } + body = out.substr(a, b + 1 - a); + } + json::Value data; + try { + data = json::Value::loads(body); + } catch (const json::ParseError&) { + throw McpError("resolver returned bad JSON: " + py::head(body, 200)); + } + if (!data.is_object()) { throw McpError("resolver returned non-object JSON"); } + if (data.get("error").truthy()) { + std::string ev = data.has("evidence") ? sv(data.get("evidence")) : "?"; + throw McpError("resolver: " + sv(data.get("error")) + " (checked: " + ev + ")"); + } + auto [entry, sub] = mcp().normalize(data.dumps(), mcp().sanitize_name(repo)); + std::vector notes = {"resolver subagent settled it"}; + notes.insert(notes.end(), sub.begin(), sub.end()); + return {entry, notes}; +} + +std::string Tools::run_mcp_add(const json::Value& args) { + std::string source = py::strip(py::str_or_empty(args.get("source"))); + std::string profile = args.get("profile").truthy() ? sv(args.get("profile")) : active_profile(); + if (source.empty()) { return "error: source must be pasted JSON/TOML config or a github URL"; } + std::optional> target; + if (source.find('\n') == std::string::npos && source.find('{') == std::string::npos && source.find('[') == std::string::npos) { + target = mcp().github_target(source); + } + std::vector notes; + Value entry; + try { + if (target.has_value()) { + try { + auto r = mcp_resolve_github(target->first, target->second); + entry = r.first; + notes = r.second; + } catch (const McpError& first) { + try { + auto r = mcp_resolve_agent(target->first, target->second, profile); + entry = r.first; + notes = {std::string("deterministic pass failed (") + first.what() + "), resolver subagent took over"}; + notes.insert(notes.end(), r.second.begin(), r.second.end()); + } catch (const McpError& second) { + throw McpError(std::string(first.what()) + "; resolver subagent also failed: " + second.what()); + } + } + entry.set("source", "github:" + target->first + "/" + target->second); + } else { + auto [text, rewritten] = mcp().rewrite_placeholders(source); + notes.insert(notes.end(), rewritten.begin(), rewritten.end()); + std::optional nm; + if (args.get("name").truthy()) { nm = sv(args.get("name")); } + auto [e, sub] = mcp().normalize(text, nm); + entry = e; + notes.insert(notes.end(), sub.begin(), sub.end()); + } + } catch (const McpError& exc) { + if (target.has_value()) { + return "error: could not resolve github " + target->first + "/" + target->second + ": " + exc.what() + "\n" + "finish it by hand: read the repo README for the run command, then call mcp_add " + "with explicit JSON like {\"mcpServers\": {\"name\": {\"command\": \"npx\", \"args\": [...]}}}"; + } + return std::string("error: ") + exc.what(); + } + std::string name = args.get("name").truthy() ? sv(args.get("name")) : py::str_or_empty(entry.get("name")); + if (!mcp_server_ok(name)) { return "error: bad server name '" + name + "'"; } + entry.set("name", name); + std::string prompt; + if (sv(entry.get("transport")) == "stdio") { + std::vector parts = {sv(entry.get("command"))}; + for (const auto& a : entry.get("args").items()) { parts.push_back(sv(a)); } + std::string shown = py::join(parts, " "); + std::vector flags; + for (const char* word : {"sudo", "rm -rf", "curl", "wget", "ssh", "nc "}) { + if (shown.find(word) != std::string::npos) { flags.push_back(word); } + } + prompt = "mcp add stdio '" + py::head(shown, 300) + "'" + (flags.empty() ? "" : " FLAGS: " + py::join(flags, ","))+ " (runs with your privileges)"; + } else { + prompt = "mcp add http " + py::head(sv(entry.get("url")), 300) + " (headers sealed, secrets resolve at call time)"; + } + if (!notes.empty()) { prompt += " [" + py::head(py::join(notes, "; "), 300) + "]"; } + if (!app.ask_approval(prompt)) { return "denied by user"; } + std::unique_ptr client; + try { + client = mcp().connect(entry, profile); + } catch (const McpHttpError& exc) { + if (exc.code == 401) { return mcp_add_401(entry, profile, notes); } + return std::string("error: verify failed, nothing stored: ") + exc.what(); + } catch (const McpError& exc) { + return std::string("error: verify failed, nothing stored: ") + exc.what(); + } + Value tools; + try { + try { + tools = client->list_tools(); + } catch (const McpHttpError& exc) { + client->close(); + if (exc.code == 401) { return mcp_add_401(entry, profile, notes); } + return std::string("error: verify failed, nothing stored: ") + exc.what(); + } catch (const McpError& exc) { + client->close(); + return std::string("error: verify failed, nothing stored: ") + exc.what(); + } + } catch (...) { + client->close(); + throw; + } + client->close(); + StorePort& st = S("mcp_save_server"); + try { + st.mcp_save_server(entry); + } catch (const std::invalid_argument& exc) { + return std::string("error: ") + exc.what(); + } + st.mcp_set_enabled(name, profile, entry.get("enabled").is_null() ? true : entry.get("enabled").truthy()); + st.mcp_store_tools(name, tools); + std::vector lines = {"added MCP server '" + name + "' (" + sv(entry.get("transport")) + ", " + + std::to_string(tools.size()) + " tools, enabled for " + profile + ")"}; + size_t shown = 0; + for (const auto& item : tools.items()) { + if (shown++ >= static_cast(MCP_MAX_TOOLS)) { break; } + lines.push_back("- " + sv(item.get("name").is_null() ? json::Value("") : item.get("name")) + ": " + + py::head(py::str_or_empty(item.get("description")), 120)); + } + if (tools.size() > static_cast(MCP_MAX_TOOLS)) { + lines.push_back("- ...and " + std::to_string(tools.size() - static_cast(MCP_MAX_TOOLS)) + " more (cap " + + std::to_string(MCP_MAX_TOOLS) + " offered)"); + } + return py::join(lines, "\n"); +} + +std::string Tools::mcp_add_401(Value& entry, const std::string& profile, const std::vector& notes) { + if (sv(entry.get("transport")) != "http") { return "error: verify failed, nothing stored: http 401"; } + auto tokens = mcp().oauth_load_tokens(sv(entry.get("name")), profile); + if (!tokens.first.empty()) { + return "error: verify failed: stored OAuth tokens refused, run mcp_login '" + sv(entry.get("name")) + "'"; + } + for (const auto& kv : entry.get("headers").members()) { + if (py::lower(kv.first) == "authorization") { return "error: verify failed: http 401, check the configured credentials"; } + } + entry.set("enabled", false); + StorePort& st = S("mcp_save_server"); + try { + st.mcp_save_server(entry); + } catch (const std::invalid_argument& exc) { + return std::string("error: ") + exc.what(); + } + st.mcp_set_enabled(sv(entry.get("name")), profile, false); + std::vector lines = {"stored MCP server '" + sv(entry.get("name")) + "' disabled: it answered 401, so it needs OAuth login", + "run mcp_login '" + sv(entry.get("name")) + "' to approve in your browser, then it verifies and enables"}; + for (const auto& n : notes) { lines.push_back("(note: " + n + ")"); } + return py::join(lines, "\n"); +} + +std::string Tools::run_mcp_list(const json::Value&) { + StorePort& st = S("mcp_list_servers"); + json::Value rows = st.mcp_list_servers(); + if (rows.size() == 0) { return "no MCP servers registered"; } + std::vector lines; + for (const auto& row : rows.items()) { + std::string detail; + if (sv(row.get("transport")) == "stdio") { + std::vector parts = {sv(row.get("command"))}; + for (const auto& a : row.get("args").items()) { parts.push_back(sv(a)); } + detail = py::head(py::join(parts, " "), 120); + } else { + detail = py::head(sv(row.get("url")), 120); + } + std::vector enabled = st.mcp_enabled_profiles(sv(row.get("name"))); + std::sort(enabled.begin(), enabled.end()); + json::Value cached = st.mcp_cached_tools(sv(row.get("name"))); + std::string src = row.get("source").truthy() ? sv(row.get("source")) : "?"; + lines.push_back(sv(row.get("name")) + " (" + sv(row.get("transport")) + ") " + detail + " | profiles: " + + (enabled.empty() ? std::string("none") : py::join(enabled, ", ")) + " | tools: " + std::to_string(cached.size()) + + " | from " + src); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_mcp_enable(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + std::string profile = args.get("profile").truthy() ? sv(args.get("profile")) : active_profile(); + StorePort& st = S("mcp_get_server"); + if (st.mcp_get_server(server).is_null()) { return "error: unknown MCP server '" + server + "'"; } + st.mcp_set_enabled(server, profile, true); + return "MCP server '" + server + "' enabled for " + profile; +} + +std::string Tools::run_mcp_disable(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + std::string profile = args.get("profile").truthy() ? sv(args.get("profile")) : active_profile(); + StorePort& st = S("mcp_get_server"); + if (st.mcp_get_server(server).is_null()) { return "error: unknown MCP server '" + server + "'"; } + st.mcp_set_enabled(server, profile, false); + return "MCP server '" + server + "' disabled for " + profile; +} + +std::string Tools::run_mcp_remove(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + StorePort& st = S("mcp_get_server"); + if (st.mcp_get_server(server).is_null()) { return "error: unknown MCP server '" + server + "'"; } + if (!app.ask_approval("mcp remove '" + server + "' (server, profile rows, pins, cached tools)")) { return "denied by user"; } + std::vector profiles = st.mcp_server_profiles(server); + st.mcp_remove_server(server); + std::set all(profiles.begin(), profiles.end()); + all.insert(active_profile()); + for (const auto& p : all) { mcp().oauth_drop_tokens(server, p); } + return "removed MCP server '" + server + "'"; +} + +std::string Tools::run_mcp_tools(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + StorePort& st = S("mcp_get_server"); + std::vector rows; + if (!server.empty()) { + json::Value r = st.mcp_get_server(server); + if (!r.is_null()) { rows.push_back(r); } + } else { + json::Value all = st.mcp_list_servers(); + for (const auto& r : all.items()) { if (!r.is_null()) { rows.push_back(r); } } + } + if (!server.empty() && rows.empty()) { return "error: unknown MCP server '" + server + "'"; } + if (rows.empty()) { return "no MCP servers registered"; } + std::vector lines; + for (const auto& row : rows) { + std::string name = sv(row.get("name")); + std::vector pins = st.mcp_tool_pins(name); + json::Value cached = st.mcp_cached_tools(name); + for (const auto& item : cached.items()) { + std::string tool = sv(item.get("tool")); + bool pinned = std::find(pins.begin(), pins.end(), tool) != pins.end(); + lines.push_back(name + "/" + tool + (pinned ? "" : " (unpinned)") + ": " + py::head(py::str_or_empty(item.get("description")), 150)); + } + } + if (lines.empty()) { return "no cached tools (call mcp_add to verify)"; } + return py::join(lines, "\n"); +} + +std::string Tools::run_mcp_call(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + std::string tool = py::str_or_empty(args.get("tool")); + json::Value call_args = args.get("args").truthy() ? args.get("args") : json::Value::object(); + if (server.empty() || tool.empty()) { return "error: mcp_call needs server and tool"; } + return mcp().call(server, tool, call_args, active_profile()); +} + +std::string Tools::run_mcp_login(const json::Value& args) { + std::string server = py::str_or_empty(args.get("server")); + std::string profile = args.get("profile").truthy() ? sv(args.get("profile")) : active_profile(); + StorePort& st = S("mcp_get_server"); + json::Value row = st.mcp_get_server(server); + if (row.is_null()) { return "error: unknown MCP server '" + server + "'"; } + if (sv(row.get("transport")) != "http") { return "error: local stdio servers need no login"; } + if (!app.ask_approval("mcp login '" + server + "' (opens a browser, waits up to " + std::to_string(MCP_OAUTH_TIMEOUT) + "s)")) { + return "denied by user"; + } + std::string note; + try { + note = mcp().oauth_login(row, profile); + } catch (const McpError& exc) { + return std::string("error: ") + exc.what(); + } + std::unique_ptr client; + try { + client = mcp().connect(row, profile); + } catch (const McpError& exc) { + return std::string("error: login ok but verify failed: ") + exc.what(); + } + Value tools; + try { + tools = client->list_tools(); + } catch (const McpError& exc) { + client->close(); + return std::string("error: login ok but verify failed: ") + exc.what(); + } catch (...) { + client->close(); + throw; + } + client->close(); + st.mcp_set_enabled(server, profile, true); + st.mcp_store_tools(server, tools); + return note + "\nverified " + std::to_string(tools.size()) + " tools, enabled for " + profile; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/tools_memory.cpp b/taicpp/src/agent/tools_memory.cpp new file mode 100644 index 0000000..9bdd788 --- /dev/null +++ b/taicpp/src/agent/tools_memory.cpp @@ -0,0 +1,729 @@ +#include "agent.h" +#include "../llm/helpers.h" +#include "../llm/py.h" +#include "../llm/term.h" +#include "consts.h" +#include "data.h" +#include "text.h" +#include +#include +#include +#include +namespace detz::agent { + +using llm::BackendError; +using llm::short_error; + +namespace { +// `isinstance(x, list) and all(isinstance(i, str) ...)` -> the strings, or nullopt +std::optional> string_list(const json::Value& v) { + if (!v.is_array()) { return std::nullopt; } + std::vector out; + for (const auto& i : v.items()) { + if (!i.is_string()) { return std::nullopt; } + out.push_back(i.str()); + } + return out; +} +std::string sv(const json::Value& v) { return py::str(v); } +UiPort* ui() { return ports().ui; } +} // namespace + +std::optional> Tools::record_tag_args(const json::Value& args) { + json::Value raw = args.get("tags").truthy() ? args.get("tags") : json::Value::array(); + auto list = string_list(raw); + if (!list.has_value()) { return std::nullopt; } + return normalize_tags(*list); +} + +std::string Tools::run_remember(const json::Value& args) { + std::string instruction = py::strip(py::str_or_empty(args.get("instruction"))); + if (instruction.empty()) { return "error: empty instruction"; } + const std::string& current = app.system_message; + json::Value prompt = json::Value::array(); + { + json::Value sys = json::Value::object(); + sys.set("role", "system"); + sys.set("content", data::MergeSystem()); + prompt.push(sys); + json::Value usr = json::Value::object(); + usr.set("role", "user"); + usr.set("content", "CURRENT SYSTEM PROMPT:\n" + current + "\n\nMEMORY INSTRUCTION:\n" + instruction); + prompt.push(usr); + } + if (ui() != nullptr) { ui()->spinner_start("merging memory"); } + llm::ChatReply merged; + try { + merged = app.chat.complete(prompt, json::Value(), nullptr, true); + } catch (const BackendError& exc) { + if (ui() != nullptr) { ui()->spinner_stop(); } + return "error: memory merge failed: " + short_error(exc); + } catch (...) { + if (ui() != nullptr) { ui()->spinner_stop(); } + throw; + } + if (ui() != nullptr) { ui()->spinner_stop(); } + std::string text = py::strip(merged.content); + // re.sub(r"^```[a-zA-Z]*\n|\n```$", "", text).strip() + { + size_t start = 0; + if (py::startswith(text, "```")) { + size_t i = 3; + while (i < text.size() && std::isalpha(static_cast(text[i])) && static_cast(text[i]) < 0x80) { ++i; } + if (i < text.size() && text[i] == '\n') { start = i + 1; } + } + size_t end = text.size(); + if (end >= start + 4 && text.compare(end - 4, 4, "\n```") == 0) { end -= 4; } + text = py::strip(text.substr(start, end - start)); + } + if (py::len(text) < 20) { return "error: merged system message too short, rejected"; } + if (static_cast(py::len(text)) > SYSTEM_MAX_CHARS) { + return "error: merged system message exceeds " + std::to_string(SYSTEM_MAX_CHARS) + " chars, rejected"; + } + app.update_system(text); + return "system message updated for profile '" + app.profile + "' (" + std::to_string(py::len(current)) + " to " + + std::to_string(py::len(text)) + " chars)"; +} + +std::string Tools::run_recall(const json::Value& args) { + std::string query = py::strip(py::str_or_empty(args.get("query"))); + if (query.empty()) { return "error: empty query"; } + json::Value raw = args.get("tags").truthy() ? args.get("tags") : json::Value::array(); + auto tags = string_list(raw); + if (!tags.has_value()) { return "error: tags must be a list of names"; } + json::Value rows = S("search_events").search_events(active_profile(), query, *tags); + if (rows.size() == 0) { return "no memories matching " + query; } + std::vector lines; + for (const auto& row : rows.items()) { + lines.push_back("[" + sv(row.at(0)) + " " + sv(row.at(1)) + "/" + sv(row.at(2)) + "] " + py::head(sv(row.at(3)), 300)); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_load_skill(const json::Value& args) { + std::string name = py::strip(py::str_or_empty(args.get("name"))); + if (ports().skills == nullptr) { throw PortNotWired("skills"); } + const json::Value& skill = app.skills.get(name); + if (skill.is_null() || !app.skills.has(name)) { + auto blueprint = ports().skills->skill_blueprint(name); + if (!blueprint.has_value()) { + std::vector names; + for (const auto& kv : app.skills.members()) { names.push_back(kv.first); } + std::sort(names.begin(), names.end()); + std::string known = names.empty() ? "none" : py::join(names, ", "); + return "error: unknown skill, known: " + known; + } + json::Value a = json::Value::object(); + a.set("name", name); + a.set("brief", blueprint->get("brief")); + a.set("scope", blueprint->get("scope")); + return "building skill '" + name + "' from blueprint, this runs deep research and takes a while:\n" + run_create_skill(a); + } + std::vector parts = {py::strip(py::str(skill.get("body"))), ports().skills->skill_provenance_note(skill)}; + auto extras = ports().skills->skill_extra_files(py::str(skill.get("root"))); + if (!extras.empty()) { parts.push_back("bundled files:\n" + py::join(extras, "\n")); } + return py::join(parts, "\n\n"); +} + +std::string Tools::run_store_secret(const json::Value& args) { + std::string name = py::strip(py::str_or_empty(args.get("name"))); + std::string value = py::str_or_empty(args.get("value")); + if (!secret_ok(name)) { return "error: invalid secret name, use letters, digits, dot, underscore, hyphen"; } + if (value.empty()) { return "error: empty value"; } + std::string username = py::head(py::strip(py::str_or_empty(args.get("username"))), 128); + std::string host = py::head(py::strip(py::str_or_empty(args.get("host"))), 255); + int64_t port = 0; + if (!args.get("port").is_null() && !py::strip(py::str(args.get("port"))).empty()) { + auto p = py::to_int(args.get("port")); + if (!p.has_value()) { return "error: port must be a number"; } + port = *p; + if (!(1 <= port && port <= 65535)) { return "error: port must be 1-65535"; } + } + std::string notes = py::head(py::strip(py::str_or_empty(args.get("notes"))), 2000); + std::string expires = py::strip(py::str_or_empty(args.get("expires"))); + if (!expires.empty()) { + try { + expires = parse_schedule_at(expires); + } catch (const std::invalid_argument& exc) { + return "error: " + short_error(exc); + } + } + json::Value raw_tags = args.get("tags").truthy() ? args.get("tags") : json::Value::array(); + auto raw = string_list(raw_tags); + if (!raw.has_value()) { return "error: tags must be a list of names"; } + std::vector tags = normalize_tags(*raw); + json::Value meta = json::Value::object(); + if (!username.empty()) { meta.set("username", username); } + if (!host.empty()) { meta.set("host", host); } + if (port) { meta.set("port", port); } + if (!notes.empty()) { meta.set("notes", notes); } + if (!expires.empty()) { meta.set("expires", expires); } + S("save_secret").save_secret(name, value, meta, tags, active_profile()); + std::vector all = {"secret"}; + all.insert(all.end(), tags.begin(), tags.end()); + return "stored secret '" + name + "' (sealed; tags: " + py::join(all, ", ") + ")"; +} + +std::string Tools::run_list_secrets(const json::Value&) { + json::Value infos = S("list_secret_infos").list_secret_infos(active_profile()); + if (infos.size() == 0) { return "secrets: none"; } + std::string now = py::utc_now_iso(); + std::vector lines; + for (const auto& info : infos.items()) { + const json::Value& meta = info.get("meta"); + std::vector parts = {"- " + sv(info.get("name"))}; + if (info.get("tags").truthy()) { + std::vector tags; + for (const auto& t : info.get("tags").items()) { tags.push_back(sv(t)); } + parts.push_back("[" + py::join(tags, " ") + "]"); + } + std::string who; + if (meta.get("username").truthy() && meta.get("host").truthy()) { + who = sv(meta.get("username")) + "@" + sv(meta.get("host")); + } else if (meta.get("host").truthy()) { + who = sv(meta.get("host")); + } else if (meta.get("username").truthy()) { + who = sv(meta.get("username")); + } + if (!who.empty() && meta.get("port").truthy()) { who += ":" + sv(meta.get("port")); } + if (!who.empty()) { parts.push_back(who); } + if (meta.get("notes").truthy()) { parts.push_back(py::head(sv(meta.get("notes")), 60)); } + if (meta.get("expires").truthy()) { + std::string exp = sv(meta.get("expires")); + parts.push_back(exp <= now ? "(expired)" : "(expires " + local_display(exp) + ")"); + } + lines.push_back(py::join(parts, " ")); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_delete_secret(const json::Value& args) { + std::string name = py::strip(py::str_or_empty(args.get("name"))); + if (!S("load_secret").load_secret(name, active_profile()).has_value()) { return "error: unknown secret '" + name + "'"; } + if (!app.ask_approval("delete secret '" + name + "'")) { return "denied by user"; } + S("delete_secret").delete_secret(name, active_profile()); + return "deleted secret '" + name + "'"; +} + +std::string Tools::run_schedule(const json::Value& args) { + std::string prompt = py::strip(py::str_or_empty(args.get("prompt"))); + if (prompt.empty()) { return "error: empty prompt"; } + const json::Value& at_raw = args.get("at"); + const json::Value& every_raw = args.get("every"); + if (at_raw.is_null() == every_raw.is_null()) { return "error: pass exactly one of at or every"; } + std::string next_run; + int64_t every_sec = 0; + if (!at_raw.is_null()) { + try { + next_run = parse_schedule_at(sv(at_raw)); + } catch (const std::invalid_argument& exc) { + return "error: " + short_error(exc); + } + } else { + auto e = py::to_int(every_raw); + if (!e.has_value()) { return "error: every must be seconds"; } + every_sec = *e; + if (every_sec < 60) { return "error: every must be at least 60 seconds"; } + next_run = py::utc_iso(py::now() + static_cast(every_sec), true); + } + std::string name = py::head(py::strip(py::str_or_empty(args.get("name"))), 80); + int64_t timeout; + { + json::Value t = args.get("timeout").truthy() ? args.get("timeout") : json::Value(600); + auto parsed = py::to_int(t); + timeout = parsed.has_value() ? std::max(60, std::min(3600, *parsed)) : 600; + } + std::string asked = py::strip(py::str_or_empty(args.get("profile"))); + if (!asked.empty() && !profile_ok(asked)) { return "error: invalid profile name"; } + if (!asked.empty() && asked != active_profile()) { return "error: cannot schedule for another profile"; } + std::string profile = active_profile(); + json::Value raw_tags = args.get("tags").truthy() ? args.get("tags") : json::Value::array(); + auto tags = string_list(raw_tags); + if (!tags.has_value()) { return "error: tags must be a list of names"; } + if (!app.ask_approval("schedule '" + (name.empty() ? py::head(prompt, 60) : name) + "'")) { return "denied by user"; } + int64_t row_id = S("add_schedule").add_schedule(name, prompt, profile, every_sec, next_run, timeout, normalize_tags(*tags)); + std::string when = every_sec ? "every " + format_delay(every_sec) : "at " + local_display(next_run); + return "scheduled #" + std::to_string(row_id) + " (" + when + ")"; +} + +std::string Tools::run_unschedule(const json::Value& args) { + json::Value idv = args.get("id").truthy() ? args.get("id") : json::Value(0); + auto id = py::to_int(idv); + if (!id.has_value()) { return "error: invalid schedule id"; } + int64_t row_id = *id; + json::Value items = S("list_schedules").list_schedules(active_profile()); + bool found = false; + for (const auto& item : items.items()) { if (item.get("id").as_int() == row_id) { found = true; } } + if (!found) { return "error: no schedule #" + std::to_string(row_id); } + if (!app.ask_approval("delete schedule #" + std::to_string(row_id))) { return "denied by user"; } + S("remove_schedule").remove_schedule(row_id, active_profile()); + return "deleted schedule #" + std::to_string(row_id); +} + +std::string Tools::run_schedules(const json::Value&) { + return format_schedule_lines(S("list_schedules").list_schedules(active_profile()), py::now()); +} + +std::string Tools::run_record_save(const json::Value& args) { + std::string content = py::str_or_empty(args.get("content")); + if (content.empty()) { return "error: content is empty"; } + std::string title = py::head(py::strip(py::str_or_empty(args.get("title"))), 200); + if (title.empty()) { title = "(untitled)"; } + std::string kind = py::lower(py::strip(args.get("kind").truthy() ? py::str(args.get("kind")) : "note")); + if (kind.empty()) { kind = "note"; } + const auto& kinds = RecordKinds(); + if (std::find(kinds.begin(), kinds.end(), kind) == kinds.end()) { + return "error: unknown kind " + kind + " (valid: " + py::join(kinds, ", ") + ")"; + } + auto tags = record_tag_args(args); + if (!tags.has_value()) { return "error: tags must be a list of names"; } + StorePort& st = S("add_record"); + std::string record_id = st.add_record(kind, title, content, *tags, active_profile()); + auto chunks = chunk_lines(content); + st.add_chunks("record", record_id, chunks, active_profile()); + std::string saved = "saved " + record_id + " (" + std::to_string(py::len(content)) + " chars, kind " + kind + ", " + + std::to_string(chunks.size()) + " chunks, tags: " + py::join(st.item_tags(record_id, active_profile()), ", ") + ")"; + std::vector links; + json::Value edges = st.edges_for(record_id, active_profile()); + for (const auto& edge : edges.items()) { + if (edge.get("direction").is_string() && edge.get("direction").str() == "out") { links.push_back(sv(edge.get("other"))); } + } + std::sort(links.begin(), links.end()); + if (!links.empty()) { saved += " [linked: " + py::join(links, ", ") + "]"; } + return saved; +} + +std::string Tools::run_create_bot(const json::Value& args) { + std::string name = py::lower(py::strip(py::str_or_empty(args.get("name")))); + if (!bot_ok(name)) { return "error: invalid bot name (lowercase letters, digits, - and _)"; } + if (name == "main") { return "error: 'main' is the default bot"; } + std::string description = py::strip(py::str_or_empty(args.get("description"))); + std::string rules = py::strip(py::str_or_empty(args.get("rules"))); + std::string behavior = py::strip(py::str_or_empty(args.get("behavior"))); + if (description.empty() && rules.empty() && behavior.empty()) { + return "error: give at least one of description, rules, or behavior"; + } + json::Value raw_nicks_v = args.get("nicknames").truthy() ? args.get("nicknames") : json::Value::array(); + auto raw_nicks = string_list(raw_nicks_v); + if (!raw_nicks.has_value()) { return "error: nicknames must be a list of names"; } + std::string profile = active_profile(); + StorePort& st = S("list_bots"); + std::set taken = {"main"}; + json::Value bots_list = st.list_bots(profile); + for (const auto& item : bots_list.items()) { + taken.insert(sv(item.get("name"))); + for (const auto& n : item.get("nicknames").items()) { taken.insert(sv(n)); } + } + std::string shortname = py::split(py::split(name, "-")[0], "_")[0]; + { + // re.split(r"[-_]", name, 1)[0]: up to the first '-' or '_' + size_t cut = name.find_first_of("-_"); + shortname = cut == std::string::npos ? name : name.substr(0, cut); + } + std::vector nicks; + std::vector cands = *raw_nicks; + cands.push_back(shortname); + for (const auto& cand : cands) { + std::string clean = py::lower(py::strip(cand)); + if (!clean.empty() && clean != name && bot_ok(clean) && std::find(nicks.begin(), nicks.end(), clean) == nicks.end()) { + nicks.push_back(clean); + } + } + for (const auto& cand : *raw_nicks) { + std::string clean = py::lower(py::strip(cand)); + if (!clean.empty() && !bot_ok(clean)) { return "error: invalid nickname '" + py::strip(cand) + "'"; } + } + if (taken.count(name)) { return "error: bot name '" + name + "' is taken"; } + for (const auto& nick : nicks) { + if (taken.count(nick)) { return "error: nickname '" + nick + "' is taken"; } + } + std::vector parts; + if (!description.empty()) { parts.push_back(description); } + if (!rules.empty()) { parts.push_back("Rules:\n" + rules); } + if (!behavior.empty()) { parts.push_back("Behavior:\n" + behavior); } + st.save_bot_system(profile, name, py::join(parts, "\n\n")); + json::Value bots = st.load_bots(profile); + std::string stamp = py::now_iso(); + json::Value entry = json::Value::object(); + json::Value nick_arr = json::Value::array(); + for (const auto& n : nicks) { nick_arr.push(n); } + entry.set("nicknames", nick_arr); + entry.set("created", stamp); + entry.set("updated", stamp); + bots.set(name, entry); + st.save_bots(profile, bots); + std::string extra = nicks.empty() ? "" : " (nicknames: " + py::join(nicks, ", ") + ")"; + return "created bot '" + name + "'" + extra + ". Switch with /bot " + name + " or mention @" + name + "."; +} + +std::string Tools::run_install(const json::Value& args) { + std::string action = py::lower(py::strip(py::str_or_empty(args.get("action")))); + if (!(action == "status" || action == "install" || action == "upgrade" || action == "reinstall" || action == "uninstall")) { + return "error: action must be status, install, upgrade, reinstall, or uninstall"; + } + if (ports().ops == nullptr) { throw PortNotWired("ops"); } + std::vector valid = ports().ops->install_targets(); + std::vector targets; + if (args.get("targets").truthy()) { + const json::Value& t = args.get("targets"); + if (!t.is_array() || t.size() == 0) { return "error: unknown target, valid: " + py::join(valid, ", "); } + for (const auto& item : t.items()) { + if (!item.is_string() || std::find(valid.begin(), valid.end(), item.str()) == valid.end()) { + return "error: unknown target, valid: " + py::join(valid, ", "); + } + targets.push_back(item.str()); + } + } else { + targets = valid; + } + if (action != "status" && !app.ask_approval("install " + action + ": " + py::join(targets, ", ") + " (vault always kept)")) { + return "denied by user"; + } + std::string report = ports().ops->install_report(action, targets, app.config.home); + if (action != "status" && std::find(targets.begin(), targets.end(), "skills") != targets.end()) { + if (ports().skills == nullptr) { throw PortNotWired("skills"); } + app.skills = ports().skills->discover_skills(app.config.home, std::filesystem::current_path().string()); + } + return report; +} + +std::string Tools::run_search(const json::Value& args) { + std::string query = py::strip(py::str_or_empty(args.get("query"))); + if (query.empty()) { return "error: empty query"; } + std::vector kinds; + if (args.get("kinds").truthy()) { + const json::Value& k = args.get("kinds"); + if (!k.is_array() || k.size() == 0) { return "error: kinds must be a list of record, event, audit, chunk"; } + for (const auto& item : k.items()) { + if (!item.is_string() || !(item.str() == "record" || item.str() == "event" || item.str() == "audit" || item.str() == "chunk")) { + return "error: kinds must be a list of record, event, audit, chunk"; + } + kinds.push_back(item.str()); + } + } else { + kinds = {"record", "event", "audit", "chunk"}; + } + int64_t limit; + { + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(10); + auto parsed = py::to_int(l); + if (!parsed.has_value()) { return "error: invalid limit"; } + limit = *parsed; + } + bool expand = args.get("expand").truthy(); + std::string profile = active_profile(); + StorePort& st = S("fts_search"); + json::Value hits_v = st.fts_search(query, kinds, profile, limit); + std::vector hits(hits_v.items().begin(), hits_v.items().end()); + if (st.seal_enabled() && std::find(kinds.begin(), kinds.end(), "event") != kinds.end()) { + bool any_event = false; + for (const auto& h : hits) { if (h.get("kind").is_string() && h.get("kind").str() == "event") { any_event = true; } } + if (!any_event) { + json::Value rows = st.events_like(profile, query, limit); + for (const auto& row : rows.items()) { + json::Value h = json::Value::object(); + h.set("kind", "event"); + h.set("item", "event " + sv(row.at(0))); + h.set("sub", row.at(2)); + h.set("title", py::head(sv(row.at(3)), 60)); + h.set("snippet", py::head(sv(row.at(3)), 160)); + h.set("rank", 99.0); + hits.push_back(h); + } + std::stable_sort(hits.begin(), hits.end(), [](const json::Value& a, const json::Value& b) { + return a.get("rank").as_double() < b.get("rank").as_double(); + }); + } + } + std::vector hit_parts; + { + size_t n = 0; + for (const auto& h : hits) { + // hits[:limit] with Python slice semantics (negative limit counts from the end) + (void)h; ++n; + } + long lim = static_cast(limit); + long total = static_cast(hits.size()); + long stop = lim >= 0 ? std::min(lim, total) : std::max(0L, total + lim); + for (long i = 0; i < stop; ++i) { hit_parts.push_back(sv(hits[static_cast(i)].get("kind")) + ":" + sv(hits[static_cast(i)].get("item"))); } + (void)n; + } + st.log_event(profile, "tool", "search-query", "q=" + py::head(query, 200) + "\tlimit=" + std::to_string(limit) + "\thits=" + py::head(py::join(hit_parts, ","), 1500)); + if (hits.empty()) { return "no matches for '" + py::head(query, 80) + "'"; } + std::vector chunk_ids; + for (const auto& h : hits) { + if (h.get("kind").is_string() && h.get("kind").str() == "chunk") { + std::string item = sv(h.get("item")); + size_t colon = item.find(':'); + if (colon == std::string::npos) { continue; } + auto id = py::parse_int(item.substr(colon + 1)); + if (id.has_value()) { chunk_ids.push_back(*id); } + } + } + json::Value chunk_rows = chunk_ids.empty() ? json::Value::object() : st.get_chunks_by_ids(chunk_ids, profile); + std::vector lines; + for (const auto& h : hits) { + std::string kind = sv(h.get("kind")); + if (kind == "record") { + lines.push_back("[record/" + sv(h.get("sub")) + "] " + sv(h.get("item")) + " :: " + py::head(sv(h.get("title")), 80) + " -- " + sv(h.get("snippet"))); + if (expand) { + std::vector neighbors; + json::Value edges = st.edges_for(sv(h.get("item")), profile); + for (const auto& edge : edges.items()) { + if (neighbors.size() < 3) { + neighbors.push_back(py::head(st.node_title(sv(edge.get("other")), profile), 60) + " (" + sv(edge.get("relation")) + ")"); + } + } + if (!neighbors.empty()) { lines.push_back(" linked: " + py::join(neighbors, "; ")); } + } + } else if (kind == "event") { + lines.push_back("[event/" + sv(h.get("sub")) + "] #" + sv(h.get("item")) + " " + py::head(sv(h.get("title")), 60) + " -- " + sv(h.get("snippet"))); + } else if (kind == "chunk") { + std::optional chunk_id; + { + std::string item = sv(h.get("item")); + size_t colon = item.find(':'); + if (colon != std::string::npos) { chunk_id = py::parse_int(item.substr(colon + 1)); } + } + json::Value chunk; + if (chunk_id.has_value()) { + // chunk_rows is keyed by the integer id; JSON object keys are strings + chunk = chunk_rows.get(std::to_string(*chunk_id)); + } + if (chunk.is_null()) { + lines.push_back("[chunk/" + sv(h.get("sub")) + "] " + py::head(sv(h.get("title")), 80) + " -- " + sv(h.get("snippet"))); + } else { + std::string text = sv(chunk.get("text")); + std::string preview = py::head(text, 500) + (py::len(text) > 500 ? "..." : ""); + lines.push_back("[chunk/" + sv(chunk.get("parent_kind")) + "] " + py::head(sv(chunk.get("parent_id")), 60) + "#" + + sv(chunk.get("idx")) + " (parent " + sv(chunk.get("parent_id")) + ", read full with record_read) -- " + preview); + } + } else { + std::string item = sv(h.get("item")); + size_t colon = item.find(':'); + std::string rest = colon == std::string::npos ? item : item.substr(colon + 1); + lines.push_back("[audit/" + sv(h.get("sub")) + "] #" + rest + " " + py::head(sv(h.get("title")), 80) + " -- " + sv(h.get("snippet"))); + } + } + return py::join(lines, "\n"); +} + +std::string Tools::run_tags(const json::Value& args) { + std::string prefix = py::strip(py::str_or_empty(args.get("prefix"))); + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(50); + auto limit = py::to_int(l); + if (!limit.has_value()) { return "error: invalid limit"; } + auto counts = S("tag_counts").tag_counts(prefix, *limit, active_profile()); + if (counts.empty()) { return "no tags yet"; } + std::vector lines; + for (const auto& tc : counts) { lines.push_back(tc.first + " (" + std::to_string(tc.second) + ")"); } + return std::to_string(counts.size()) + " tags:\n" + py::join(lines, "\n"); +} + +std::string Tools::run_record_read(const json::Value& args) { + std::string record_id = py::strip(py::str_or_empty(args.get("id"))); + int64_t offset, limit; + { + json::Value o = args.get("offset").truthy() ? args.get("offset") : json::Value(0); + auto parsed = py::to_int(o); + if (!parsed.has_value()) { return "error: invalid offset"; } + offset = std::max(0, *parsed); + } + { + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(4000); + auto parsed = py::to_int(l); + if (!parsed.has_value()) { return "error: invalid limit"; } + limit = *parsed; + } + StorePort& st = S("read_record"); + json::Value page = st.read_record(record_id, offset, std::max(1, std::min(limit, 20000)), active_profile()); + if (page.is_null()) { return "error: unknown record " + record_id; } + st.log_event(active_profile(), "tool", "record-read", py::head(record_id, 200)); + std::string head = record_id + " chars " + sv(page.get("start")) + "-" + sv(page.get("end")) + " of " + sv(page.get("total")) + + " :: " + st.node_title(record_id, active_profile()); + if (page.get("tags").truthy()) { + std::vector tags; + for (const auto& t : page.get("tags").items()) { tags.push_back(sv(t)); } + head += " [" + py::join(tags, ", ") + "]"; + } + return head + "\n" + sv(page.get("slice")); +} + +std::string Tools::run_record_search(const json::Value& args) { + std::string query = py::str_or_empty(args.get("query")); + std::string kind_s = py::lower(py::strip(py::str_or_empty(args.get("kind")))); + std::optional kind; + if (!kind_s.empty()) { kind = kind_s; } + if (kind.has_value()) { + const auto& kinds = RecordKinds(); + if (std::find(kinds.begin(), kinds.end(), *kind) == kinds.end()) { + return "error: unknown kind " + *kind + " (valid: " + py::join(kinds, ", ") + ")"; + } + } + auto tags = record_tag_args(args); + if (!tags.has_value()) { return "error: tags must be a list of names"; } + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(8); + auto limit = py::to_int(l); + if (!limit.has_value()) { return "error: invalid limit"; } + json::Value found = S("search_records").search_records(query, kind, *tags, *limit, active_profile()); + if (found.size() == 0) { return "no records match"; } + std::vector lines; + for (const auto& item : found.items()) { + std::vector t; + for (const auto& x : item.get("tags").items()) { t.push_back(sv(x)); } + lines.push_back(sv(item.get("id")) + " [" + sv(item.get("kind")) + "] " + sv(item.get("title")) + " (" + sv(item.get("size")) + + " chars, " + sv(item.get("reads")) + " reads) tags: " + py::join(t, ", ")); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_record_delete(const json::Value& args) { + std::string record_id = py::strip(py::str_or_empty(args.get("id"))); + if (record_id.empty()) { return "error: record id is required"; } + if (!app.ask_approval("delete record " + record_id)) { return "denied by user"; } + if (S("delete_record").delete_record(record_id, active_profile())) { return "deleted record " + record_id; } + return "error: unknown record " + record_id; +} + +std::string Tools::run_graph_link(const json::Value& args) { + std::string src = py::strip(py::str_or_empty(args.get("src"))); + std::string dst = py::strip(py::str_or_empty(args.get("dst"))); + if (src.empty() || dst.empty()) { return "error: src and dst are required"; } + std::string relation; + try { + relation = S("add_edge").add_edge(src, dst, args.get("relation"), active_profile()); + } catch (const std::invalid_argument& exc) { + return std::string("error: ") + exc.what(); + } + return "linked " + src + " -[" + relation + "]-> " + dst; +} + +std::string Tools::run_graph_query(const json::Value& args) { + std::string node = py::strip(py::str_or_empty(args.get("node"))); + auto parsed = parse_node_id(node); + StorePort& st = S("has_node"); + if (!parsed.has_value() || !st.has_node(parsed->first, parsed->second, active_profile())) { + return "error: unknown node " + node; + } + json::Value d = args.get("depth").truthy() ? args.get("depth") : json::Value(2); + auto depth = py::to_int(d); + if (!depth.has_value()) { return "error: invalid depth"; } + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(30); + auto limit = py::to_int(l); + if (!limit.has_value()) { return "error: invalid limit"; } + json::Value found = st.traverse(node, *depth, *limit, active_profile()); + std::vector lines; + for (const auto& item : found.items()) { + int64_t dep = item.get("depth").as_int(); + if (dep == 0) { + lines.push_back(sv(item.get("node")) + " :: " + sv(item.get("title"))); + } else { + lines.push_back(std::string(static_cast(2 * dep), ' ') + sv(item.get("via")) + " :: " + sv(item.get("title"))); + } + } + return py::join(lines, "\n"); +} + +std::string Tools::run_audit(const json::Value& args) { + auto opt = [&](const char* k) -> std::optional { + std::string s = py::strip(py::str_or_empty(args.get(k))); + if (s.empty()) { return std::nullopt; } + return s; + }; + auto path = opt("path"); + auto tag = opt("tag"); + auto query = opt("query"); + json::Value l = args.get("limit").truthy() ? args.get("limit") : json::Value(20); + auto limit = py::to_int(l); + if (!limit.has_value()) { return "error: invalid limit"; } + json::Value rows = query.has_value() ? S("audit_search").audit_search(*query, active_profile(), *limit) + : S("audit_history").audit_history(path, tag, *limit, active_profile()); + if (rows.size() == 0) { return "no audit rows match"; } + std::vector lines; + for (const auto& row : rows.items()) { + std::string old_size = row.get("old_size").is_null() ? "-" : sv(row.get("old_size")); + std::string new_size = row.get("new_size").is_null() ? "-" : sv(row.get("new_size")); + std::string line = "#" + sv(row.get("id")) + " " + py::head(sv(row.get("ts")), 19) + " " + sv(row.get("actor")) + " " + + sv(row.get("action")) + " " + sv(row.get("path")) + " (" + old_size + "->" + new_size + " bytes)"; + if (row.get("message").truthy()) { line += " :: " + py::head(sv(row.get("message")), 120); } + if (row.get("tags").truthy()) { line += " [" + sv(row.get("tags")) + "]"; } + lines.push_back(line); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_restore(const json::Value& args) { + json::Value idv = args.get("id").truthy() ? args.get("id") : json::Value(0); + auto id = py::to_int(idv); + if (!id.has_value()) { return "error: invalid audit id"; } + int64_t row_id = *id; + json::Value row = S("audit_get").audit_get(row_id, active_profile()); + if (row.is_null()) { return "error: unknown audit #" + std::to_string(row_id); } + std::string action = sv(row.get("action")); + if (action == "snapshot" || action == "release") { + return "error: audit #" + std::to_string(row_id) + " is a " + action + + " row, restorable rows are write, edit, delete, shell-snapshot, restore"; + } + std::string target, which; + json::Value full_size; + if (!row.get("new").is_null()) { + target = sv(row.get("new")); which = "post"; full_size = row.get("new_size"); + } else if (!row.get("old").is_null()) { + target = sv(row.get("old")); which = "pre"; full_size = row.get("old_size"); + } else { + return "error: audit #" + std::to_string(row_id) + " holds no content"; + } + if (target.find("[truncated, full size") != std::string::npos) { + return "error: audit #" + std::to_string(row_id) + " " + which + "-image is truncated (full size " + sv(full_size) + + " bytes), cannot restore safely"; + } + std::string path = sv(row.get("path")); + if (!app.ask_approval("restore " + path + " from audit #" + std::to_string(row_id) + " (" + which + "-image)")) { + return "denied by user"; + } + std::string message = "restored from audit #" + std::to_string(row_id) + " (" + which + "-image)"; + if (py::startswith(path, "sandbox:")) { return box_restore(path.substr(8), target, message); } + Capped current; + if (isfile(path)) { current = read_capped(path); } + { + std::string parent; + size_t slash = path.rfind('/'); + if (slash != std::string::npos) { parent = path.substr(0, slash); if (parent.empty()) { parent = "/"; } } + if (!parent.empty()) { + std::error_code ec; + std::filesystem::create_directories(parent, ec); + if (ec) { return "error: " + py::head(os_error_message(ec.value(), parent), 300); } + } + std::string err; + if (!write_text_file(path, target, err)) { return "error: " + py::head(err, 300); } + read_files.insert({"home", abspath(path)}); + } + audit_file("restore", path, std::nullopt, target, message); + S("upsert_file_record").upsert_file_record(path, target, std::nullopt, active_profile()); + if (!current.size.has_value()) { note_diff(path, current.text, target); } + return "restored " + path + " from audit #" + std::to_string(row_id) + " (" + which + "-image)"; +} + +std::string Tools::run_release(const json::Value& args) { + std::string part = py::lower(py::strip(py::str_or_empty(args.get("part")))); + std::string message = py::strip(py::str_or_empty(args.get("message"))); + if (!(part == "major" || part == "minor" || part == "patch")) { return "error: part must be major, minor, or patch"; } + if (message.empty()) { return "error: release message is required"; } + if (ports().ops == nullptr) { throw PortNotWired("ops"); } + std::string upcoming; + try { + std::string current = ports().ops->current_version(); + upcoming = ports().ops->next_version(current, part); + } catch (const std::exception& exc) { + return "error: " + short_error(exc); + } + if (!app.ask_approval("release " + upcoming + " (" + part + "): " + py::head(message, 120))) { return "denied by user"; } + try { + auto [old_v, new_v, dest, row_id] = ports().ops->do_release(part, message, audit_actor()); + return "released " + old_v + " -> " + new_v + ", backup " + dest + ", audit #" + std::to_string(row_id); + } catch (const std::exception& exc) { + return "error: " + short_error(exc); + } +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/tools_ops.cpp b/taicpp/src/agent/tools_ops.cpp new file mode 100644 index 0000000..090fb2a --- /dev/null +++ b/taicpp/src/agent/tools_ops.cpp @@ -0,0 +1,682 @@ +#include "agent.h" +#include "../llm/helpers.h" +#include "../llm/proc.h" +#include "../llm/py.h" +#include "../llm/term.h" +#include "consts.h" +#include "text.h" +#include "zipfile.h" +#include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::agent { + +using llm::Ansi; +using llm::BackendError; +using llm::env_get; +using llm::paint; +using llm::short_error; + +namespace { +UiPort* ui() { return ports().ui; } +std::string sv(const json::Value& v) { return py::str(v); } +// int(args.get(key) or dflt) -> nullopt on TypeError/ValueError +std::optional int_or(const json::Value& args, const char* key, int64_t dflt) { + json::Value v = args.get(key).truthy() ? args.get(key) : json::Value(dflt); + return py::to_int(v); +} +const char* kNoPlexer = "error: no plexer session active (neither tai's native wrapper nor tmux)"; +} // namespace + +// ------------------------------------------------------------------ terminals +std::string Tools::run_terminal_content(const json::Value& args) { + int lines = 100; + { + auto v = int_or(args, "lines", 100); + if (v.has_value()) { lines = static_cast(std::max(10, std::min(500, *v))); } + } + std::string target = py::head(py::strip(py::str_or_empty(args.get("target"))), 128); + std::string history = shell_history_section(); + if (target.empty()) { + std::string socket_path = env_get(PLEXER_SOCKET_ENV); + if (!socket_path.empty()) { return run_terminal_content_native(socket_path, lines, history); } + target = py::head(py::strip(env_get("TMUX_PANE")), 128); + } + if (llm::which("tmux").empty()) { return "tmux not available"; } + std::vector header; + if (!target.empty()) { + header = {"pane " + target}; + } else { + std::string ph = tmux_pane_header(); + if (!ph.empty()) { header = {ph}; } + } + std::vector argv = {"tmux", "capture-pane", "-p", "-S", "-" + std::to_string(lines)}; + if (!target.empty()) { argv.push_back("-t"); argv.push_back(target); } + llm::ProcResult done = llm::run_process(argv, std::nullopt, 10); + if (done.spawn_failed) { return "terminal capture failed: " + llm::short_error_text(llm::oserror_text(done, "tmux")) + history; } + if (done.timed_out) { return "terminal capture failed: " + llm::short_error_text(llm::timeout_text(argv, 10)) + history; } + std::string out = py::universal_newlines(py::utf8_replace(done.out)); + if (done.returncode != 0) { + return "terminal capture failed: " + py::head(py::strip(py::universal_newlines(py::utf8_replace(done.err))), 200) + history; + } + if (py::strip(out).empty()) { return "terminal pane is empty" + history; } + header.push_back(llm::truncate(out, 6000, 2000)); + return py::join(header, "\n") + history; +} + +std::string Tools::run_terminal_content_native(const std::string& socket_path, int lines, const std::string& history) { + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + std::string pane_env = env_get(PLEXER_PANE_ENV); + std::string content; + try { + content = ports().plexer->native_read_pane(socket_path, pane_env, lines); + } catch (const PlexerFailure& exc) { + return std::string("terminal capture failed: ") + exc.what() + history; + } + if (py::strip(content).empty()) { return "terminal pane is empty" + history; } + std::string pane = py::strip(pane_env); + std::string header = !pane.empty() ? "native pane " + pane : "native plexer pane"; + return header + "\n" + llm::truncate(content, 6000, 2000) + history; +} + +std::string Tools::run_list_terminals(const json::Value&) { + auto [panes, error] = tmux_panes(); + if (!error.empty()) { return error; } + if (panes.empty()) { return "no open terminals"; } + std::set sessions, windows; + for (const auto& p : panes) { + sessions.insert(py::split(p.ref, ":")[0]); + size_t dot = p.ref.rfind('.'); + windows.insert(dot == std::string::npos ? p.ref : p.ref.substr(0, dot)); + } + auto plural = [](size_t n) { return n == 1 ? "" : "s"; }; + std::vector lines = {std::to_string(sessions.size()) + " session" + plural(sessions.size()) + " \xC2\xB7 " + + std::to_string(windows.size()) + " window" + plural(windows.size()) + " \xC2\xB7 " + + std::to_string(panes.size()) + " pane" + plural(panes.size())}; + for (size_t i = 0; i < panes.size() && i < static_cast(TERMINAL_LIST_MAX_PANES); ++i) { + const auto& p = panes[i]; + lines.push_back(p.ref + " " + p.id + " " + p.command + " " + p.path + (p.active ? " *" : "")); + } + if (panes.size() > static_cast(TERMINAL_LIST_MAX_PANES)) { + lines.push_back("... +" + std::to_string(panes.size() - static_cast(TERMINAL_LIST_MAX_PANES)) + " more"); + } + return py::join(lines, "\n"); +} + +std::string Tools::run_tail_terminals(const json::Value& args) { + int lines = TAIL_TERMINALS_DEFAULT_LINES; + { + auto v = int_or(args, "lines", TAIL_TERMINALS_DEFAULT_LINES); + if (v.has_value()) { lines = static_cast(std::max(1, std::min(TAIL_TERMINALS_MAX_LINES, *v))); } + } + auto [panes, error] = tmux_panes(); + if (!error.empty()) { return error; } + if (panes.empty()) { return "no open terminals"; } + std::vector blocks; + for (size_t i = 0; i < panes.size() && i < static_cast(TAIL_TERMINALS_MAX_PANES); ++i) { + const auto& pane = panes[i]; + std::vector argv = {"tmux", "capture-pane", "-p", "-t", pane.id, "-S", "-" + std::to_string(lines)}; + llm::ProcResult done = llm::run_process(argv, std::nullopt, 10); + std::string head = "=== " + pane.ref + " " + pane.id + " (" + pane.command + ") ===\n"; + if (done.spawn_failed || done.timed_out) { blocks.push_back(head + "(capture failed)"); continue; } + std::string out = py::universal_newlines(py::utf8_replace(done.out)); + if (done.returncode != 0 || py::strip(out).empty()) { blocks.push_back(head + "(empty)"); continue; } + blocks.push_back(head + py::head(py::strip(out), TAIL_PER_PANE_CHARS)); + } + if (panes.size() > static_cast(TAIL_TERMINALS_MAX_PANES)) { + blocks.push_back("... +" + std::to_string(panes.size() - static_cast(TAIL_TERMINALS_MAX_PANES)) + + " more panes, capture one with get_current_terminal_content"); + } + return llm::truncate(py::join(blocks, "\n\n"), 6000, 2000); +} + +std::string Tools::run_export_terminal_log(const json::Value& args) { + if (env_get("TMUX").empty()) { return "error: tmux is not active (this tool only works inside a live tmux session)"; } + if (llm::which("tmux").empty()) { return "error: tmux not available"; } + json::Value raw_tags = args.get("tags").truthy() ? args.get("tags") : json::Value::array(); + std::vector extra_tags; + if (!raw_tags.is_array()) { return "error: tags must be a list of strings"; } + for (const auto& t : raw_tags.items()) { + if (!t.is_string()) { return "error: tags must be a list of strings"; } + extra_tags.push_back(t.str()); + } + std::vector argv = {"tmux", "capture-pane", "-p", "-S", "-"}; + llm::ProcResult done = llm::run_process(argv, std::nullopt, 30); + if (done.spawn_failed) { return "error: " + llm::short_error_text(llm::oserror_text(done, "tmux")); } + if (done.timed_out) { return "error: " + llm::short_error_text(llm::timeout_text(argv, 30)); } + if (done.returncode != 0) { + return "error: tmux capture failed: " + py::head(py::strip(py::universal_newlines(py::utf8_replace(done.err))), 200); + } + std::string raw = py::universal_newlines(py::utf8_replace(done.out)); + if (py::strip(raw).empty()) { return "terminal pane history is empty"; } + std::string profile = active_profile(); + std::string pane_header = tmux_pane_header(); + if (pane_header.empty()) { pane_header = "pane"; } + char stamp_buf[32]; + { + time_t t = static_cast(py::now()); + std::tm tmv{}; + gmtime_r(&t, &tmv); + std::strftime(stamp_buf, sizeof stamp_buf, "%Y%m%dT%H%M%SZ", &tmv); + } + std::string stamp = stamp_buf; + std::string slug; + { + // re.sub(r"[^A-Za-z0-9_.-]+", "-", pane_header).strip("-") or "pane" + bool pending = false; + for (unsigned char c : pane_header) { + bool keep = (c < 0x80) && (std::isalnum(c) || c == '_' || c == '.' || c == '-'); + if (keep) { if (pending) { slug += '-'; pending = false; } slug += static_cast(c); } + else { pending = true; } + } + if (pending) { slug += '-'; } + slug = py::strip_chars(slug, "-"); + if (slug.empty()) { slug = "pane"; } + } + std::string folder = app.config.home + "/termlogs/" + profile; + std::string path = folder + "/" + slug + "-" + stamp + ".log"; + { + std::error_code ec; + fs::create_directories(folder, ec); + if (ec) { return "error: " + py::head(os_error_message(ec.value(), folder), 300); } + std::string err; + if (!write_text_file(path, raw, err)) { return "error: " + py::head(err, 300); } + } + if (app.store == nullptr) { + return "captured " + pane_header + " to " + path + " (" + std::to_string(raw.size()) + " bytes); vault unavailable, not indexed for search"; + } + std::string title = "tmux " + pane_header + " captured " + stamp; + std::vector tags = {"termlog"}; + tags.insert(tags.end(), extra_tags.begin(), extra_tags.end()); + std::string record_id = app.store->add_record("termlog", title, raw, tags, profile); + auto chunks = chunk_lines(collapse_repeated_lines(raw)); + app.store->add_chunks("record", record_id, chunks, profile); + return "captured " + pane_header + " to " + path + " (" + std::to_string(raw.size()) + " bytes), indexed as " + record_id + " in " + + std::to_string(chunks.size()) + " chunks -- use search (kind chunk) to answer questions about this terminal session"; +} + +// ------------------------------------------------------------------ plexer +std::string Tools::run_plexer_send(const json::Value& args) { + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + auto controller = ports().plexer->detect(); + if (!controller) { return kNoPlexer; } + std::string text = py::str_or_empty(args.get("text")); + bool enter = args.get("enter").truthy(); + try { + controller->send_keys(text, enter); + } catch (const PlexerFailure& exc) { + return std::string("error: ") + exc.what(); + } + return "sent " + std::to_string(py::len(text)) + " chars via " + controller->kind() + " backend" + (enter ? " + enter" : ""); +} + +std::string Tools::run_plexer_read(const json::Value& args) { + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + auto controller = ports().plexer->detect(); + if (!controller) { return kNoPlexer; } + int lines = 200; + { + auto v = int_or(args, "lines", 200); + if (v.has_value()) { lines = static_cast(std::max(1, std::min(*v, 5000))); } + } + std::string content; + try { + content = controller->read_pane(lines); + } catch (const PlexerFailure& exc) { + return std::string("error: ") + exc.what(); + } + return llm::truncate(content.empty() ? std::string("(empty)") : content, 6000, 2000); +} + +std::string Tools::run_plexer_type(const json::Value& args) { + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + auto controller = ports().plexer->detect(); + if (!controller) { return kNoPlexer; } + std::string text = py::str_or_empty(args.get("text")); + if (text.empty()) { return "error: empty text"; } + double delay = 0.03; + { + json::Value d = args.get("delay").truthy() ? args.get("delay") : json::Value(0.03); + auto f = py::to_float(d); + if (f.has_value()) { delay = std::max(0.0, std::min(*f, 2.0)); } + } + bool enter = !(args.get("enter").is_bool() && !args.get("enter").as_bool()); + try { + controller->type_text(text, delay, enter); + } catch (const PlexerFailure& exc) { + return std::string("error: ") + exc.what(); + } + char buf[32]; + std::snprintf(buf, sizeof buf, "%.2f", delay); + return "typed " + std::to_string(py::len(text)) + " chars via " + controller->kind() + " backend at " + buf + "s/char"; +} + +std::string Tools::run_tui(const std::string& op, const json::Value& args) { + if (ports().tui == nullptr) { throw PortNotWired("tui"); } + json::Value payload = json::Value::object(); + payload.set("op", op); + for (const auto& kv : args.members()) { + if (!kv.second.is_null()) { payload.set(kv.first, kv.second); } + } + if (op == "screen" && args.get("until").truthy()) { + json::Value t = args.get("timeout").truthy() ? args.get("timeout") : json::Value(GRR_CONTROL_WAIT); + if (t.is_int() || t.is_bool()) { payload.set("wait", t.as_int() + 2); } + else if (t.is_double()) { payload.set("wait", t.as_double() + 2); } + else if (t.is_string()) { throw std::runtime_error("can only concatenate str (not \"int\") to str"); } + else if (t.is_array()) { throw std::runtime_error("can only concatenate list (not \"int\") to list"); } + else { throw std::runtime_error("unsupported operand type(s) for +: '" + py_type_name(t) + "' and 'int'"); } + } + return ports().tui->control_call(payload).dumps(); +} + +std::string Tools::run_plexer_status(const json::Value&) { + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + auto controller = ports().plexer->detect(); + if (!controller) { return "no plexer session active (neither tai's native wrapper nor tmux)"; } + try { + return controller->status().dumps(); + } catch (const PlexerFailure& exc) { + return std::string("error: ") + exc.what(); + } +} + +std::string Tools::run_plexer_show(const json::Value& args) { + const json::Value& spec = args.get("spec"); + if (!spec.is_object()) { return "error: spec must be a JSON object (the flow contract is uiflow.md)"; } + if (ports().plexer == nullptr) { throw PortNotWired("plexer"); } + auto [flow_app, error] = ports().plexer->validate_flow_spec(spec); + if (!error.empty()) { return "error: invalid flow spec: " + error; } + if (!args.get("timeout").is_null()) { + auto override_v = py::to_int(args.get("timeout")); + if (!override_v.has_value()) { return "error: timeout must be an integer number of seconds"; } + if (!(PLEXER_FLOW_TIMEOUT_MIN <= *override_v && *override_v <= PLEXER_FLOW_TIMEOUT_MAX)) { + return "error: timeout must be " + std::to_string(PLEXER_FLOW_TIMEOUT_MIN) + " to " + std::to_string(PLEXER_FLOW_TIMEOUT_MAX) + " seconds"; + } + flow_app.set("timeout", *override_v); + } + if (app.auto_mode) { return "plexer_show unavailable in autonomous mode: decide yourself and state the assumption"; } + if (!app.persist || app.depth > 0) { + return "plexer_show unavailable to background workers: decide yourself and state the assumption"; + } + auto controller = ports().plexer->detect(); + if (!controller) { return kNoPlexer; } + json::Value outcome; + try { + outcome = ports().plexer->run_flow(flow_app, *controller); + } catch (const PlexerFailure& exc) { + return std::string("error: ") + exc.what(); + } + app.user_stepped_in = true; + return outcome.dumps(); +} + +// ------------------------------------------------------------------ agents +std::string Tools::run_fork(const json::Value& args) { + std::string task = py::strip(py::str_or_empty(args.get("task"))); + if (task.empty()) { return "error: empty task"; } + if (app.depth >= FORK_MAX_DEPTH) { return "error: fork depth limit reached"; } + int64_t timeout = 600; + { + auto v = int_or(args, "timeout", 600); + if (v.has_value()) { timeout = std::max(30, std::min(3600, *v)); } + } + std::string asked = py::strip(py::str_or_empty(args.get("profile"))); + if (!asked.empty() && !profile_ok(asked)) { return "error: invalid profile name"; } + if (!asked.empty() && asked != active_profile()) { return "error: cannot fork as another profile"; } + std::string profile = active_profile(); + int agent_id; + try { + agent_id = spawn_agent(task, profile, static_cast(timeout), app.config, app.depth + 1, app.runner_override); + } catch (const AgentsFull& exc) { + return std::string("error: ") + exc.what() + ", poll existing agents first"; + } + return "agent " + std::to_string(agent_id) + " started: " + py::head(task, 80) + ". Collect its result with poll."; +} + +std::string Tools::run_poll(const json::Value& args) { + auto id = int_or(args, "id", 0); + if (!id.has_value()) { return "error: invalid agent id"; } + int agent_id = static_cast(*id); + int64_t wait = 0; + { + auto v = int_or(args, "wait", 0); + wait = v.has_value() ? std::max(0, std::min(120, *v)) : 0; + } + auto [status, text] = poll_agent(agent_id, 0, active_profile()); + if (status == "running" && wait > 0) { + std::unique_ptr board; + if (ports().shell != nullptr) { board = ports().shell->make_lane_board(active_profile()); } + if (board) { board->start(); } + try { + std::tie(status, text) = poll_agent(agent_id, static_cast(wait), active_profile()); + } catch (...) { + if (board) { board->stop(); board_shown = true; } + throw; + } + if (board) { board->stop(); board_shown = true; } + } + if (status == "done" || status == "timeout" || status == "error") { + std::lock_guard guard(AgentRegistry::lock()); + auto it = AgentRegistry::records().find(agent_id); + if (it != AgentRegistry::records().end()) { it->second->collected = true; } + } + if ((status == "done" || status == "timeout") && app.store != nullptr) { + std::string full, task_text; + std::optional spilled; + { + std::lock_guard guard(AgentRegistry::lock()); + auto it = AgentRegistry::records().find(agent_id); + if (it != AgentRegistry::records().end()) { + full = it->second->result; + task_text = it->second->task; + spilled = it->second->spilled; + } + } + if (static_cast(py::len(full)) > SPILL_LIMIT) { + if (!spilled.has_value() || spilled->empty()) { + spilled = app.store->add_record("output", "agent " + std::to_string(agent_id) + ": " + py::head(task_text, 80), full, + {"agent-output"}, active_profile()); + app.store->add_chunks("record", *spilled, chunk_lines(full), active_profile()); + std::lock_guard guard(AgentRegistry::lock()); + auto it = AgentRegistry::records().find(agent_id); + if (it != AgentRegistry::records().end()) { it->second->spilled = spilled; } + } + return "[" + status + "] " + spilled_view(full, *spilled, py::len(full)); + } + } + return "[" + status + "] " + text; +} + +std::string Tools::run_ask_user(const json::Value& args) { + std::string question = py::strip(py::str_or_empty(args.get("question"))); + if (question.empty()) { return "error: question must be a non-empty string"; } + json::Value raw_options = args.get("options").truthy() ? args.get("options") : json::Value::array(); + if (!raw_options.is_array()) { return "error: options must be a list of strings"; } + std::vector options; + { + size_t n = 0; + for (const auto& item : raw_options.items()) { + if (n++ >= static_cast(ASK_USER_MAX_OPTIONS)) { break; } + json::Value label; + if (item.is_object()) { + label = item.has("label") ? item.get("label") : (item.has("value") ? item.get("value") : json::Value("")); + } else { + label = item; + } + std::string text = py::replace(py::strip(py::str_or_empty(label)), "\n", " "); + if (!text.empty()) { options.push_back(py::head(text, 200)); } + } + } + if (app.auto_mode) { return "ask_user unavailable in autonomous mode: decide yourself and state the assumption"; } + if (!app.persist || app.depth > 0) { + return "ask_user unavailable to background workers: decide yourself and state the assumption"; + } + std::string raw; + if (app.prompter) { + auto got = app.prompter("ask", question, options); + if (!got.has_value()) { return "user gave no answer (input closed): proceed with your best assumption"; } + raw = *got; + } else { + if (!llm::stdin_isatty()) { + return "error: no interactive user on this stdin, decide yourself and state the assumption"; + } + llm::print_out(paint("?- " + question, Ansi::CYAN, Ansi::BOLD)); + for (size_t pos = 0; pos < options.size(); ++pos) { + llm::print_out(paint(" " + std::to_string(pos + 1) + ") " + options[pos], Ansi::DIM)); + } + std::string prompt = options.empty() + ? paint("answer (free input, empty skips) ", Ansi::YELLOW) + : paint("answer [1-" + std::to_string(options.size()) + ", option text, or free input] ", Ansi::YELLOW); + std::fputs(prompt.c_str(), stdout); + std::fflush(stdout); + if (!std::getline(std::cin, raw)) { + return "user gave no answer (input closed): proceed with your best assumption"; + } + } + app.user_stepped_in = true; + std::string answer = py::strip(raw); + if (answer.empty()) { return "user gave no answer (empty): proceed with your best assumption"; } + if (static_cast(py::len(answer)) > ASK_USER_MAX_ANSWER) { answer = py::head(answer, ASK_USER_MAX_ANSWER) + "...[truncated]"; } + if (!options.empty()) { + if (py::isdigit_ascii(answer)) { + auto pos = py::parse_int(answer); + if (pos.has_value() && 1 <= *pos && *pos <= static_cast(options.size())) { + return "user chose option " + std::to_string(*pos) + ": " + options[static_cast(*pos - 1)]; + } + } + std::string lowered = py::lower(answer); + for (size_t pos = 0; pos < options.size(); ++pos) { + if (py::lower(options[pos]) == lowered) { + return "user chose option " + std::to_string(pos + 1) + ": " + options[pos]; + } + } + } + return "user answered: " + answer; +} + +std::string Tools::run_sysinfo(const json::Value& args) { + if (ports().ops == nullptr) { throw PortNotWired("ops"); } + const json::Value& names = args.get("checks"); + if (names.is_null()) { return ports().ops->collect_sysinfo(std::nullopt); } + if (!names.is_array() || names.size() == 0) { return "error: checks must be a non-empty list"; } + std::vector valid = ports().ops->sysinfo_check_names(); + std::vector unknown, picked; + for (const auto& item : names.items()) { + std::string s = sv(item); + picked.push_back(s); + if (std::find(valid.begin(), valid.end(), s) == valid.end()) { unknown.push_back(s); } + } + if (!unknown.empty()) { + return "error: unknown checks: " + py::join(unknown, ", ") + " (valid: " + py::join(valid, ", ") + ")"; + } + return ports().ops->collect_sysinfo(picked); +} + +// ------------------------------------------------------------------ skills +std::string Tools::run_create_skill(const json::Value& args) { + std::string name = py::strip(py::str_or_empty(args.get("name"))); + std::string brief = py::strip(py::str_or_empty(args.get("brief"))); + if (name.empty() || !skill_name_ok(name)) { return "error: invalid skill name, use lowercase letters, digits, and hyphens"; } + if (brief.empty()) { return "error: empty brief"; } + if (app.depth >= FORK_MAX_DEPTH) { return "error: create_skill depth limit reached"; } + std::string scope = py::lower(py::strip(args.get("scope").truthy() ? py::str(args.get("scope")) : "project")); + if (scope != "project" && scope != "home") { return "error: scope must be project or home"; } + int64_t timeout = 1200; + { + auto v = int_or(args, "timeout", 1200); + if (v.has_value()) { timeout = std::max(60, std::min(3600, *v)); } + } + if (ports().skills == nullptr) { throw PortNotWired("skills"); } + std::string cwd = fs::current_path().string(); + std::string skill_dir = ports().skills->skill_scope_dir(scope, app.config.home, cwd, name); + std::string skill_file = skill_dir + "/SKILL.md"; + bool existed = isfile(skill_file); + if (ui() != nullptr) { ui()->spinner_start("creating skill " + name); } + std::string result; + try { + if (!app.runner_override) { + result = default_runner(app.config, app.profile, app.depth + 1, create_skill_prompt(name, brief, skill_dir), + static_cast(timeout), CREATE_SKILL_STEPS); + } else { + result = app.runner_override(create_skill_prompt(name, brief, skill_dir), app.profile, static_cast(timeout)); + } + } catch (...) { + if (ui() != nullptr) { ui()->spinner_stop(); } + throw; + } + if (ui() != nullptr) { ui()->spinner_stop(); } + if (isfile(skill_file)) { + app.skills = ports().skills->discover_skills(app.config.home, cwd); + app.apply_system(); + std::string action = existed ? "replaced" : "created"; + return "skill '" + name + "' " + action + " at " + skill_file + "\n" + llm::truncate(result, 2000, 500); + } + std::string status = result.find("[time limit reached]") != std::string::npos ? "timeout" : "done"; + return "skill '" + name + "' was not created (" + status + "), worker output:\n" + llm::truncate(result, 3000, 1000); +} + +std::optional Tools::find_skill_root(const std::string& base) { + if (isfile(base + "/SKILL.md")) { return base; } + std::vector entries; + std::error_code ec; + for (const auto& e : fs::directory_iterator(base, ec)) { entries.push_back(e.path().filename().string()); } + if (ec) { return std::nullopt; } + std::sort(entries.begin(), entries.end()); + std::vector candidates; + for (const auto& e : entries) { if (isfile(base + "/" + e + "/SKILL.md")) { candidates.push_back(e); } } + if (candidates.size() == 1) { return base + "/" + candidates[0]; } + return std::nullopt; +} + +std::pair>, std::string> Tools::resolve_skill_source(const std::string& source, + const std::string& workdir) { + if (py::endswith(source, ".git") || py::startswith(source, "git@") || py::startswith(source, "git+")) { + std::string git_url = py::startswith(source, "git+") ? source.substr(4) : source; + if (llm::which("git").empty()) { return {std::nullopt, "error: git is not installed, supply a local path or .zip instead"}; } + std::string clone_dir = workdir + "/clone"; + llm::ProcResult done = llm::run_process({"git", "clone", "--depth", "1", git_url, clone_dir}, std::nullopt, 120); + if (done.spawn_failed) { return {std::nullopt, "error: git clone failed: " + llm::short_error_text(llm::oserror_text(done, "git"))}; } + if (done.timed_out) { + return {std::nullopt, "error: git clone failed: " + llm::short_error_text(llm::timeout_text({"git", "clone", "--depth", "1", git_url, clone_dir}, 120))}; + } + if (done.returncode != 0) { + return {std::nullopt, "error: git clone failed: " + py::head(py::strip(py::universal_newlines(py::utf8_replace(done.err))), 300)}; + } + auto found = find_skill_root(clone_dir); + if (!found.has_value()) { return {std::nullopt, "error: no SKILL.md found in " + git_url}; } + return {std::make_pair(*found, true), ""}; + } + std::string path = expanduser(source); + if (!path_exists(path)) { return {std::nullopt, "error: no such file or directory: " + source}; } + if (isfile(path) && py::endswith(path, ".zip")) { + std::string extract_dir = workdir + "/zip"; + try { + zip_extractall(path, extract_dir); + } catch (const BadZipFile& exc) { + return {std::nullopt, "error: bad zip file: " + short_error(exc)}; + } catch (const std::runtime_error& exc) { + std::string what = exc.what(); + if (what.rfind("[Errno", 0) == 0) { return {std::nullopt, "error: bad zip file: " + py::head(what, 300)}; } + throw; + } + auto found = find_skill_root(extract_dir); + if (!found.has_value()) { return {std::nullopt, "error: no SKILL.md found in " + source}; } + return {std::make_pair(*found, true), ""}; + } + if (isfile(path) && py::endswith(path, ".md")) { return {std::make_pair(path, false), ""}; } + if (isdir(path)) { + auto found = find_skill_root(path); + if (!found.has_value()) { return {std::nullopt, "error: no SKILL.md found under " + source}; } + return {std::make_pair(*found, false), ""}; + } + return {std::nullopt, "error: source must be a directory, a SKILL.md file, a .zip, or a git URL"}; +} + +namespace { +struct TempWorkdir { + std::string path; + TempWorkdir() { + std::string tmpl = (fs::temp_directory_path() / "tai-skill-XXXXXX").string(); + std::vector buf(tmpl.begin(), tmpl.end()); + buf.push_back('\0'); + if (mkdtemp(buf.data()) != nullptr) { path = buf.data(); } + } + ~TempWorkdir() { std::error_code ec; if (!path.empty()) { fs::remove_all(path, ec); } } +}; +} // namespace + +std::string Tools::run_install_skill(const json::Value& args) { + std::string source = py::strip(py::str_or_empty(args.get("source"))); + if (source.empty()) { return "error: empty source"; } + if (ports().skills == nullptr) { throw PortNotWired("skills"); } + SkillsPort& sk = *ports().skills; + std::string cwd = fs::current_path().string(); + std::string scope = py::lower(py::strip(py::str_or_empty(args.get("scope")))); + if (scope.empty()) { scope = sk.default_skill_scope(cwd); } + if (scope != "project" && scope != "home") { return "error: scope must be project or home"; } + std::string mode_s = py::lower(py::strip(py::str_or_empty(args.get("mode")))); + std::optional mode; + if (!mode_s.empty()) { mode = mode_s; } + if (mode.has_value() && *mode != "copy" && *mode != "symlink") { return "error: mode must be copy or symlink"; } + bool force = args.get("force").truthy(); + TempWorkdir work; + if (ui() != nullptr) { ui()->spinner_start("installing skill from " + source); } + std::pair>, std::string> resolved_failure; + try { + resolved_failure = resolve_skill_source(source, work.path); + } catch (...) { + if (ui() != nullptr) { ui()->spinner_stop(); } + throw; + } + if (ui() != nullptr) { ui()->spinner_stop(); } + if (!resolved_failure.second.empty()) { return resolved_failure.second; } + std::string real_source = resolved_failure.first->first; + bool ephemeral = resolved_failure.first->second; + std::optional source_dir; + if (isdir(real_source)) { source_dir = real_source; } + std::string skill_file = source_dir.has_value() ? real_source + "/SKILL.md" : real_source; + json::Value skill; + try { + skill = sk.parse_skill_file(skill_file); + } catch (const std::exception& exc) { + return "error: " + short_error(exc); + } + if (skill.is_null()) { + return "error: " + source + " has no valid SKILL.md (needs name, description, and a name of lowercase letters, digits, hyphens)"; + } + std::string name = py::strip(py::str_or_empty(args.get("name"))); + if (name.empty()) { name = py::str(skill.get("name")); } + if (!skill_name_ok(name)) { return "error: invalid skill name, use lowercase letters, digits, and hyphens"; } + if (!mode.has_value()) { + mode = (source_dir.has_value() && !ephemeral && sk.is_external_skill_root(*source_dir)) ? "symlink" : "copy"; + } else if (*mode == "symlink" && (ephemeral || !source_dir.has_value())) { + return "error: symlink mode needs a persistent source directory, not a git/zip source or a bare SKILL.md file"; + } + std::string scope_dir = sk.skill_scope_dir(scope, app.config.home, cwd, ""); + std::string dest = scope_dir + "/" + name; + std::error_code ec; + bool existed = fs::is_symlink(dest, ec) || fs::exists(dest, ec); + if (existed && !force && !app.ask_approval("overwrite existing skill '" + name + "' at " + dest)) { return "denied by user"; } + if (existed) { + if (fs::is_symlink(dest, ec) || fs::is_regular_file(dest, ec)) { fs::remove(dest, ec); } + else { fs::remove_all(dest, ec); } + } + fs::create_directories(scope_dir, ec); + if (*mode == "symlink") { + fs::create_symlink(abspath(*source_dir), dest, ec); + } else if (source_dir.has_value()) { + fs::copy(*source_dir, dest, fs::copy_options::recursive, ec); + } else { + fs::create_directories(dest, ec); + fs::copy_file(real_source, dest + "/SKILL.md", ec); + } + if (ec) { throw std::runtime_error(os_error_message(ec.value(), dest)); } + json::Value meta = json::Value::object(); + meta.set("source", source); + meta.set("mode", *mode); + meta.set("scope", scope); + meta.set("installed_at", py::utc_now_iso()); + sk.save_skill_install_meta(scope_dir, name, meta); + if (app.store != nullptr) { + try { + app.store->audit_event(audit_actor(), "skill-install", dest, + "installed skill '" + name + "' from " + source + " (" + *mode + ", " + scope + " scope)", + std::nullopt, std::nullopt, {"skill", "install"}, std::nullopt, std::nullopt, ""); + } catch (const std::exception&) { + } + } + app.skills = sk.discover_skills(app.config.home, cwd); + app.apply_system(); + auto extras = sk.skill_extra_files(dest); + std::string tail = extras.empty() ? "" : "\nbundled files: " + py::join(extras, ", "); + std::string replaced = existed ? " (replacing the previous install)" : ""; + return "skill '" + name + "' installed (" + *mode + ", " + scope + " scope) at " + dest + " from " + source + replaced + "\n" + + py::str(skill.get("description")) + tail; +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/tools_web.cpp b/taicpp/src/agent/tools_web.cpp new file mode 100644 index 0000000..d6d06cc --- /dev/null +++ b/taicpp/src/agent/tools_web.cpp @@ -0,0 +1,298 @@ +#include "agent.h" +#include "../llm/helpers.h" +#include "../llm/http_transport.h" +#include "../llm/py.h" +#include "../llm/consts.h" +#include "consts.h" +#include "text.h" +#include +#include +#include +namespace detz::agent { + +using llm::HttpRequest; +using llm::HttpResult; +using llm::short_error; + +static const char* http_reason(long code) { + static const std::map t = { + {100, "Continue"}, {101, "Switching Protocols"}, {102, "Processing"}, {103, "Early Hints"}, {200, "OK"}, + {201, "Created"}, {202, "Accepted"}, {203, "Non-Authoritative Information"}, {204, "No Content"}, + {205, "Reset Content"}, {206, "Partial Content"}, {207, "Multi-Status"}, {208, "Already Reported"}, + {226, "IM Used"}, {300, "Multiple Choices"}, {301, "Moved Permanently"}, {302, "Found"}, {303, "See Other"}, + {304, "Not Modified"}, {305, "Use Proxy"}, {307, "Temporary Redirect"}, {308, "Permanent Redirect"}, + {400, "Bad Request"}, {401, "Unauthorized"}, {402, "Payment Required"}, {403, "Forbidden"}, {404, "Not Found"}, + {405, "Method Not Allowed"}, {406, "Not Acceptable"}, {407, "Proxy Authentication Required"}, + {408, "Request Timeout"}, {409, "Conflict"}, {410, "Gone"}, {411, "Length Required"}, + {412, "Precondition Failed"}, {413, "Content Too Large"}, {414, "URI Too Long"}, + {415, "Unsupported Media Type"}, {416, "Range Not Satisfiable"}, {417, "Expectation Failed"}, + {418, "I'm a Teapot"}, {421, "Misdirected Request"}, {422, "Unprocessable Content"}, {423, "Locked"}, + {424, "Failed Dependency"}, {425, "Too Early"}, {426, "Upgrade Required"}, {428, "Precondition Required"}, + {429, "Too Many Requests"}, {431, "Request Header Fields Too Large"}, {451, "Unavailable For Legal Reasons"}, + {500, "Internal Server Error"}, {501, "Not Implemented"}, {502, "Bad Gateway"}, {503, "Service Unavailable"}, + {504, "Gateway Timeout"}, {505, "HTTP Version Not Supported"}, {506, "Variant Also Negotiates"}, + {507, "Insufficient Storage"}, {508, "Loop Detected"}, {510, "Not Extended"}, {511, "Network Authentication Required"}}; + auto it = t.find(code); + return it == t.end() ? "" : it->second; +} + +// str(HTTPError): "HTTP Error 404: Not Found" +static std::string http_error_str(long code) { + return "HTTP Error " + std::to_string(code) + ": " + http_reason(code); +} + +// OSError text for a failed urlopen (HTTPError or URLError) +static std::string url_failure_text(const HttpResult& r) { + if (r.transport_failed()) { return r.error; } + return http_error_str(r.status); +} + +std::string Tools::run_web_search(const json::Value& args) { + std::string query = py::strip(py::str_or_empty(args.get("query"))); + if (query.empty()) { return "error: empty query"; } + std::string params = "query=" + url_quote_plus(query); + if (args.get("images").truthy()) { params += "&type=images"; } + if (args.get("content").truthy()) { params += "&content=true"; } + HttpRequest req; + req.url = std::string(llm::kRsearchBase) + "/search?" + params; + req.headers = {{"Accept", "application/json"}}; + req.timeout = 30; + HttpResult res = llm::transport().send(req); + if (!res.ok()) { return "error: search failed: " + py::head(url_failure_text(res), 300); } + json::Value data; + try { + data = json::Value::loads(py::utf8_replace(res.body)); + } catch (const json::ParseError& exc) { + return "error: search failed: " + short_error(exc); + } + if (!data.is_object()) { throw std::runtime_error("'" + py_type_name(data) + "' object has no attribute 'get'"); } + json::Value results = data.get("results").truthy() ? data.get("results") : json::Value::array(); + std::vector lines; + int pos = 0; + for (const auto& item : results.items()) { + ++pos; + if (!item.is_object()) { throw std::runtime_error("'" + py_type_name(item) + "' object has no attribute 'get'"); } + lines.push_back(std::to_string(pos) + ". " + py::str(item.get("title")) + "\n " + py::str(item.get("url")) + "\n " + + py::head(py::str_or_empty(item.get("description")), 400)); + if (item.get("content").truthy()) { + lines.push_back(" content: " + llm::truncate(py::str(item.get("content")), 2000, 500)); + } + } + result_note = std::to_string(results.size()) + " results"; + return lines.empty() ? std::string("no results") : py::join(lines, "\n"); +} + +std::string Tools::run_describe_image(const json::Value& args) { + std::string source = py::strip(py::str_or_empty(args.get("source"))); + if (source.empty()) { return "error: empty source"; } + HttpRequest req; + req.headers = {{"Accept", "application/json"}}; + if (py::startswith(source, "http://") || py::startswith(source, "https://")) { + req.method = "GET"; + req.url = std::string(llm::kRsearchBase) + "/describe?url=" + url_quote_plus(source); + } else { + std::string path = expanduser(source); + std::error_code ec; + auto size = std::filesystem::file_size(path, ec); + if (ec) { return "error: " + py::head(os_error_message(ec.value(), path), 300); } + if (size > 10u * 1024u * 1024u) { return "error: image larger than 10 MB"; } + FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) { return "error: " + py::head(os_error_message(errno, path), 300); } + std::string body; + char buf[65536]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) { body.append(buf, n); } + std::fclose(f); + req.method = "POST"; + req.url = std::string(llm::kRsearchBase) + "/describe"; + req.headers.emplace_back("Content-Type", "application/octet-stream"); + req.body = body; + } + req.timeout = 60; + HttpResult res = llm::transport().send(req); + if (!res.ok()) { return "error: describe failed: " + py::head(url_failure_text(res), 300); } + json::Value data; + try { + data = json::Value::loads(py::utf8_replace(res.body)); + } catch (const json::ParseError& exc) { + return "error: describe failed: " + short_error(exc); + } + if (!data.is_object()) { throw std::runtime_error("'" + py_type_name(data) + "' object has no attribute 'get'"); } + json::Value d = data.get("description").truthy() ? data.get("description") + : data.get("error").truthy() ? data.get("error") : json::Value("no description"); + std::string text = py::str(d); + result_note = text; + return text; +} + +namespace { +// re.sub(r"(?s)|", " ", text) +std::string strip_script_style(const std::string& s) { + std::string out; + size_t i = 0; + while (i < s.size()) { + if (s[i] == '<') { + if (s.compare(i, 7, "", i + 7); + if (e != std::string::npos) { out += ' '; i = e + 9; continue; } + } + if (s.compare(i, 6, "", i + 6); + if (e != std::string::npos) { out += ' '; i = e + 8; continue; } + } + } + out += s[i++]; + } + return out; +} +// re.sub(r"<[^>]+>", " ", text) +std::string strip_tags(const std::string& s) { + std::string out; + size_t i = 0; + while (i < s.size()) { + if (s[i] == '<') { + size_t e = s.find('>', i + 1); + if (e != std::string::npos && e > i + 1) { out += ' '; i = e + 1; continue; } + } + out += s[i++]; + } + return out; +} +// re.sub(r"\s+", " ", text) +std::string collapse_ws(const std::string& s) { + std::string out; + bool in_ws = false; + for (uint32_t cp : py::decode(s)) { + if (py::is_space_cp(cp)) { + if (!in_ws) { out += ' '; in_ws = true; } + } else { + py::put_utf8(out, cp); + in_ws = false; + } + } + return out; +} +bool header_name_ok(const std::string& s) { + if (s.empty()) { return false; } + for (unsigned char c : s) { if (!(std::isalnum(c) || c == '-') || c >= 0x80) { return false; } } + return true; +} +} // namespace + +std::string Tools::run_web_fetch(const json::Value& args) { + std::string url = py::strip(py::str_or_empty(args.get("url"))); + if (!(py::startswith(url, "http://") || py::startswith(url, "https://"))) { + return "error: url must start with http:// or https://"; + } + std::string method = py::upper(py::strip(args.get("method").truthy() ? py::str(args.get("method")) : "GET")); + const auto& methods = FetchMethods(); + if (std::find(methods.begin(), methods.end(), method) == methods.end()) { + return "error: method must be one of " + py::join(methods, ", "); + } + HttpRequest req; + req.method = method; + req.url = url; + std::vector> headers = {{"User-Agent", std::string("tai/") + llm::kVersion}}; + json::Value custom = args.get("headers").truthy() ? args.get("headers") : json::Value::object(); + if (!custom.is_object()) { return "error: headers must be an object of string names to string values"; } + for (const auto& kv : custom.members()) { + if (!kv.second.is_string()) { return "error: headers must be an object of string names to string values"; } + } + for (const auto& kv : custom.members()) { + std::string nm = py::strip(kv.first); + if (!header_name_ok(nm) || kv.second.str().find('\n') != std::string::npos || + kv.second.str().find('\r') != std::string::npos) { + return "error: invalid header '" + py::head(kv.first, 40) + "'"; + } + bool replaced = false; + for (auto& h : headers) { if (h.first == nm) { h.second = kv.second.str(); replaced = true; } } + if (!replaced) { headers.emplace_back(nm, kv.second.str()); } + } + if (!args.get("body").is_null()) { + if (!args.get("body").is_string()) { return "error: body must be a string"; } + req.body = args.get("body").str(); + } + int64_t timeout; + { + json::Value t = args.get("timeout").truthy() ? args.get("timeout") : json::Value(30); + auto parsed = py::to_int(t); + if (!parsed.has_value()) { return "error: invalid timeout"; } + timeout = *parsed; + } + timeout = std::max(1, std::min(timeout, 120)); + std::string auth_name = py::strip(py::str_or_empty(args.get("auth_secret"))); + if (!auth_name.empty()) { + auto value = S("load_secret").load_secret(auth_name, active_profile()); + if (!value.has_value()) { return "error: unknown secret '" + auth_name + "'"; } + std::string header = py::strip(args.get("auth_header").truthy() ? py::str(args.get("auth_header")) : "Authorization"); + if (header.empty()) { header = "Authorization"; } + if (!header_name_ok(header)) { return "error: invalid auth header name"; } + std::string prefix = !args.get("auth_prefix").is_null() ? py::str(args.get("auth_prefix")) : "Bearer "; + if (prefix.find('\n') != std::string::npos || prefix.find('\r') != std::string::npos) { + return "error: invalid auth prefix"; + } + std::string host = url_hostname(url); + if (!secret_grant("web_fetch", host, {auth_name})) { return "denied by user"; } + bool replaced = false; + for (auto& h : headers) { if (h.first == header) { h.second = prefix + *value; replaced = true; } } + if (!replaced) { headers.emplace_back(header, prefix + *value); } + } + req.headers = headers; + req.timeout = static_cast(timeout); + req.max_bytes = 200000; + HttpResult res = llm::transport().send(req); + if (res.transport_failed()) { return "error: fetch failed: " + py::head(res.error, 300); } + if (!res.ok()) { + std::string detail = py::strip(py::utf8_replace(res.body.substr(0, 2000))); + if (!detail.empty()) { + return "error: HTTP " + std::to_string(res.status) + ": " + llm::truncate(detail, 1000, 300); + } + return "error: HTTP " + std::to_string(res.status) + ": " + http_error_str(res.status); + } + std::string text = py::utf8_replace(res.body); + const std::string& content_type = res.content_type; + bool raw = args.get("raw").truthy(); + if (!raw && (content_type.find("html") != std::string::npos || content_type.empty())) { + std::string stripped = strip_tags(strip_script_style(text)); + text = py::strip(collapse_ws(html_unescape(stripped))); + } + text = llm::truncate(py::strip(text), 5000, 1000); + if (text.empty()) { text = "empty page"; } + std::string ct = py::strip(py::split(content_type, ";")[0]); + std::string head = "HTTP " + std::to_string(res.status) + " \xC2\xB7 " + (ct.empty() ? std::string("unknown") : ct) + " \xC2\xB7 " + + llm::human_size(static_cast(res.body.size())); + return head + "\n" + text; +} + +std::string Tools::run_speak(const json::Value& args) { + std::string text = py::strip(py::str_or_empty(args.get("text"))); + if (text.empty()) { return "error: empty text"; } + std::string voice = args.get("voice").truthy() ? py::str(args.get("voice")) : app.config.voice; + if (ports().ops == nullptr) { throw PortNotWired("ops"); } + std::string audio; + try { + audio = ports().ops->edge_synthesize(py::head(text, 8000), voice); + } catch (const std::exception& exc) { + return "error: speech failed: " + short_error(exc); + } + if (audio.empty()) { return "error: speech returned no audio"; } + std::string path = app.config.audio_dir + "/speech-" + std::to_string(static_cast(py::now())) + ".mp3"; + FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) { return "error: " + py::head(os_error_message(errno, path), 300); } + std::fwrite(audio.data(), 1, audio.size(), f); + std::fclose(f); + return "saved " + path + " (" + std::to_string(audio.size()) + " bytes), played: " + ports().ops->play_audio(path); +} + +std::string Tools::run_listen(const json::Value& args) { + int64_t seconds = 5; + { + json::Value t = args.get("seconds").truthy() ? args.get("seconds") : json::Value(5); + auto parsed = py::to_int(t); + seconds = parsed.has_value() ? std::max(1, std::min(30, *parsed)) : 5; + } + if (ports().ops == nullptr) { throw PortNotWired("ops"); } + return ports().ops->listen_audio(static_cast(seconds), app.config.audio_dir); +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/zipfile.cpp b/taicpp/src/agent/zipfile.cpp new file mode 100644 index 0000000..d09562c --- /dev/null +++ b/taicpp/src/agent/zipfile.cpp @@ -0,0 +1,263 @@ +#include "zipfile.h" +#include "text.h" +#include "../llm/py.h" +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::agent { + +namespace { + +// ---------------------------------------------------------------- inflate +class Inflater { + public: + Inflater(const uint8_t* data, size_t size) : in_(data), size_(size) {} + std::string run() { + int last; + do { + last = bits(1); + int type = bits(2); + if (type == 0) { stored(); } + else if (type == 1) { fixed(); } + else if (type == 2) { dynamic(); } + else { throw BadZipFile("Error -3 while decompressing data: invalid block type"); } + } while (!last); + return out_; + } + + private: + struct Huff { std::vector count, symbol; }; + + int bits(int need) { + long val = bitbuf_; + while (bitcnt_ < need) { + if (pos_ >= size_) { throw BadZipFile("Error -5 while decompressing data: incomplete or truncated stream"); } + val |= static_cast(in_[pos_++]) << bitcnt_; + bitcnt_ += 8; + } + bitbuf_ = static_cast(val >> need); + bitcnt_ -= need; + return static_cast(val & ((1L << need) - 1)); + } + + void stored() { + bitbuf_ = 0; + bitcnt_ = 0; + if (pos_ + 4 > size_) { throw BadZipFile("Error -5 while decompressing data: incomplete or truncated stream"); } + unsigned len = in_[pos_] | (in_[pos_ + 1] << 8); + unsigned nlen = in_[pos_ + 2] | (in_[pos_ + 3] << 8); + pos_ += 4; + if (len != (~nlen & 0xFFFF)) { throw BadZipFile("Error -3 while decompressing data: invalid stored block lengths"); } + if (pos_ + len > size_) { throw BadZipFile("Error -5 while decompressing data: incomplete or truncated stream"); } + out_.append(reinterpret_cast(in_ + pos_), len); + pos_ += len; + } + + static Huff build(const short* length, int n) { + Huff h; + h.count.assign(16, 0); + h.symbol.assign(static_cast(n), 0); + for (int s = 0; s < n; ++s) { h.count[static_cast(length[s])]++; } + std::vector offs(16, 0); + for (int len = 1; len < 15; ++len) { offs[static_cast(len + 1)] = static_cast(offs[static_cast(len)] + h.count[static_cast(len)]); } + for (int s = 0; s < n; ++s) { + if (length[s] != 0) { h.symbol[static_cast(offs[static_cast(length[s])]++)] = static_cast(s); } + } + return h; + } + + int decode(const Huff& h) { + int code = 0, first = 0, index = 0; + for (int len = 1; len <= 15; ++len) { + code |= bits(1); + int count = h.count[static_cast(len)]; + if (code - count < first) { return h.symbol[static_cast(index + (code - first))]; } + index += count; + first += count; + first <<= 1; + code <<= 1; + } + throw BadZipFile("Error -3 while decompressing data: invalid code"); + } + + void codes(const Huff& lencode, const Huff& distcode) { + static const short lens[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258}; + static const short lext[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; + static const short dists[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; + static const short dext[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; + for (;;) { + int symbol = decode(lencode); + if (symbol < 256) { + out_ += static_cast(symbol); + } else if (symbol == 256) { + return; + } else { + symbol -= 257; + if (symbol >= 29) { throw BadZipFile("Error -3 while decompressing data: invalid literal/length code"); } + int len = lens[symbol] + bits(lext[symbol]); + int ds = decode(distcode); + if (ds >= 30) { throw BadZipFile("Error -3 while decompressing data: invalid distance code"); } + size_t dist = static_cast(dists[ds] + bits(dext[ds])); + if (dist > out_.size()) { throw BadZipFile("Error -3 while decompressing data: invalid distance too far back"); } + for (int i = 0; i < len; ++i) { out_ += out_[out_.size() - dist]; } + } + } + } + + void fixed() { + short lengths[288]; + int s = 0; + for (; s < 144; ++s) { lengths[s] = 8; } + for (; s < 256; ++s) { lengths[s] = 9; } + for (; s < 280; ++s) { lengths[s] = 7; } + for (; s < 288; ++s) { lengths[s] = 8; } + Huff lencode = build(lengths, 288); + short dl[30]; + for (int i = 0; i < 30; ++i) { dl[i] = 5; } + Huff distcode = build(dl, 30); + codes(lencode, distcode); + } + + void dynamic() { + static const short order[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + int nlen = bits(5) + 257, ndist = bits(5) + 1, ncode = bits(4) + 4; + if (nlen > 286 || ndist > 30) { throw BadZipFile("Error -3 while decompressing data: too many length or distance symbols"); } + short lengths[320]; + int index = 0; + for (; index < ncode; ++index) { lengths[order[index]] = static_cast(bits(3)); } + for (; index < 19; ++index) { lengths[order[index]] = 0; } + Huff lencode = build(lengths, 19); + index = 0; + while (index < nlen + ndist) { + int symbol = decode(lencode); + if (symbol < 16) { + lengths[index++] = static_cast(symbol); + } else { + int len = 0, rep; + if (symbol == 16) { + if (index == 0) { throw BadZipFile("Error -3 while decompressing data: invalid bit length repeat"); } + len = lengths[index - 1]; + rep = 3 + bits(2); + } else if (symbol == 17) { + rep = 3 + bits(3); + } else { + rep = 11 + bits(7); + } + if (index + rep > nlen + ndist) { throw BadZipFile("Error -3 while decompressing data: invalid bit length repeat"); } + while (rep--) { lengths[index++] = static_cast(len); } + } + } + Huff lc = build(lengths, nlen); + Huff dc = build(lengths + nlen, ndist); + codes(lc, dc); + } + + const uint8_t* in_; + size_t size_; + size_t pos_{0}; + int bitbuf_{0}; + int bitcnt_{0}; + std::string out_; +}; + +uint32_t crc32_of(const std::string& data) { + static uint32_t table[256]; + static bool init = false; + if (!init) { + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (int k = 0; k < 8; ++k) { c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; } + table[i] = c; + } + init = true; + } + uint32_t crc = 0xFFFFFFFFu; + for (unsigned char b : data) { crc = table[(crc ^ b) & 0xFF] ^ (crc >> 8); } + return crc ^ 0xFFFFFFFFu; +} + +uint16_t u16(const std::string& s, size_t o) { return static_cast(static_cast(s[o]) | (static_cast(s[o + 1]) << 8)); } +uint32_t u32(const std::string& s, size_t o) { + return static_cast(static_cast(s[o])) | (static_cast(static_cast(s[o + 1])) << 8) | + (static_cast(static_cast(s[o + 2])) << 16) | (static_cast(static_cast(s[o + 3])) << 24); +} + +std::string sanitize(const std::string& name) { + std::string out; + size_t pos = 0; + while (pos <= name.size()) { + size_t slash = name.find('/', pos); + std::string part = name.substr(pos, slash == std::string::npos ? std::string::npos : slash - pos); + if (!(part.empty() || part == "." || part == "..")) { + if (!out.empty()) { out += '/'; } + out += part; + } + if (slash == std::string::npos) { break; } + pos = slash + 1; + } + return out; +} +} // namespace + +void zip_extractall(const std::string& zip_path, const std::string& dest) { + FILE* f = std::fopen(zip_path.c_str(), "rb"); + if (f == nullptr) { throw std::runtime_error(os_error_message(errno, zip_path)); } + std::string data; + char buf[65536]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) { data.append(buf, n); } + std::fclose(f); + // end of central directory + long eocd = -1; + size_t lo = data.size() > 22 + 65535 ? data.size() - 22 - 65535 : 0; + for (long i = static_cast(data.size()) - 22; i >= static_cast(lo); --i) { + if (u32(data, static_cast(i)) == 0x06054b50u) { eocd = i; break; } + } + if (eocd < 0) { throw BadZipFile("File is not a zip file"); } + size_t entries = u16(data, static_cast(eocd) + 10); + size_t cd_off = u32(data, static_cast(eocd) + 16); + size_t p = cd_off; + fs::create_directories(dest); + for (size_t e = 0; e < entries; ++e) { + if (p + 46 > data.size() || u32(data, p) != 0x02014b50u) { throw BadZipFile("Bad magic number for central directory"); } + uint16_t method = u16(data, p + 10); + uint32_t crc = u32(data, p + 16), csize = u32(data, p + 20), usize = u32(data, p + 24); + uint16_t nlen = u16(data, p + 28), xlen = u16(data, p + 30), clen = u16(data, p + 32); + uint32_t lho = u32(data, p + 42); + std::string name = data.substr(p + 46, nlen); + p += 46u + nlen + xlen + clen; + std::string arc = sanitize(name); + bool is_dir = !name.empty() && name.back() == '/'; + fs::path target = fs::path(dest) / arc; + if (is_dir) { + std::error_code ec; + fs::create_directories(target, ec); + continue; + } + if (static_cast(lho) + 30 > data.size() || u32(data, lho) != 0x04034b50u) { throw BadZipFile("Bad magic number for file header"); } + size_t dstart = lho + 30u + u16(data, lho + 26) + u16(data, lho + 28); + if (dstart + csize > data.size()) { throw BadZipFile("Truncated file header"); } + std::string content; + if (method == 0) { + content = data.substr(dstart, csize); + } else if (method == 8) { + Inflater inf(reinterpret_cast(data.data()) + dstart, csize); + content = inf.run(); + } else { + throw std::runtime_error("That compression method is not supported"); + } + if (content.size() != usize || crc32_of(content) != crc) { throw BadZipFile("Bad CRC-32 for file " + py::repr_str(name)); } + std::error_code ec; + fs::create_directories(target.parent_path(), ec); + FILE* out = std::fopen(target.c_str(), "wb"); + if (out == nullptr) { throw std::runtime_error(os_error_message(errno, target.string())); } + if (!content.empty()) { std::fwrite(content.data(), 1, content.size(), out); } + std::fclose(out); + } +} + +} // namespace detz::agent diff --git a/taicpp/src/agent/zipfile.h b/taicpp/src/agent/zipfile.h new file mode 100644 index 0000000..442544e --- /dev/null +++ b/taicpp/src/agent/zipfile.h @@ -0,0 +1,15 @@ +#pragma once +// Minimal zipfile.ZipFile(path).extractall(dest): stored and deflate members, +// Python's member-name sanitising. No dependency beyond the standard library. +#include +#include +namespace detz::agent { + +struct BadZipFile : public std::runtime_error { + using std::runtime_error::runtime_error; +}; +// Throws BadZipFile (not a zip, unsupported method, corrupt data) or +// std::runtime_error with OSError-style text when files cannot be written. +void zip_extractall(const std::string& zip_path, const std::string& dest); + +} // namespace detz::agent diff --git a/taicpp/src/llm/chat_client.cpp b/taicpp/src/llm/chat_client.cpp new file mode 100644 index 0000000..913fff1 --- /dev/null +++ b/taicpp/src/llm/chat_client.cpp @@ -0,0 +1,1031 @@ +#include "chat_client.h" +#include "consts.h" +#include "http_transport.h" +#include "py.h" +#include "term.h" +#include +#include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::llm { + +namespace { +std::set& native_tools() { static std::set s; return s; } +std::set& core_tools() { static std::set s; return s; } + +class MuseTempDir { + public: + MuseTempDir() { + std::string tmpl = (fs::temp_directory_path() / "tai-muse-XXXXXX").string(); + std::vector buf(tmpl.begin(), tmpl.end()); + buf.push_back('\0'); + if (mkdtemp(buf.data()) != nullptr) { path_ = buf.data(); } + } + ~MuseTempDir() { + if (!path_.empty()) { std::error_code ec; fs::remove_all(path_, ec); } + } + MuseTempDir(const MuseTempDir&) = delete; + MuseTempDir& operator=(const MuseTempDir&) = delete; + const std::string& path() const { return path_; } + private: + std::string path_; +}; + +std::string http_error_text(const HttpResult& res) { + return "HTTP " + std::to_string(res.status) + ": " + py::head(res.body, 300); +} + +bool contains(const std::vector& v, const std::string& s) { + return std::find(v.begin(), v.end(), s) != v.end(); +} +} // namespace + +std::string CliRaw::text() const { return !py::strip(outfile).empty() ? outfile : out; } + +void ChatClient::set_tool_catalog(std::set native, std::set core) { + native_tools() = std::move(native); + core_tools() = std::move(core); +} +std::function& ChatClient::sleeper() { + static std::function s = [](double secs) { + std::this_thread::sleep_for(std::chrono::duration(secs)); + }; + return s; +} + +ChatClient::ChatClient(LlmConfig& config) + : prefer(std::make_pair(std::string(kDevplaceLabel), std::string(kPreferredModel))), + health(config.home), + config_(config) {} + +std::string ChatClient::muse_key() const { + if (fixed_key.has_value()) { return *fixed_key; } + return scope.first + "/" + scope.second; +} + +void ChatClient::reset_muse() { + if (config_.home.empty()) { return; } + std::string key = muse_key(); + muse_transact(config_.home, [&](json::Value& data) { data.erase(key); }); +} + +std::vector ChatClient::openrouter_models() { + std::vector models; + for (const auto& m : config_.openrouter_models) { if (!m.empty()) { models.push_back(m); } } + const std::string& explicit_model = config_.explicit_model; + if (!explicit_model.empty() && !contains(models, explicit_model)) { models.insert(models.begin(), explicit_model); } + auto free = health.cached_list("openrouter:free"); + std::vector free_list; + if (!free.has_value()) { + auto fetched = fetch_model_ids(std::string(kOpenrouterBase) + "/models", config_.openrouter_key); + if (fetched.has_value()) { + for (const auto& i : *fetched) { if (is_free_openrouter(i)) { free_list.push_back(i); } } + } + if (!free_list.empty()) { health.store_list("openrouter:free", free_list); } + } else { + free_list = *free; + } + free_list = prefer_muse(free_list); + std::vector merged; + for (const auto& i : free_list) { if (!contains(models, i)) { merged.push_back(i); } } + for (const auto& i : models) { merged.push_back(i); } + if (merged.empty()) { merged.push_back(kOpenrouterModel); } + return merged; +} + +std::vector ChatClient::update_models() { + struct Source { std::string name, label; std::function>()> fetch; }; + std::vector sources; + sources.push_back({"opencode:free", kOpencodeLabel, []() -> std::optional> { + if (opencode_binary().empty()) { return std::nullopt; } + std::vector out; + if (opencode_free_models(out)) { return out; } + return std::nullopt; + }}); + if (config_.openrouter_enabled) { + sources.push_back({"openrouter:free", kOpenrouterLabel, [this]() -> std::optional> { + auto fetched = fetch_model_ids(std::string(kOpenrouterBase) + "/models", config_.openrouter_key); + if (!fetched.has_value()) { return std::nullopt; } + std::vector out; + for (const auto& i : *fetched) { if (is_free_openrouter(i)) { out.push_back(i); } } + return out; + }}); + } + if (!config_.devplace_key.empty()) { + sources.push_back({"devplace:free", kDevplaceLabel, [this]() -> std::optional> { + auto fetched = fetch_model_ids(config_.devplace_base + "/models", config_.devplace_key); + if (!fetched.has_value() || fetched->empty()) { return std::nullopt; } + std::vector out; + for (const auto& i : *fetched) { if (contains(DevplaceFreeRoutes(), i) || is_free_model(i)) { out.push_back(i); } } + return out; + }}); + } + if (config_.zen_enabled) { + sources.push_back({"zen:free", kZenLabel, []() -> std::optional> { + auto fetched = fetch_model_ids(std::string(kZenBase) + "/models", "", opencode_headers()); + std::vector out; + if (fetched.has_value()) { for (const auto& i : *fetched) { if (is_free_model(i)) { out.push_back(i); } } } + if (out.empty()) { return std::nullopt; } + return out; + }}); + } + if (config_.pollinations_enabled) { + sources.push_back({"pollinations:free", kPollinationsLabel, []() { return fetch_pollinations_free(); }}); + } + if (config_.ollama_enabled) { + sources.push_back({"ollama:local", kOllamaLabel, []() { return fetch_model_ids(std::string(kOllamaBase) + "/models", "", {}, 3); }}); + } + std::vector report; + for (const auto& src : sources) { + std::optional> fresh; + try { + fresh = src.fetch(); + } catch (const std::exception& exc) { + fresh = std::nullopt; + print_err(paint(src.label + ": discovery failed: " + short_error(exc), Ansi::DIM)); + } + if (!fresh.has_value() || fresh->empty()) { + report.push_back({src.label, std::nullopt, {}, {}}); + continue; + } + ListDiff diff = health.replace_list(src.name, src.label, *fresh); + report.push_back({src.label, fresh->size(), diff.added, diff.removed}); + } + return report; +} + +std::vector ChatClient::opencode_models() { + std::vector models = config_.opencode_models; + if (models.empty()) { + auto cached = health.cached_list("opencode:free"); + if (!cached.has_value()) { + std::vector found; + if (!opencode_free_models(found) || found.empty()) { + found.clear(); + for (const auto& i : ZenFreeModels()) { if (is_free_model(i)) { found.push_back(i); } } + } + models = found; + health.store_list("opencode:free", models); + } else { + models = *cached; + } + } + return prefer_muse(models); +} + +std::vector ChatClient::zen_models() { + std::vector models = config_.zen_models; + if (!models.empty()) { return prefer_muse(models); } + auto cached = health.cached_list("zen:free"); + std::vector out; + if (!cached.has_value()) { + std::vector fetched; + auto ids = fetch_model_ids(std::string(kZenBase) + "/models", "", opencode_headers()); + if (ids.has_value()) { for (const auto& i : *ids) { if (is_free_model(i)) { fetched.push_back(i); } } } + out = !fetched.empty() ? fetched : ZenFreeModels(); + if (!fetched.empty()) { health.store_list("zen:free", fetched); } + } else { + out = *cached; + } + return prefer_muse(out); +} + +std::vector ChatClient::pollinations_models() { + if (!config_.pollinations_models.empty()) { return config_.pollinations_models; } + auto cached = health.cached_list("pollinations:free"); + if (!cached.has_value()) { + auto fetched = fetch_pollinations_free(); + std::vector out = (fetched.has_value() && !fetched->empty()) ? *fetched : PollinationsModels(); + if (fetched.has_value() && !fetched->empty()) { health.store_list("pollinations:free", *fetched); } + return out; + } + return *cached; +} + +std::vector ChatClient::ollama_models() { + if (!config_.ollama_enabled) { return {}; } + std::vector models = config_.ollama_models; + if (models.empty()) { + auto cached = health.cached_list("ollama:local", kOllamaListTtl); + if (!cached.has_value()) { + auto fetched = fetch_model_ids(std::string(kOllamaBase) + "/models", "", {}, 3); + models = fetched.value_or(std::vector{}); + if (!models.empty()) { health.store_list("ollama:local", models); } + } else { + models = *cached; + } + } + return models; +} + +std::vector ChatClient::devplace_models() { + std::vector models = config_.devplace_models; + if (!models.empty()) { return prefer_muse(models); } + auto cached = health.cached_list("devplace:free"); + std::vector free; + if (!cached.has_value()) { + auto fetched = fetch_model_ids(config_.devplace_base + "/models", config_.devplace_key); + if (!fetched.has_value()) { return DevplaceFreeRoutes(); } + for (const auto& i : *fetched) { if (contains(DevplaceFreeRoutes(), i) || is_free_model(i)) { free.push_back(i); } } + if (!free.empty()) { health.store_list("devplace:free", free); } + } else { + free = *cached; + } + return prefer_muse(free); +} + +std::vector ChatClient::candidates() { + std::vector found; + auto add = [&](const std::string& provider, const std::string& chat_url, const std::string& models_url, + const std::string& key, const std::string& model, const std::string& profile) { + Candidate c; + c.provider = provider; c.chat_url = chat_url; c.models_url = models_url; + c.key = key; c.model = model; c.profile = profile; + found.push_back(c); + }; + if (config_.opencode_enabled && !opencode_binary().empty()) { + for (const auto& model : opencode_models()) { add(kOpencodeLabel, kOpencodeChatUrl, kOpencodeModelsUrl, "", model, ""); } + } + for (const auto& model : ollama_models()) { + add(kOllamaLabel, std::string(kOllamaBase) + "/chat/completions", std::string(kOllamaBase) + "/models", "", model, ""); + } + if (!config_.devplace_key.empty()) { + for (const auto& model : devplace_models()) { + add(kDevplaceLabel, config_.devplace_base + "/chat/completions", config_.devplace_base + "/models", config_.devplace_key, model, ""); + } + } + for (const auto& entry : config_.backends) { + std::vector models = split_models(entry.model); + if (models.empty()) { models.push_back(config_.model); } + for (const auto& model : models) { + add(entry.label, entry.base + "/chat/completions", entry.base + "/models", entry.key, model, entry.profile); + } + } + if (config_.openrouter_enabled) { + for (const auto& model : openrouter_models()) { + add(kOpenrouterLabel, std::string(kOpenrouterBase) + "/chat/completions", std::string(kOpenrouterBase) + "/models", config_.openrouter_key, model, ""); + } + } + if (config_.pollinations_enabled) { + for (const auto& model : pollinations_models()) { + add(kPollinationsLabel, std::string(kPollinationsBase) + "/chat/completions", std::string(kPollinationsBase) + "/models", "", model, ""); + } + } + if (config_.zen_enabled) { + for (const auto& model : zen_models()) { + add(kZenLabel, std::string(kZenBase) + "/chat/completions", std::string(kZenBase) + "/models", "", model, "opencode"); + } + } + add(kMuseLabel, kMuseChatUrl, kMuseModelsUrl, "", config_.model, ""); + if (!grok_binary().empty()) { add(kGrokLabel, kGrokChatUrl, kGrokModelsUrl, "", config_.model, ""); } + if (!claude_binary().empty()) { add(kClaudeLabel, kClaudeChatUrl, kClaudeModelsUrl, "", config_.model, ""); } + if (!codex_binary().empty()) { add(kCodexLabel, kCodexChatUrl, kCodexModelsUrl, "", config_.model, ""); } + if (!gemini_binary().empty()) { add(kGeminiLabel, kGeminiChatUrl, kGeminiModelsUrl, "", config_.model, ""); } + return found; +} + +static bool is_cli_url(const std::string& url) { return contains(CliChatUrls(), url); } + +std::vector ChatClient::ordered_candidates() { + std::vector all = candidates(); + std::vector http, cli; + for (auto& c : all) { (is_cli_url(c.chat_url) ? cli : http).push_back(c); } + std::vector idx(http.size()); + for (size_t i = 0; i < idx.size(); ++i) { idx[i] = i; } + std::stable_sort(idx.begin(), idx.end(), [&](size_t a, size_t b) { + int ta = openrouter_free_tier(http[a]), tb = openrouter_free_tier(http[b]); + if (ta != tb) { return ta < tb; } + double wa = health.weight(http[a].provider, http[a].model), wb = health.weight(http[b].provider, http[b].model); + if (wa != wb) { return wa > wb; } + return a < b; + }); + std::vector ranked, usable; + for (size_t i : idx) { ranked.push_back(http[i]); } + for (auto& c : ranked) { if (!health.circuit_open(c.provider, c.model)) { usable.push_back(c); } } + std::vector base = usable.empty() ? ranked : usable; + for (auto& c : cli) { base.push_back(c); } + return apply_pins(base); +} + +std::vector ChatClient::apply_pins(std::vector cands) { + std::vector pinned = cands; + if (provider_pin.has_value() && !provider_pin->empty()) { + std::vector keep; + for (auto& c : pinned) { if (py::lower(c.provider) == *provider_pin) { keep.push_back(c); } } + pinned = keep; + } + if (model_pin.has_value() && !model_pin->empty()) { + std::vector keep; + for (auto& c : pinned) { if (c.model == *model_pin) { keep.push_back(c); } } + pinned = keep; + } + if (pinned.empty() && provider_pin.has_value() && *provider_pin == kDevplaceLabel && model_pin.has_value() && + !model_pin->empty() && !config_.devplace_key.empty()) { + Candidate c; + c.provider = kDevplaceLabel; + c.chat_url = config_.devplace_base + "/chat/completions"; + c.models_url = config_.devplace_base + "/models"; + c.key = config_.devplace_key; + c.model = *model_pin; + pinned.push_back(c); + } + if (pinned.empty() && !cands.empty() && !pin_fallback_said) { + pin_fallback_said = true; + print_err(paint(pin_note() + ": trying every other backend", Ansi::DIM)); + } + // cands = pinned + [cand for cand in cands if not any(cand is item for item in pinned)] + // `is` identity: a pinned element is one of cands, except the synthesised devplace route. + std::vector out = pinned; + auto same = [](const Candidate& a, const Candidate& b) { + return a.provider == b.provider && a.chat_url == b.chat_url && a.models_url == b.models_url && a.key == b.key && + a.model == b.model && a.profile == b.profile && a.runner == b.runner && a.engine == b.engine; + }; + // Identity semantics: each pinned item consumes exactly the element it came from. + std::vector consumed(cands.size(), false); + { + // pinned entries that came from cands are matched left to right. + size_t from = 0; + for (const auto& p : pinned) { + for (size_t i = from; i < cands.size(); ++i) { + if (!consumed[i] && same(cands[i], p)) { consumed[i] = true; from = i + 1; break; } + } + } + } + for (size_t i = 0; i < cands.size(); ++i) { if (!consumed[i]) { out.push_back(cands[i]); } } + cands = out; + if (prefer.has_value() && !(provider_pin.has_value() && !provider_pin->empty()) && !(model_pin.has_value() && !model_pin->empty())) { + const std::string& label = prefer->first; + const std::string& model = prefer->second; + std::vector first; + for (auto& c : cands) { if (c.provider == label && c.model == model) { first.push_back(c); } } + if (first.empty() && label == kDevplaceLabel && !config_.devplace_key.empty()) { + Candidate c; + c.provider = kDevplaceLabel; + c.chat_url = config_.devplace_base + "/chat/completions"; + c.models_url = config_.devplace_base + "/models"; + c.key = config_.devplace_key; + c.model = model; + first.push_back(c); + } + std::vector rest; + std::vector used(cands.size(), false); + size_t from = 0; + for (const auto& f : first) { + for (size_t i = from; i < cands.size(); ++i) { + if (!used[i] && same(cands[i], f)) { used[i] = true; from = i + 1; break; } + } + } + for (size_t i = 0; i < cands.size(); ++i) { if (!used[i]) { rest.push_back(cands[i]); } } + std::vector merged = first; + for (auto& c : rest) { merged.push_back(c); } + cands = merged; + } + return cands; +} + +std::string ChatClient::pin_note() const { + std::vector bits; + if (provider_pin.has_value() && !provider_pin->empty()) { bits.push_back("provider '" + *provider_pin + "'"); } + if (model_pin.has_value() && !model_pin->empty()) { bits.push_back("model '" + *model_pin + "'"); } + if (bits.empty()) { return "no backends available"; } + return "no backend offers " + py::join(bits, " and "); +} + +std::vector ChatClient::backends() { + std::vector rows; + std::set> seen; + for (const auto& c : const_cast(this)->candidates()) { + auto key = std::make_pair(c.provider, c.chat_url); + if (seen.count(key)) { continue; } + seen.insert(key); + rows.push_back({c.provider, c.chat_url, c.models_url, c.key, c.model}); + } + return rows; +} + +std::vector> ChatClient::headers( + const std::string& key, const std::vector>& extra) const { + std::vector> h = {{"Content-Type", "application/json"}}; + auto put = [&](const std::string& k, const std::string& v) { + for (auto& kv : h) { if (kv.first == k) { kv.second = v; return; } } + h.emplace_back(k, v); + }; + if (!key.empty()) { put("Authorization", "Bearer " + key); } + for (const auto& kv : extra) { put(kv.first, kv.second); } + return h; +} + +std::string ChatClient::summarize_errors(const std::vector& errors, size_t shown, size_t width) { + if (errors.empty()) { return "no details"; } + std::vector actionable; + for (const auto& e : errors) { if (e.find("CLI not found") == std::string::npos) { actionable.push_back(e); } } + const std::vector& pool = actionable.empty() ? errors : actionable; + std::vector unique; + for (const auto& e : pool) { if (!contains(unique, e)) { unique.push_back(e); } } + std::vector head(unique.begin(), unique.begin() + static_cast(std::min(shown, unique.size()))); + std::string text = py::join(head, "; "); + if (unique.size() > shown) { text += "; +" + std::to_string(unique.size() - shown) + " more"; } + size_t skipped = errors.size() - pool.size(); + if (skipped) { text += " (" + std::to_string(skipped) + " missing CLI skipped)"; } + return py::head(text, static_cast(width)); +} + +ChatReply ChatClient::complete(const json::Value& messages, const json::Value& tools, const StreamSink* stream_sink, + bool ephemeral, std::optional deadline) { + try { + return complete_inner(messages, tools, stream_sink, ephemeral, deadline); + } catch (const BackendError& exc) { + json::Value meta = json::Value::object(); + meta.set("error", short_error(exc)); + log_event("ERROR", "llm", "fatal", "all backends failed: " + short_error(exc), scope.first, scope.second, meta); + throw; + } +} + +ChatReply ChatClient::complete_inner(const json::Value& in_messages, const json::Value& tools, const StreamSink* sink, + bool ephemeral, std::optional deadline) { + json::Value messages = clean_history(in_messages); + json::Value payload = json::Value::object(); + payload.set("model", config_.model); + payload.set("messages", messages); + if (tools.truthy()) { + payload.set("tools", tools); + payload.set("tool_choice", "auto"); + } + double limit = config_.retry_seconds; + std::optional give_up; + if (limit > 0) { give_up = py::now() + limit; } + if (deadline.has_value()) { give_up = give_up.has_value() ? std::min(*give_up, *deadline) : *deadline; } + int round_no = 0; + for (;;) { + std::vector errors; + round_transient = false; + std::vector cands = round_no == 0 ? ordered_candidates() : all_candidates(); + if (cands.empty()) { throw BackendError(pin_note()); } + for (const auto& cand : cands) { + auto reply = try_candidate(cand, messages, tools, payload, sink, ephemeral, errors); + if (reply.has_value()) { return *reply; } + } + if (!round_transient) { + throw BackendError("no backend can take this request, retrying will not help (" + summarize_errors(errors, 6, 900) + ")"); + } + const auto& backoff = RetryBackoff(); + int pause = backoff[std::min(static_cast(round_no), backoff.size() - 1)]; + round_no += 1; + if (give_up.has_value() && py::now() + pause > *give_up) { + throw BackendError("all backends failed after " + std::to_string(round_no) + " rounds (" + summarize_errors(errors, 6, 900) + ")"); + } + std::set> providers; + for (const auto& c : cands) { providers.insert({c.provider, c.chat_url}); } + json::Value meta = json::Value::object(); + meta.set("round", round_no); + meta.set("pause_s", pause); + meta.set("candidates", static_cast(cands.size())); + meta.set("errors", summarize_errors(errors)); + log_event("WARNING", "llm", "round_retry", + "all " + std::to_string(cands.size()) + " candidates failed (round " + std::to_string(round_no) + "), retrying in " + std::to_string(pause) + "s", + scope.first, scope.second, meta); + if (stderr_isatty()) { std::fputs("\n", stderr); } + print_err(paint("all " + std::to_string(cands.size()) + " candidate" + (cands.size() == 1 ? "" : "s") + " across " + + std::to_string(providers.size()) + " backend" + (providers.size() == 1 ? "" : "s") + " failed (round " + + std::to_string(round_no) + "), retrying in " + std::to_string(pause) + "s: " + summarize_errors(errors), + Ansi::DIM)); + if (round_no == 1 && opencode_binary().empty()) { + BoxPort* box = box_port(); + std::string engine = box != nullptr ? box->container_engine() : ""; + if (!engine.empty() && box->box_state(engine) == "missing") { + print_err(paint( + "no local opencode CLI, and no sandbox box set up yet either -- that box " + "would also work as a last-resort backend (see box_opencode_candidates()). " + "Run `tai.py --env sandbox` once to build it: a small image, podman or " + "docker, and it only ever touches that one container, never the rest of " + "this machine.", Ansi::DIM)); + } + } + sleeper()(static_cast(pause)); + } +} + +std::vector ChatClient::all_candidates() { + std::vector all = candidates(); + std::vector http, cli; + for (auto& c : all) { (is_cli_url(c.chat_url) ? cli : http).push_back(c); } + std::vector idx(http.size()); + for (size_t i = 0; i < idx.size(); ++i) { idx[i] = i; } + std::stable_sort(idx.begin(), idx.end(), [&](size_t a, size_t b) { + int ta = openrouter_free_tier(http[a]), tb = openrouter_free_tier(http[b]); + if (ta != tb) { return ta < tb; } + double wa = health.weight(http[a].provider, http[a].model), wb = health.weight(http[b].provider, http[b].model); + if (wa != wb) { return wa > wb; } + return a < b; + }); + std::vector merged; + for (size_t i : idx) { merged.push_back(http[i]); } + for (auto& c : cli) { merged.push_back(c); } + for (auto& c : box_opencode_candidates()) { merged.push_back(c); } + return apply_pins(merged); +} + +std::vector ChatClient::box_opencode_candidates() { + if (!config_.opencode_enabled || !opencode_binary().empty()) { return {}; } + BoxPort* box = box_port(); + if (box == nullptr) { return {}; } + std::string engine = box->container_engine(); + if (engine.empty() || box->box_state(engine) != "running") { return {}; } + if (box->box_opencode_binary(engine).empty()) { return {}; } + std::vector out; + for (const auto& model : box_opencode_models(engine)) { + Candidate c; + c.provider = kOpencodeLabel; c.chat_url = kOpencodeChatUrl; c.models_url = kOpencodeModelsUrl; + c.model = model; c.runner = "box"; c.engine = engine; + out.push_back(c); + } + return out; +} + +std::vector ChatClient::box_opencode_models(const std::string& engine) { + auto cached = health.cached_list("opencode:box:free"); + if (cached.has_value()) { return prefer_muse(*cached); } + BoxPort* box = box_port(); + if (box == nullptr) { return {}; } + ProcResult done = box->box_exec(engine, {"opencode", "models", "opencode"}, kModelListTimeout * 4); + if (done.spawn_failed || done.timed_out) { return {}; } + std::vector models; + for (const auto& line : py::splitlines(done.out)) { + std::string s = py::strip(line); + if (py::startswith(s, "opencode/")) { models.push_back(s.substr(s.find('/') + 1)); } + } + std::vector free; + for (auto& m : models) { if (is_free_model(m)) { free.push_back(m); } } + if (!free.empty()) { health.store_list("opencode:box:free", free); } + return prefer_muse(free); +} + +ChatReply ChatClient::call_candidate(const Candidate& cand, const json::Value& messages, const json::Value& tools, + const json::Value& attempt, const StreamSink* sink, bool ephemeral) { + const std::string& label = cand.provider; + const std::string& chat_url = cand.chat_url; + std::vector> extra; + if (cand.profile == "opencode") { extra = opencode_headers(); } + double timeout = sink != nullptr ? kStreamTimeout : kHttpTimeout; + if (chat_url == kMuseChatUrl) { return muse_turn(messages, tools, sink, label, timeout, ephemeral); } + auto raw_out = [](const std::string& out) { CliRaw r; r.out = out; return r; }; + if (chat_url == kGrokChatUrl) { + return cli_json_turn(label, messages, tools, sink, timeout, + [&](const std::string& p, double t) { return raw_out(grok_run(p, t)); }, + [](const CliRaw& r) { return grok_parse_output(r.out); }); + } + if (chat_url == kClaudeChatUrl) { + return cli_json_turn(label, messages, tools, sink, timeout, + [&](const std::string& p, double t) { return raw_out(claude_run(p, t)); }, + [](const CliRaw& r) { return claude_parse_output(r.out); }); + } + if (chat_url == kCodexChatUrl) { + return cli_json_turn(label, messages, tools, sink, timeout, + [&](const std::string& p, double t) { + CodexOut c = codex_run(p, t); + CliRaw r; r.out = c.stdout_text; r.outfile = c.outfile_text; r.used_schema = c.used_schema; + return r; + }, + [](const CliRaw& r) { return codex_parse_output(r.out, r.outfile, r.used_schema); }); + } + if (chat_url == kGeminiChatUrl) { + return cli_json_turn(label, messages, tools, sink, timeout, + [&](const std::string& p, double t) { return raw_out(gemini_run(p, t)); }, + [](const CliRaw& r) { return gemini_parse_output(r.out); }); + } + if (chat_url == kOpencodeChatUrl) { + if (cand.runner == "box") { + BoxPort* box = box_port(); + std::string engine = cand.engine; + std::string model = cand.model; + return cli_json_turn(label + " (box)", messages, tools, sink, timeout, + [&, box, engine, model](const std::string& p, double t) { + if (box == nullptr) { throw BackendError("opencode (box) run failed: no container engine"); } + return raw_out(box->box_opencode_run(engine, model, p, t)); + }, + [](const CliRaw& r) { return opencode_parse(r.out); }); + } + std::string model = cand.model; + return cli_json_turn(label, messages, tools, sink, timeout, + [&, model](const std::string& p, double t) { return raw_out(opencode_run(model, p, t)); }, + [](const CliRaw& r) { return opencode_parse(r.out); }); + } + if (sink == nullptr) { return single_shot(chat_url, cand.key, attempt, label, extra); } + return streaming(chat_url, cand.key, attempt, label, [sink](const std::string& piece) { (*sink)(piece); }, extra); +} + +std::optional ChatClient::try_candidate(const Candidate& cand, const json::Value& messages, + const json::Value& tools, const json::Value& payload, + const StreamSink* sink, bool ephemeral, + std::vector& errors) { + const std::string& label = cand.provider; + const std::string& model = cand.model; + json::Value attempt = payload; + attempt.set("model", model); + struct Try { json::Value tools; json::Value body; }; + std::vector tries; + tries.push_back({tools, attempt}); + if (attempt.get("tools").truthy()) { + json::Value stripped = json::Value::object(); + for (const auto& kv : attempt.members()) { + if (kv.first != "tools" && kv.first != "tool_choice") { stripped.set(kv.first, kv.second); } + } + tries.push_back({json::Value(), stripped}); + } + for (size_t pos = 0; pos < tries.size(); ++pos) { + double started = py::now(); + try { + ChatReply reply = call_candidate(cand, messages, tries[pos].tools, tries[pos].body, sink, ephemeral); + double elapsed = py::now() - started; + double tps = static_cast(std::max(1, static_cast(py::len(reply.content)) / 4)) / std::max(elapsed, 0.001); + health.record(label, model, true, elapsed * 1000, tps); + if (log_enabled()) { log_completion(cand, messages, reply, tries[pos].tools, elapsed * 1000, pos > 0); } + return reply; + } catch (const BackendError& exc) { + if (pos == 0 && exc.status() == 400 && tries.size() > 1) { continue; } + health.record(label, model, false); + bool transient = transient_failure(exc); + round_transient = round_transient || transient; + errors.push_back(label + "/" + model + ": " + exc.what()); + json::Value meta = json::Value::object(); + meta.set("backend", label); + meta.set("model", model); + meta.set("error", short_error(exc)); + meta.set("transient", transient); + log_event("WARNING", "llm", "failed", label + "/" + model + " failed: " + short_error(exc), scope.first, scope.second, meta); + return std::nullopt; + } catch (const std::exception& exc) { + health.record(label, model, false); + round_transient = true; + errors.push_back(label + "/" + model + ": " + short_error(exc)); + json::Value meta = json::Value::object(); + meta.set("backend", label); + meta.set("model", model); + meta.set("error", short_error(exc)); + meta.set("transient", true); + log_event("WARNING", "llm", "failed", label + "/" + model + " failed: " + short_error(exc), scope.first, scope.second, meta); + return std::nullopt; + } + } + return std::nullopt; +} + +void ChatClient::log_completion(const Candidate& cand, const json::Value& messages, const ChatReply& reply, + const json::Value& use_tools, double elapsed_ms, bool retried) { + auto text_of = [](const json::Value& item) -> std::string { + const json::Value& content = item.get("content"); + if (content.is_string()) { return content.str(); } + if (content.is_null()) { return ""; } + json::DumpOptions o; + o.sort_keys = true; + return content.dumps(o); + }; + std::string system_text, user_text; + for (const auto& item : messages.items()) { + if (item.get("role").is_string() && item.get("role").str() == "system") { system_text = text_of(item); break; } + } + for (auto it = messages.items().rbegin(); it != messages.items().rend(); ++it) { + if (it->get("role").is_string() && it->get("role").str() == "user") { user_text = text_of(*it); break; } + } + const std::string& assistant_text = reply.content; + json::Value tools_v = use_tools.truthy() ? use_tools : json::Value::array(); + int64_t tokens_in = conversation_tokens(messages, tools_v); + json::Value calls = json::Value::array(); + for (const auto& c : reply.tool_calls) { + json::Value o = json::Value::object(); + o.set("id", c.id); o.set("name", c.name); o.set("arguments", c.arguments); + calls.push(o); + } + int64_t tokens_out = estimate_tokens(assistant_text) + estimate_tokens(calls.dumps()); + std::vector names; + for (const auto& schema : tools_v.items()) { + if (schema.is_object()) { + json::Value fn = schema.get("function"); + names.push_back(py::str_or_empty(fn.get("name"))); // (schema.get("function") or {}).get("name", "") + } + } + int64_t mcp_count = 0, core_count = 0; + for (const auto& n : names) { + if (!native_tools().count(n)) { ++mcp_count; } + if (core_tools().count(n)) { ++core_count; } + } + int64_t lazy_count = static_cast(names.size()) - mcp_count - core_count; + std::vector sorted_names = names; + std::sort(sorted_names.begin(), sorted_names.end()); + json::Value meta = json::Value::object(); + meta.set("backend", cand.provider); + meta.set("model", cand.model); + meta.set("exec_ms", static_cast(elapsed_ms)); + meta.set("tokens_in", tokens_in); + meta.set("tokens_out", tokens_out); + meta.set("tokens_total", tokens_in + tokens_out); + meta.set("cost_usd", json::Value()); + meta.set("history_msgs", static_cast(messages.size())); + meta.set("system_chars", static_cast(py::len(system_text))); + meta.set("user_chars", static_cast(py::len(user_text))); + meta.set("assistant_chars", static_cast(py::len(assistant_text))); + meta.set("system_text", system_text); + meta.set("user_text", user_text); + meta.set("assistant_text", assistant_text); + meta.set("tool_calls", static_cast(reply.tool_calls.size())); + meta.set("retried_without_tools", retried); + meta.set("tools_sent", static_cast(names.size())); + meta.set("tools_mcp", mcp_count); + meta.set("tools_core", core_count); + meta.set("tools_lazy", lazy_count); + json::Value name_arr = json::Value::array(); + for (auto& n : sorted_names) { name_arr.push(n); } + meta.set("tool_names", name_arr); + log_event("INFO", "llm", "completion", + cand.provider + "/" + cand.model + " answered in " + std::to_string(static_cast(elapsed_ms)) + "ms (" + + std::to_string(tokens_in) + " tokens in, " + std::to_string(tokens_out) + " out)", + scope.first, scope.second, meta); +} + +ChatReply ChatClient::single_shot(const std::string& url, const std::string& key, const json::Value& payload, + const std::string& label, + const std::vector>& extra) { + HttpRequest req; + req.method = "POST"; + req.url = url; + req.headers = headers(key, extra); + req.body = payload.dumps(); + req.timeout = kHttpTimeout; + HttpResult res = transport().send(req); + if (res.transport_failed()) { throw BackendError(short_error_text(res.error)); } + if (!res.ok()) { throw BackendError(http_error_text(res), static_cast(res.status)); } + json::Value data; + try { + data = json::Value::loads(res.body); + } catch (const json::ParseError& exc) { + throw std::runtime_error(exc.what()); // ValueError: generic failure, treated as transient + } + std::string failure = embedded_failure_message(data); + if (!failure.empty()) { throw BackendError("upstream error: " + py::head(failure, 300)); } + const json::Value* message = nullptr; + if (data.is_object() && data.get("choices").is_array() && data.get("choices").size() > 0) { + const json::Value& first = data.get("choices").at(0); + if (first.is_object() && first.has("message")) { message = &first.get("message"); } + } + if (message == nullptr) { + throw BackendError("upstream returned no choices: " + py::head(data.dumps(), 300)); + } + if (!message->is_object()) { + throw std::runtime_error("'" + std::string(message->is_null() ? "NoneType" : "str") + "' object has no attribute 'get'"); + } + return normalize(*message, label); +} + +ChatReply ChatClient::normalize(const json::Value& message, const std::string& label) { + ChatReply reply; + const json::Value& calls = message.get("tool_calls"); + if (calls.truthy()) { + for (const auto& call : calls.items()) { + json::Value func = call.get("function").truthy() ? call.get("function") : json::Value::object(); + ToolCall tc; + tc.id = call.get("id").truthy() ? py::str(call.get("id")) : py::uuid4_hex(); + tc.name = tool_name_clean(func.get("name")); + tc.arguments = tool_args_text(func.get("arguments")); + reply.tool_calls.push_back(tc); + } + } + reply.content = py::str_or_empty(message.get("content")); + reply.reasoning = message.get("reasoning").truthy() ? py::str(message.get("reasoning")) : py::str_or_empty(message.get("reasoning_content")); + reply.backend = label; + return reply; +} + +ChatReply ChatClient::streaming(const std::string& url, const std::string& key, const json::Value& payload_in, + const std::string& label, const StreamSink& sink, + const std::vector>& extra) { + json::Value payload = payload_in; + payload.set("stream", true); + HttpRequest req; + req.method = "POST"; + req.url = url; + req.headers = headers(key, extra); + req.body = payload.dumps(); + req.timeout = kStreamTimeout; + std::vector content_parts, reasoning_parts; + struct Slot { std::string id, name, arguments; }; + std::map calls; + int64_t last_pos = 0; + bool done = false; + std::string failure; + auto on_line = [&](const std::string& raw) -> bool { + std::string line = py::strip(raw); + if (!py::startswith(line, "data:")) { return true; } + std::string data = py::strip(line.substr(5)); + if (data == "[DONE]") { done = true; return false; } + json::Value event; + try { + event = json::Value::loads(data); + } catch (const json::ParseError&) { + return true; + } + if (!event.is_object()) { + failure = "'" + std::string(event.is_array() ? "list" : "str") + "' object has no attribute 'get'"; + return false; + } + json::Value choices = event.get("choices").truthy() ? event.get("choices") : json::Value::array(); + if (choices.size() == 0) { return true; } + json::Value delta = choices.at(0).get("delta").truthy() ? choices.at(0).get("delta") : json::Value::object(); + const json::Value& piece = delta.get("content"); + if (piece.truthy()) { + std::string p = py::str(piece); + content_parts.push_back(p); + sink(p); + } + json::Value think = delta.get("reasoning").truthy() ? delta.get("reasoning") : delta.get("reasoning_content"); + if (think.truthy()) { reasoning_parts.push_back(py::str(think)); } + const json::Value& tcs = delta.get("tool_calls"); + if (tcs.truthy()) { + for (const auto& call : tcs.items()) { + json::Value func = call.get("function").truthy() ? call.get("function") : json::Value::object(); + int64_t pos = call.has("index") ? call.get("index").as_int() : 0; + if (!calls.count(pos) && !calls.empty() && !call.get("id").truthy() && !func.get("name").truthy()) { + pos = last_pos; + } + last_pos = pos; + Slot& slot = calls[pos]; + if (call.get("id").truthy()) { slot.id = py::str(call.get("id")); } + if (func.get("name").truthy()) { slot.name += py::str(func.get("name")); } + if (func.get("arguments").truthy()) { slot.arguments += tool_args_text(func.get("arguments")); } + } + } + return true; + }; + HttpResult res = transport().stream(req, on_line); + if (res.transport_failed()) { throw BackendError(short_error_text(res.error)); } + if (!res.ok()) { throw BackendError(http_error_text(res), static_cast(res.status)); } + if (!failure.empty()) { throw std::runtime_error(failure); } + if (!res.stream_error.empty() && content_parts.empty() && calls.empty()) { + throw BackendError(short_error_text(res.stream_error)); + } + ChatReply reply; + for (auto& kv : calls) { + ToolCall tc; + tc.name = tool_name_clean(kv.second.name); + tc.arguments = tool_args_valid(kv.second.arguments); + tc.id = kv.second.id.empty() ? py::uuid4_hex() : kv.second.id; + reply.tool_calls.push_back(tc); + } + reply.content = py::join(content_parts, ""); + reply.reasoning = py::join(reasoning_parts, ""); + reply.backend = label; + return reply; +} + +ChatReply ChatClient::muse_turn(const json::Value& messages, const json::Value& tools, const StreamSink* sink, + const std::string& label, double timeout, bool ephemeral) { + std::string binary = muse_binary(); + if (binary.empty()) { throw BackendError("muse CLI not found (MUSE_BIN or PATH)"); } + const std::string& home = config_.home; + if (ephemeral || home.empty()) { + MuseTempDir folder; + std::string schema_path = folder.path() + "/schema.json"; + std::string prompt_path = folder.path() + "/prompt.txt"; + muse_write_file(schema_path, muse_output_schema().dumps()); + muse_write_file(prompt_path, muse_prompt(messages, tools)); + json::Value answer = muse_run(muse_argv(config_.explicit_model, schema_path, prompt_path), folder.path(), sink, timeout); + return muse_reply(answer, label); + } + std::string key = muse_key(); + auto release = muse_acquire(home, "session:" + key); + auto gate = std::make_shared(); + gate->on_reap = [this, release](bool ok) { + try { + if (!ok) { reset_muse(); } + } catch (...) { + } + release(); + }; + try { + return muse_persistent(messages, tools, sink, label, timeout, home, key, gate); + } catch (...) { + if (!gate->armed.load()) { release(); } + throw; + } +} + +ChatReply ChatClient::muse_persistent(const json::Value& messages, const json::Value& tools, const StreamSink* sink, + const std::string& label, double timeout, const std::string& home, + const std::string& key, const std::shared_ptr& gate) { + std::string session_id; + std::vector pending; + muse_transact(home, [&](json::Value& data) { + const json::Value& entry = data.get(key); + bool have_pending = false; + std::vector split; + std::string sid; + if (entry.is_object() && entry.get("id").is_string() && entry.get("hashes").is_array()) { + auto s = muse_session_split(messages, entry.get("hashes")); + if (s.has_value() && !muse_session_alive(entry.get("id").str())) { s = std::nullopt; } + if (s.has_value()) { split = *s; sid = entry.get("id").str(); have_pending = true; } + } + if (!have_pending || split.empty()) { + sid = py::uuid4(); + split = messages.items(); + } + session_id = sid; + pending = split; + }); + json::Value answer; + try { + MuseTempDir folder; + std::string schema_path = folder.path() + "/schema.json"; + std::string prompt_path = folder.path() + "/prompt.txt"; + muse_write_file(schema_path, muse_output_schema().dumps()); + json::Value pending_v = json::Value::array(); + for (auto& p : pending) { pending_v.push(p); } + muse_write_file(prompt_path, muse_prompt(pending_v, tools)); + answer = muse_run(muse_argv(config_.explicit_model, schema_path, prompt_path, session_id), folder.path(), sink, timeout, gate); + } catch (const BackendError&) { + reset_muse(); + throw; + } + muse_transact(home, [&](json::Value& data) { + json::Value entry = json::Value::object(); + entry.set("id", session_id); + json::Value hashes = json::Value::array(); + for (const auto& m : messages.items()) { hashes.push(muse_message_hash(m)); } + entry.set("hashes", hashes); + entry.set("updated", py::now_iso()); + data.set(key, entry); + }); + return muse_reply(answer, label); +} + +ChatReply ChatClient::cli_json_turn(const std::string& label, const json::Value& messages, const json::Value& tools, + const StreamSink* sink, double timeout, + const std::function& run, + const std::function& parse) { + CliRaw raw = run(opencode_prompt(messages, tools), timeout); + json::Value answer = parse(raw); + std::string text = raw.text(); + if (tools.truthy() && opencode_find_answer(text).is_null()) { + try { + CliRaw again = run(cli_json_repair_prompt(messages, tools, text), timeout); + answer = parse(again); + } catch (const BackendError&) { + } + } + if (sink != nullptr && answer.get("content").truthy()) { (*sink)(py::str(answer.get("content"))); } + return muse_reply(answer, label); +} + +ChatReply ChatClient::muse_reply(const json::Value& answer, const std::string& label) { + ChatReply reply; + const json::Value& calls = answer.get("tool_calls"); + if (calls.truthy()) { + for (const auto& call : calls.items()) { + ToolCall tc; + tc.id = py::uuid4_hex(); + tc.name = tool_name_clean(call.get("name")); + tc.arguments = tool_args_text(call.get("arguments")); + reply.tool_calls.push_back(tc); + } + } + reply.content = py::str_or_empty(answer.get("content")); + reply.backend = label; + return reply; +} + +std::string models_report(ChatClient& chat) { + std::vector lines = {"label/model | weight ok/fail avg-ms avg-tps status"}; + std::set> seen; + for (const auto& cand : chat.ordered_candidates()) { + auto key = std::make_pair(cand.provider, cand.model); + if (seen.count(key)) { continue; } + seen.insert(key); + json::Value entry = chat.health.entry(cand.provider, cand.model); + auto n = [&](const char* k) { return entry.get(k).truthy() ? entry.get(k).as_double() : 0.0; }; + int64_t ok = static_cast(n("s")); + int64_t fail = static_cast(n("f")); + std::string avg_ms = "-", avg_tps = "-"; + if (ok && n("lat") != 0.0) { avg_ms = std::to_string(static_cast(std::nearbyint(n("lat") / static_cast(ok)))); } + if (ok && n("tps") != 0.0) { + char buf[32]; + std::snprintf(buf, sizeof buf, "%.1f", std::nearbyint(n("tps") / static_cast(ok) * 10.0) / 10.0); + avg_tps = buf; + } + std::string status; + if (cand.chat_url == kMuseChatUrl) { status = muse_health(); } + else if (chat.health.circuit_open(cand.provider, cand.model)) { status = "circuit open"; } + else if (!ok && !fail) { status = "untested"; } + else { status = "ready"; } + char wbuf[32]; + std::snprintf(wbuf, sizeof wbuf, "%.4f", model_weight(entry)); + lines.push_back(cand.provider + "/" + cand.model + " | " + wbuf + " " + std::to_string(ok) + "/" + std::to_string(fail) + " " + + avg_ms + " " + avg_tps + " " + status); + } + return py::join(lines, "\n"); +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/chat_client.h b/taicpp/src/llm/chat_client.h new file mode 100644 index 0000000..a94eafe --- /dev/null +++ b/taicpp/src/llm/chat_client.h @@ -0,0 +1,130 @@ +#pragma once +// ChatClient (tai.py 7996-8648): candidate ordering, retries with backoff, +// OpenAI-compatible single-shot and streaming calls, CLI backends, health +// bookkeeping, model discovery (update_models) and the models report. +#include "cli_backends.h" +#include "helpers.h" +#include "json.h" +#include "model_health.h" +#include "models.h" +#include "muse.h" +#include "ports.h" +#include +#include +#include +#include +#include +namespace detz::llm { + +struct ToolCall { std::string id, name, arguments; }; + +// The dict ChatClient.complete returns. +struct ChatReply { + std::string role{"assistant"}; + std::string content; + std::string reasoning; + std::vector tool_calls; + std::string backend; +}; + +struct UpdateRow { + std::string label; + std::optional count; // nullopt: source unreachable, list kept + std::vector added, removed; +}; + +// What a prompt-JSON CLI run produced: stdout, and (codex) the -o file text. +struct CliRaw { + std::string out, outfile; + bool used_schema{false}; + std::string text() const; // raw_text(): the outfile when non-blank, else stdout +}; + +class ChatClient { + public: + explicit ChatClient(LlmConfig& config); + + // Public state mirrored from the Python attributes. + std::pair scope{"default", "main"}; + std::optional fixed_key; + std::optional provider_pin; + std::optional model_pin; + bool round_transient{false}; + bool pin_fallback_said{false}; + std::optional> prefer; // (DEVPLACE_LABEL, PREFERRED_MODEL) + ModelHealth health; + + LlmConfig& config() { return config_; } + + std::string muse_key() const; + void reset_muse(); + + std::vector openrouter_models(); + std::vector update_models(); + std::vector opencode_models(); + std::vector zen_models(); + std::vector pollinations_models(); + std::vector ollama_models(); + std::vector devplace_models(); + + std::vector candidates(); + std::vector ordered_candidates(); + std::vector all_candidates(); + std::string pin_note() const; + // (provider, chat_url, models_url, key, model), first per (provider, chat_url). + struct BackendRow { std::string provider, chat_url, models_url, key, model; }; + std::vector backends(); + + static std::string summarize_errors(const std::vector& errors, size_t shown = 2, size_t width = 320); + + // messages: array of message objects; tools: array of tool schemas or null. + ChatReply complete(const json::Value& messages, const json::Value& tools = json::Value(), + const StreamSink* stream_sink = nullptr, bool ephemeral = false, + std::optional deadline = std::nullopt); + + // Process-wide hooks. + // tai.py reads TOOL_SCHEMAS / CORE_TOOLS in log_completion; the agent module + // registers them here (names only) so src/llm stays independent of src/agent. + static void set_tool_catalog(std::set native, std::set core); + // time.sleep seam so tests do not wait out RETRY_BACKOFF. + static std::function& sleeper(); + + private: + ChatReply complete_inner(const json::Value& messages, const json::Value& tools, const StreamSink* sink, + bool ephemeral, std::optional deadline); + std::optional try_candidate(const Candidate& cand, const json::Value& messages, const json::Value& tools, + const json::Value& payload, const StreamSink* sink, bool ephemeral, + std::vector& errors); + ChatReply call_candidate(const Candidate& cand, const json::Value& messages, const json::Value& tools, + const json::Value& attempt, const StreamSink* sink, bool ephemeral); + std::vector apply_pins(std::vector cands); + std::vector box_opencode_candidates(); + std::vector box_opencode_models(const std::string& engine); + void log_completion(const Candidate& cand, const json::Value& messages, const ChatReply& reply, + const json::Value& use_tools, double elapsed_ms, bool retried); + std::vector> headers(const std::string& key, + const std::vector>& extra) const; + ChatReply single_shot(const std::string& url, const std::string& key, const json::Value& payload, + const std::string& label, const std::vector>& extra); + ChatReply streaming(const std::string& url, const std::string& key, const json::Value& payload, + const std::string& label, const StreamSink& sink, + const std::vector>& extra); + ChatReply normalize(const json::Value& message, const std::string& label); + ChatReply muse_turn(const json::Value& messages, const json::Value& tools, const StreamSink* sink, + const std::string& label, double timeout, bool ephemeral); + ChatReply muse_persistent(const json::Value& messages, const json::Value& tools, const StreamSink* sink, + const std::string& label, double timeout, const std::string& home, const std::string& key, + const std::shared_ptr& gate); + ChatReply cli_json_turn(const std::string& label, const json::Value& messages, const json::Value& tools, + const StreamSink* sink, double timeout, + const std::function& run, + const std::function& parse); + ChatReply muse_reply(const json::Value& answer, const std::string& label); + + LlmConfig& config_; +}; + +// models_report(chat): table text of the `/models` command. +std::string models_report(ChatClient& chat); + +} // namespace detz::llm diff --git a/taicpp/src/llm/cli_backends.cpp b/taicpp/src/llm/cli_backends.cpp new file mode 100644 index 0000000..97c5ab4 --- /dev/null +++ b/taicpp/src/llm/cli_backends.cpp @@ -0,0 +1,690 @@ +#include "cli_backends.h" +#include "consts.h" +#include "http_transport.h" +#include "models.h" +#include "py.h" +#include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::llm { + +namespace { +// TemporaryDirectory(prefix=...) +class TempDir { + public: + explicit TempDir(const std::string& prefix) { + std::string tmpl = (fs::temp_directory_path() / (prefix + "XXXXXX")).string(); + std::vector buf(tmpl.begin(), tmpl.end()); + buf.push_back('\0'); + if (mkdtemp(buf.data()) != nullptr) { path_ = buf.data(); } + } + ~TempDir() { + if (!path_.empty()) { + std::error_code ec; + fs::remove_all(path_, ec); + } + } + TempDir(const TempDir&) = delete; + TempDir& operator=(const TempDir&) = delete; + const std::string& path() const { return path_; } + private: + std::string path_; +}; + +std::string env_or_which(const char* env, const char* a, const char* b = nullptr) { + std::string e = env_get(env); + if (!e.empty()) { return e; } + std::string w = which(a); + if (w.empty() && b != nullptr) { w = which(b); } + return w; +} + +std::string universal_newlines(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == '\r') { + out += '\n'; + if (i + 1 < s.size() && s[i + 1] == '\n') { ++i; } + } else { + out += s[i]; + } + } + return out; +} + +bool write_text(const std::string& path, const std::string& text) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) { return false; } + out << text; + return static_cast(out); +} + +std::string read_text(const std::string& path, bool& ok) { + std::ifstream in(path, std::ios::binary); + ok = static_cast(in); + if (!in) { return ""; } + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +bool contains_ci(const std::string& text, const std::regex& re) { return std::regex_search(text, re); } +} // namespace + +std::string muse_binary() { return env_or_which("MUSE_BIN", "muse"); } +std::string opencode_binary() { return env_or_which("OPENCODE_BIN", "opencode"); } +std::string grok_binary() { return env_or_which("GROK_BIN", "grok", "grokcli"); } +std::string claude_binary() { return env_or_which("CLAUDE_BIN", "claude"); } +std::string codex_binary() { return env_or_which("CODEX_BIN", "codex"); } +std::string gemini_binary() { return env_or_which("GEMINI_BIN", "gemini"); } + +json::Value muse_output_schema() { + json::Value call = json::Value::loads( + R"({"type": "object", "additionalProperties": false, "properties": {"name": {"type": "string"}, "arguments": {"type": "string"}}, "required": ["name", "arguments"]})"); + json::Value schema = json::Value::loads( + R"({"type": "object", "additionalProperties": false, "properties": {"content": {"type": "string"}, "tool_calls": {"type": "array", "items": null}}, "required": ["content", "tool_calls"]})"); + schema["properties"]["tool_calls"].set("items", call); + return schema; +} + +std::string muse_prompt(const json::Value& messages, const json::Value& tools) { + std::vector parts; + if (messages.truthy()) { + for (const auto& message : messages.items()) { + std::string role = py::upper(message.get("role").truthy() ? py::str(message.get("role")) : std::string("user")); + json::Value bodyv = message.get("content").truthy() ? message.get("content") : json::Value(""); + std::string body; + if (bodyv.is_array()) { + std::vector texts; + for (const auto& part : bodyv.items()) { + if (part.is_object()) { texts.push_back(py::str_or_empty(part.get("text"))); } + } + body = py::join(texts, " "); + } else { + body = py::str(bodyv); + } + if (message.get("tool_call_id").truthy()) { + parts.push_back(role + " " + py::str(message.get("tool_call_id")) + ":\n" + body); + } else { + parts.push_back(role + ":\n" + body); + } + if (message.get("tool_calls").truthy()) { + for (const auto& call : message.get("tool_calls").items()) { + json::Value func = call.get("function").truthy() ? call.get("function") : json::Value::object(); + parts.push_back("TOOL CALL " + py::str_or_empty(call.get("id")) + ":\n" + py::str_or_empty(func.get("name")) + + " " + py::str_or_empty(func.get("arguments"))); + } + } + } + } + if (tools.truthy()) { parts.push_back("TOOLS:\n" + tools.dumps()); } + parts.push_back("Answer with content text plus tool_calls for every tool to invoke now. Empty tool_calls ends the turn."); + parts.push_back("Reply with data only: never invoke assistant or harness tools yourself. The TOOLS block above is response vocabulary, not executable tools."); + std::vector kept; + for (auto& p : parts) { if (!py::strip(p).empty()) { kept.push_back(p); } } + return py::join(kept, "\n\n"); +} + +std::string opencode_prompt(const json::Value& messages, const json::Value& tools) { + std::string schema = muse_output_schema().dumps(); + std::string rules = + "You are the model behind another agent. The transcript above is the whole conversation; lines starting with TOOL are results of tool calls that already ran. " + "Do not use any of your own tools and do not repeat a tool call whose result is already shown. " + "If the results answer the user, put the final answer in content with an empty tool_calls list; otherwise request the next tool calls from TOOLS. " + "An empty tool_calls list ends the turn at once, nothing else runs. " + "Never narrate a planned action without its tool call in the same reply. " + "Reply with one JSON object only, no code fence, matching this schema: "; + return muse_prompt(messages, tools) + "\n\n" + rules + schema; +} + +json::Value opencode_find_answer(const std::string& text) { + std::vector starts; + for (size_t i = 0; i < text.size(); ++i) { if (text[i] == '{') { starts.push_back(i); } } + size_t from = starts.size() > static_cast(kOpencodeParseMaxScans) ? starts.size() - static_cast(kOpencodeParseMaxScans) : 0; + for (size_t k = starts.size(); k > from; --k) { + size_t pos = starts[k - 1]; + try { + size_t end = 0; + json::Value answer = json::Value::raw_decode(text, pos, end); + if (answer.is_object() && (answer.has("content") || answer.has("tool_calls"))) { return answer; } + } catch (const json::ParseError&) { + continue; + } + } + return json::Value(); +} + +json::Value opencode_parse(const std::string& raw) { + std::string text = py::strip(raw); + // ^```(?:json)?\s*(.*?)\s*```$ with re.S + if (text.size() >= 6 && text.compare(0, 3, "```") == 0 && text.compare(text.size() - 3, 3, "```") == 0) { + size_t p = 3; + if (text.compare(p, 4, "json") == 0 && p + 4 <= text.size() - 3) { p += 4; } + std::string inner = text.substr(p, text.size() - 3 - p); + text = py::strip(inner); + } + json::Value found = opencode_find_answer(text); + if (found.is_object()) { return found; } + json::Value out = json::Value::object(); + out.set("content", text); + out.set("tool_calls", json::Value::array()); + return out; +} + +std::string cli_json_repair_prompt_impl(const json::Value& messages, const json::Value& tools, const std::string& bad_text) { + return opencode_prompt(messages, tools) + + "\n\nYour previous reply contained no JSON object, so no tool ran and the turn is about to end. " + "Reply now with exactly one JSON object matching the schema above and no other text. " + "Previous reply for reference:\n" + py::head(py::strip(bad_text), 2000); +} +std::string cli_json_repair_prompt(const json::Value& messages, const json::Value& tools, const std::string& bad_text) { + return cli_json_repair_prompt_impl(messages, tools, bad_text); +} + +std::string opencode_parse_events(const std::string& stdout_text, int returncode, const std::string& stderr_text) { + std::vector texts; + for (const auto& line : py::splitlines(stdout_text)) { + json::Value event; + try { + event = json::Value::loads(line); + } catch (const json::ParseError&) { + continue; + } + if (!event.is_object()) { continue; } + const json::Value& type = event.get("type"); + if (type.is_string() && type.str() == "error") { + json::Value err = event.get("error").truthy() ? event.get("error") : json::Value::object(); + json::Value data = err.get("data").truthy() ? err.get("data") : json::Value::object(); + std::string message = data.get("message").truthy() ? py::str(data.get("message")) : py::head(event.get("error").dumps(), 300); + int status = 0; + if (data.get("statusCode").truthy()) { status = static_cast(data.get("statusCode").as_int()); } + throw BackendError("opencode: " + message, status); + } + json::Value part = event.get("part").truthy() ? event.get("part") : json::Value::object(); + if (type.is_string() && type.str() == "text" && part.get("text").truthy()) { + texts.push_back(py::str(part.get("text"))); + } + } + if (texts.empty()) { + std::string src = !stderr_text.empty() ? stderr_text : stdout_text; + throw BackendError("opencode returned no text (exit " + std::to_string(returncode) + "): " + py::tail(src, 300)); + } + return py::join(texts, ""); +} + +Opencode::Hook& Opencode::hook() { + static Hook h; + return h; +} +std::string Opencode::run(const std::string& model, const std::string& prompt, double timeout) { + if (hook()) { return hook()(model, prompt, timeout); } + std::string binary = opencode_binary(); + if (binary.empty()) { throw BackendError("opencode CLI not found (OPENCODE_BIN or PATH)"); } + std::vector argv = {binary, "run", "--pure", "--agent", kOpencodeAgent, "--format", "json", "-m", "opencode/" + model}; + TempDir folder("tai-opencode-"); + ProcResult done = run_process(argv, prompt, timeout, folder.path()); + if (done.timed_out) { throw BackendError("opencode run timed out after " + std::to_string(static_cast(timeout)) + "s"); } + if (done.spawn_failed) { throw BackendError("opencode run failed: " + short_error_text(done.spawn_error)); } + return opencode_parse_events(universal_newlines(done.out), done.returncode, universal_newlines(done.err)); +} +std::string opencode_run(const std::string& model, const std::string& prompt, double timeout) { + return Opencode::run(model, prompt, timeout); +} + +bool opencode_free_models(std::vector& out) { + std::string binary = opencode_binary(); + if (binary.empty()) { return false; } + ProcResult done = run_process({binary, "models", "opencode"}, std::nullopt, kModelListTimeout * 4); + if (done.spawn_failed || done.timed_out) { return false; } + std::vector models; + for (const auto& line : py::splitlines(universal_newlines(done.out))) { + std::string s = py::strip(line); + if (py::startswith(s, "opencode/")) { models.push_back(s.substr(s.find('/') + 1)); } + } + out.clear(); + for (auto& m : models) { if (is_free_model(m)) { out.push_back(m); } } + return !out.empty(); +} + +bool cli_unknown_flag(const std::string& stderr_text) { + static const std::regex re("unknown (option|flag|command)|unrecognised|unrecognized|invalid flag|not found|no such option", std::regex::icase); + return contains_ci(stderr_text, re); +} +bool cli_auth_failure(const std::string& text) { + static const std::regex re("not (logged in|authenticated)|auth|api key|unauthorized|401|login required|please (login|log in|authenticate)", std::regex::icase); + return contains_ci(text, re); +} +ProcResult cli_run(const std::vector& argv, const std::optional& prompt_input, + double timeout, const std::string& label) { + ProcResult done = run_process(argv, prompt_input, timeout, "", !prompt_input.has_value()); + if (done.timed_out) { throw BackendError(label + " run timed out after " + std::to_string(static_cast(timeout)) + "s"); } + if (done.spawn_failed) { throw BackendError(label + " run failed: " + short_error_text(done.spawn_error)); } + done.out = universal_newlines(done.out); + done.err = universal_newlines(done.err); + return done; +} + +// ---------------------------------------------------------------- grok +json::Value grok_parse_output(const std::string& stdout_text) { + json::Value data; + try { + data = json::Value::loads(stdout_text); + } catch (const json::ParseError&) { + return opencode_parse(stdout_text); + } + if (data.is_object()) { + if (data.get("is_error").truthy() || data.get("error").truthy()) { + json::Value detail = data.get("error").truthy() ? data.get("error") + : data.get("result").truthy() ? data.get("result") + : data.get("text").truthy() ? data.get("text") : json::Value("unknown error"); + throw BackendError("grok: " + py::head(py::str(detail), 300)); + } + if (data.get("content").is_string() || data.get("tool_calls").is_array()) { + if (data.has("content") || data.has("tool_calls")) { return data; } + } + for (const char* key : {"text", "result", "output", "answer", "content"}) { + const json::Value& value = data.get(key); + if (value.is_string() && !py::strip(value.str()).empty()) { return opencode_parse(value.str()); } + } + } + return opencode_parse(stdout_text); +} + +std::string grok_run(const std::string& prompt, double timeout) { + std::string binary = grok_binary(); + if (binary.empty()) { throw BackendError("grok CLI not found (GROK_BIN or PATH)"); } + TempDir folder("tai-grok-"); + std::string prompt_path = folder.path() + "/prompt.txt"; + write_text(prompt_path, prompt); + std::vector> variants = { + {binary, "--prompt-file", prompt_path, "--output-format", "json", "--no-auto-update"}, + {binary, "--prompt-file", prompt_path, "--output-format", "json"}, + {binary, "-p", prompt, "--output-format", "json"}, + {binary, "-p", prompt}, + }; + std::string last_error; + for (size_t i = 0; i < variants.size(); ++i) { + ProcResult done = cli_run(variants[i], std::nullopt, timeout, "grok"); + if (done.returncode != 0 && cli_unknown_flag(done.err) && i + 1 < variants.size()) { + last_error = py::head(py::strip(done.err), 200); + continue; + } + if (done.returncode != 0 && py::strip(done.out).empty()) { + std::string blob = py::strip(done.err + "\n" + done.out); + if (cli_auth_failure(blob)) { + throw BackendError("grok not authenticated (set XAI_API_KEY or run grok login): " + py::head(blob, 200)); + } + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "no output"); + throw BackendError("grok exit " + std::to_string(done.returncode) + ": " + detail); + } + if (py::strip(done.out).empty()) { + std::string blob = py::strip(done.err); + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "empty stdout"); + throw BackendError("grok returned no output: " + detail); + } + return done.out; + } + throw BackendError("grok failed: " + (!last_error.empty() ? last_error : std::string("no usable output"))); +} + +// ---------------------------------------------------------------- claude +json::Value claude_parse_output(const std::string& stdout_text) { + json::Value data; + try { + data = json::Value::loads(stdout_text); + } catch (const json::ParseError&) { + return opencode_parse(stdout_text); + } + if (!data.is_object()) { return opencode_parse(stdout_text); } + if (data.get("is_error").truthy()) { + json::Value r = data.get("result").truthy() ? data.get("result") : json::Value("unknown error"); + throw BackendError("claude: " + py::head(py::str(r), 300)); + } + const json::Value& structured = data.get("structured_output"); + if (structured.is_object() && (structured.has("content") || structured.has("tool_calls"))) { return structured; } + const json::Value& result = data.get("result"); + if (result.is_string() && !py::strip(result.str()).empty()) { return opencode_parse(result.str()); } + return opencode_parse(stdout_text); +} + +std::string claude_run(const std::string& prompt, double timeout) { + std::string binary = claude_binary(); + if (binary.empty()) { throw BackendError("claude CLI not found (CLAUDE_BIN or PATH)"); } + std::string schema = muse_output_schema().dumps(); + struct Variant { std::vector argv; std::optional input; }; + std::vector variants; + if (prompt.size() > 120000) { + variants = {{{binary, "-p", "--output-format", "json", "--json-schema", schema}, prompt}, + {{binary, "-p", "--output-format", "json"}, prompt}}; + } else { + variants = {{{binary, "-p", prompt, "--output-format", "json", "--json-schema", schema}, std::nullopt}, + {{binary, "-p", prompt, "--output-format", "json"}, std::nullopt}, + {{binary, "-p", prompt}, std::nullopt}}; + } + std::string last_error; + for (size_t i = 0; i < variants.size(); ++i) { + ProcResult done = cli_run(variants[i].argv, variants[i].input, timeout, "claude"); + if (done.returncode != 0 && cli_unknown_flag(done.err) && i + 1 < variants.size()) { + last_error = py::head(py::strip(done.err), 200); + continue; + } + if (done.returncode != 0 && py::strip(done.out).empty()) { + std::string blob = py::strip(done.err + "\n" + done.out); + if (cli_auth_failure(blob)) { throw BackendError("claude not authenticated (run claude login): " + py::head(blob, 200)); } + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "no output"); + throw BackendError("claude exit " + std::to_string(done.returncode) + ": " + detail); + } + if (py::strip(done.out).empty()) { + std::string blob = py::strip(done.err); + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "empty stdout"); + throw BackendError("claude returned no output: " + detail); + } + return done.out; + } + throw BackendError("claude failed: " + (!last_error.empty() ? last_error : std::string("no usable output"))); +} + +// ---------------------------------------------------------------- codex +std::pair, std::string> codex_jsonl_texts(const std::string& stdout_text) { + std::vector texts; + std::string last_error; + for (const auto& raw : py::splitlines(stdout_text)) { + std::string line = py::strip(raw); + if (!py::startswith(line, "{")) { continue; } + json::Value event; + try { + event = json::Value::loads(line); + } catch (const json::ParseError&) { + continue; + } + if (!event.is_object()) { continue; } + const json::Value& etype = event.get("type"); + std::string t = etype.is_string() ? etype.str() : ""; + if (t == "item.completed") { + json::Value item = event.get("item").truthy() ? event.get("item") : json::Value::object(); + if (item.get("type").is_string() && item.get("type").str() == "agent_message" && item.get("text").truthy()) { + texts.push_back(py::str(item.get("text"))); + } + } else if (t == "turn.failed" || t == "error") { + json::Value msg = event.get("message").truthy() ? event.get("message") : (event.get("error").truthy() ? event.get("error") : json::Value("")); + if (msg.is_object()) { + msg = msg.get("message").truthy() ? msg.get("message") : json::Value(msg.dumps()); + } + if (msg.truthy()) { last_error = py::head(py::str(msg), 300); } + } + } + return {texts, last_error}; +} + +json::Value codex_parse_output(const std::string& stdout_text, const std::string& outfile_text, bool used_schema) { + if (!py::strip(outfile_text).empty()) { + if (used_schema) { + try { + json::Value answer = json::Value::loads(outfile_text); + if (answer.is_object() && (answer.has("content") || answer.has("tool_calls"))) { return answer; } + } catch (const json::ParseError&) { + } + } + return opencode_parse(outfile_text); + } + auto [texts, last_error] = codex_jsonl_texts(stdout_text); + if (!texts.empty()) { return opencode_parse(py::join(texts, "")); } + if (!last_error.empty()) { throw BackendError("codex: " + last_error); } + return opencode_parse(stdout_text); +} + +std::vector codex_sandbox_args() { + std::string v = py::lower(py::strip(env_get("TAI_CODEX_SANDBOX"))); + for (const char* w : {"1", "true", "yes", "on", "read-only", "readonly"}) { + if (v == w) { return {"--sandbox", "read-only"}; } + } + return {}; +} + +CodexOut codex_run(const std::string& prompt, double timeout) { + std::string binary = codex_binary(); + if (binary.empty()) { throw BackendError("codex CLI not found (CODEX_BIN or PATH)"); } + std::string schema = muse_output_schema().dumps(); + TempDir folder("tai-codex-"); + std::string schema_path = folder.path() + "/schema.json"; + std::string out_path = folder.path() + "/last.txt"; + write_text(schema_path, schema); + std::vector base = {binary, "exec", "--json", "--ephemeral"}; + for (auto& a : codex_sandbox_args()) { base.push_back(a); } + base.push_back("--skip-git-repo-check"); + struct Variant { std::vector argv; std::optional input; bool used_schema; }; + auto with = [&](std::vector extra) { + std::vector a = base; + for (auto& e : extra) { a.push_back(e); } + return a; + }; + std::vector variants; + if (prompt.size() > 120000) { + variants = {{with({"--output-schema", schema_path, "-o", out_path, "-"}), prompt, true}, + {with({"-o", out_path, "-"}), prompt, false}}; + } else { + variants = {{with({"--output-schema", schema_path, "-o", out_path, prompt}), std::nullopt, true}, + {with({"-o", out_path, prompt}), std::nullopt, false}, + {with({prompt}), std::nullopt, false}}; + } + std::string last_error; + for (size_t i = 0; i < variants.size(); ++i) { + const Variant& v = variants[i]; + ProcResult done = run_process(v.argv, v.input, timeout, folder.path()); + if (done.timed_out) { throw BackendError("codex run timed out after " + std::to_string(static_cast(timeout)) + "s"); } + if (done.spawn_failed) { throw BackendError("codex run failed: " + short_error_text(done.spawn_error)); } + done.out = universal_newlines(done.out); + done.err = universal_newlines(done.err); + if (done.returncode != 0 && cli_unknown_flag(done.err) && i + 1 < variants.size()) { + last_error = py::head(py::strip(done.err), 200); + continue; + } + std::string outfile_text; + if (std::find(v.argv.begin(), v.argv.end(), "-o") != v.argv.end()) { + bool ok = false; + outfile_text = read_text(out_path, ok); + if (!ok) { outfile_text.clear(); } + } + if (done.returncode != 0 && !(!py::strip(outfile_text).empty() || !py::strip(done.out).empty())) { + std::string blob = py::strip(done.err + "\n" + done.out); + if (cli_auth_failure(blob)) { throw BackendError("codex not authenticated (run codex login): " + py::head(blob, 200)); } + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "no output"); + throw BackendError("codex exit " + std::to_string(done.returncode) + ": " + detail); + } + if (!(!py::strip(outfile_text).empty() || !py::strip(done.out).empty())) { + std::string blob = py::strip(done.err); + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "empty stdout"); + throw BackendError("codex returned no output: " + detail); + } + return {done.out, outfile_text, v.used_schema}; + } + throw BackendError("codex failed: " + (!last_error.empty() ? last_error : std::string("no usable output"))); +} + +// ---------------------------------------------------------------- gemini +json::Value gemini_parse_output(const std::string& stdout_text) { + json::Value data; + try { + data = json::Value::loads(stdout_text); + } catch (const json::ParseError&) { + return opencode_parse(stdout_text); + } + if (!data.is_object()) { return opencode_parse(stdout_text); } + const json::Value& err = data.get("error"); + if (err.truthy()) { + std::string text = err.is_object() ? err.dumps() : py::str(err); + throw BackendError("gemini: " + py::head(text, 300)); + } + const json::Value& response = data.get("response"); + if (response.is_string() && !py::strip(response.str()).empty()) { return opencode_parse(response.str()); } + return opencode_parse(stdout_text); +} + +bool gemini_plan_enabled() { + std::string v = py::lower(py::strip(env_get("TAI_GEMINI_PLAN"))); + for (const char* w : {"1", "true", "yes", "on", "plan"}) { if (v == w) { return true; } } + return false; +} + +std::string gemini_run(const std::string& prompt, double timeout) { + std::string binary = gemini_binary(); + if (binary.empty()) { throw BackendError("gemini CLI not found (GEMINI_BIN or PATH)"); } + std::vector base = {binary, "-p", prompt}; + std::vector> variants; + if (gemini_plan_enabled()) { + auto v = base; v.insert(v.end(), {"--output-format", "json", "--approval-mode", "plan"}); + variants.push_back(v); + } + { + auto v = base; v.insert(v.end(), {"--output-format", "json"}); + variants.push_back(v); + } + variants.push_back(base); + std::string last_error; + for (size_t i = 0; i < variants.size(); ++i) { + ProcResult done = cli_run(variants[i], std::nullopt, timeout, "gemini"); + if (done.returncode != 0 && cli_unknown_flag(done.err) && i + 1 < variants.size()) { + last_error = py::head(py::strip(done.err), 200); + continue; + } + if (done.returncode != 0 && py::strip(done.out).empty()) { + std::string blob = py::strip(done.err + "\n" + done.out); + if (cli_auth_failure(blob)) { + throw BackendError("gemini not authenticated (run gemini login or set GEMINI_API_KEY): " + py::head(blob, 200)); + } + std::string hint; + if (done.returncode == 42) { hint = "input error"; } + else if (done.returncode == 53) { hint = "turn limit exceeded"; } + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "no output"); + throw BackendError("gemini exit " + std::to_string(done.returncode) + (hint.empty() ? "" : " (" + hint + ")") + ": " + detail); + } + if (py::strip(done.out).empty()) { + std::string blob = py::strip(done.err); + std::string detail = !py::head(blob, 300).empty() ? py::head(blob, 300) : (!last_error.empty() ? last_error : "empty stdout"); + throw BackendError("gemini returned no output: " + detail); + } + return done.out; + } + throw BackendError("gemini failed: " + (!last_error.empty() ? last_error : std::string("no usable output"))); +} + +// ---------------------------------------------------------------- health +std::string cli_version_detail(const ProcResult& done, const std::string& fallback) { + std::string o = py::strip(done.out), e = py::strip(done.err); + if (o.empty() && e.empty()) { return fallback; } + std::string chosen = !o.empty() ? o : e; + auto lines = py::splitlines(chosen); + return lines.empty() ? std::string() : py::head(lines[0], 60); +} +std::string cli_version_only_health(const std::string& binary, const std::string& fallback_name) { + ProcResult done = run_process({binary, "--version"}, std::nullopt, 15); + if (done.spawn_failed) { return "fail: " + py::head(short_error_text(done.spawn_error), 60); } + if (done.timed_out) { return "fail: " + py::head(std::string("Command '['") + binary + "', '--version']' timed out after 15 seconds", 60); } + done.out = universal_newlines(done.out); + done.err = universal_newlines(done.err); + if (done.returncode != 0) { return "fail: exit " + std::to_string(done.returncode); } + return "cli found (" + cli_version_detail(done, fallback_name) + "), login unverified: no status command"; +} +std::string muse_health() { + std::string binary = muse_binary(); + if (binary.empty()) { return "missing: muse CLI not found"; } + return cli_version_only_health(binary, "muse"); +} +std::string grok_health() { + std::string binary = grok_binary(); + if (binary.empty()) { return "missing: grok CLI not found"; } + return cli_version_only_health(binary, "grok"); +} +std::string claude_health() { + std::string binary = claude_binary(); + if (binary.empty()) { return "missing: claude CLI not found"; } + ProcResult done = run_process({binary, "auth", "status"}, std::nullopt, 15); + if (done.spawn_failed) { return "fail: " + py::head(short_error_text(done.spawn_error), 60); } + if (done.timed_out) { return "fail: Command timed out after 15 seconds"; } + json::Value info; + try { + info = json::Value::loads(universal_newlines(done.out)); + } catch (const json::ParseError&) { + info = json::Value(); + } + if (info.is_object() && info.has("loggedIn")) { + if (info.get("loggedIn").truthy()) { + std::string method = info.get("authMethod").truthy() ? py::str(info.get("authMethod")) : "logged in"; + return "ok, subscription (" + method + ")"; + } + return "not logged in"; + } + return cli_version_only_health(binary, "claude"); +} +std::string codex_health() { + std::string binary = codex_binary(); + if (binary.empty()) { return "missing: codex CLI not found"; } + ProcResult done = run_process({binary, "login", "status"}, std::nullopt, 15); + if (done.spawn_failed) { return "fail: " + short_error_text(done.spawn_error); } + if (done.timed_out) { return "fail: Command timed out after 15 seconds"; } + done.out = universal_newlines(done.out); + done.err = universal_newlines(done.err); + std::string detail = cli_version_detail(done, "codex"); + if (done.returncode != 0) { return "not logged in: " + detail; } + return "ok, subscription (" + detail + ")"; +} +std::string gemini_health() { + std::string binary = gemini_binary(); + if (binary.empty()) { return "missing: gemini CLI not found"; } + return cli_version_only_health(binary, "gemini"); +} + +std::vector> opencode_headers() { + static const std::string alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + static std::mt19937 rng{std::random_device{}()}; + auto rand_id = [&]() { + std::string s; + for (int i = 0; i < 26; ++i) { s += alphabet[rng() % alphabet.size()]; } + return s; + }; + static const std::string session = "ses_" + rand_id(); + std::string ua = env_get("OPENCODE_CLIENT_USER_AGENT"); + std::string name = env_get("OPENCODE_CLIENT_NAME"); + return {{"User-Agent", ua.empty() ? std::string(kOpencodeUserAgent) : ua}, + {"x-opencode-client", name.empty() ? std::string(kOpencodeClientName) : name}, + {"x-opencode-project", "global"}, + {"x-opencode-session", session}, + {"x-opencode-request", "msg_" + rand_id()}}; +} + +std::string probe_backend(const std::string& models_url, const std::string& key) { + if (models_url == kMuseModelsUrl) { return muse_health(); } + if (models_url == kGrokModelsUrl) { return grok_health(); } + if (models_url == kClaudeModelsUrl) { return claude_health(); } + if (models_url == kCodexModelsUrl) { return codex_health(); } + if (models_url == kGeminiModelsUrl) { return gemini_health(); } + if (models_url == kOpencodeModelsUrl) { + std::string b = opencode_binary(); + return !b.empty() ? "ok, " + b : "opencode CLI not found"; + } + HttpRequest req; + req.method = "GET"; + req.url = models_url; + req.headers = {{"Authorization", "Bearer " + key}}; + req.timeout = 6; + double started = py::now(); + HttpResult res = transport().send(req); + if (res.transport_failed()) { return "fail: " + py::head(short_error_text(res.error), 60); } + if (res.status == 401) { return "needs key"; } + if (!res.ok()) { return "http " + std::to_string(res.status); } + json::Value payload; + try { + payload = json::Value::loads(res.body); + } catch (const json::ParseError&) { + return "bad response"; + } + const json::Value& data = payload.is_object() ? payload.get("data") : json::Value(); + size_t count = data.is_array() ? data.size() : 0; + return "ok, " + std::to_string(count) + " models, " + std::to_string(static_cast((py::now() - started) * 1000)) + "ms"; +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/cli_backends.h b/taicpp/src/llm/cli_backends.h new file mode 100644 index 0000000..9268169 --- /dev/null +++ b/taicpp/src/llm/cli_backends.h @@ -0,0 +1,70 @@ +#pragma once +// Prompt-JSON CLI backends of tai.py (6963-7370, 7676-7757): opencode, grok, +// claude, codex, gemini binaries, prompt builders, output parsers, runners and +// health probes. The muse CLI (sessions, streaming) lives in muse.h. +#include "helpers.h" +#include "json.h" +#include "proc.h" +#include +#include +#include +#include +namespace detz::llm { + +std::string muse_binary(); // MUSE_BIN or which("muse"), "" when missing +std::string opencode_binary(); +std::string grok_binary(); // GROK_BIN, grok, grokcli +std::string claude_binary(); +std::string codex_binary(); +std::string gemini_binary(); + +json::Value muse_output_schema(); +std::string muse_prompt(const json::Value& messages, const json::Value& tools); +std::string opencode_prompt(const json::Value& messages, const json::Value& tools); +std::string cli_json_repair_prompt(const json::Value& messages, const json::Value& tools, const std::string& bad_text); + +// opencode_find_answer: nullptr-like (is_null) when no answer object is found. +json::Value opencode_find_answer(const std::string& text); +json::Value opencode_parse(const std::string& text); +std::string opencode_parse_events(const std::string& stdout_text, int returncode, const std::string& stderr_text); + +bool cli_unknown_flag(const std::string& stderr_text); +bool cli_auth_failure(const std::string& text); +ProcResult cli_run(const std::vector& argv, const std::optional& prompt_input, + double timeout, const std::string& label); + +json::Value grok_parse_output(const std::string& stdout_text); +std::string grok_run(const std::string& prompt, double timeout); +json::Value claude_parse_output(const std::string& stdout_text); +std::string claude_run(const std::string& prompt, double timeout); +struct CodexOut { std::string stdout_text, outfile_text; bool used_schema{false}; }; +std::pair, std::string> codex_jsonl_texts(const std::string& stdout_text); +json::Value codex_parse_output(const std::string& stdout_text, const std::string& outfile_text, bool used_schema); +std::vector codex_sandbox_args(); +CodexOut codex_run(const std::string& prompt, double timeout); +json::Value gemini_parse_output(const std::string& stdout_text); +bool gemini_plan_enabled(); +std::string gemini_run(const std::string& prompt, double timeout); + +// Opencode.opencode_run: patchable runner (tests install a hook). +struct Opencode { + using Hook = std::function; + static Hook& hook(); + static std::string run(const std::string& model, const std::string& prompt, double timeout); +}; +std::string opencode_run(const std::string& model, const std::string& prompt, double timeout); +// "opencode models opencode" free list; empty vector + false when unavailable. +bool opencode_free_models(std::vector& out); + +std::string cli_version_detail(const ProcResult& done, const std::string& fallback); +std::string cli_version_only_health(const std::string& binary, const std::string& fallback_name); +std::string muse_health(); +std::string grok_health(); +std::string claude_health(); +std::string codex_health(); +std::string gemini_health(); +std::string probe_backend(const std::string& models_url, const std::string& key); + +std::vector> opencode_headers(); + +} // namespace detz::llm diff --git a/taicpp/src/llm/consts.h b/taicpp/src/llm/consts.h new file mode 100644 index 0000000..ede8a15 --- /dev/null +++ b/taicpp/src/llm/consts.h @@ -0,0 +1,70 @@ +#pragma once +// Provider and model constants, copied literally from tai.py lines 108-180. +#include +#include +namespace detz::llm { +inline constexpr const char* kVersion = "1.44.2"; // tai.py VERSION this port tracks +inline constexpr int kHttpTimeout = 60; // Consts.HTTP_TIMEOUT +inline constexpr int kStreamTimeout = 300; // Consts.STREAM_TIMEOUT +inline constexpr int kModelListTimeout = 8; + +inline constexpr const char* kMuseLabel = "muse"; +inline constexpr const char* kMuseChatUrl = "muse://exec"; +inline constexpr const char* kMuseModelsUrl = "muse://models"; +inline constexpr const char* kGrokLabel = "grok"; +inline constexpr const char* kGrokChatUrl = "grok://exec"; +inline constexpr const char* kGrokModelsUrl = "grok://models"; +inline constexpr const char* kClaudeLabel = "claude"; +inline constexpr const char* kClaudeChatUrl = "claude://exec"; +inline constexpr const char* kClaudeModelsUrl = "claude://models"; +inline constexpr const char* kCodexLabel = "codex"; +inline constexpr const char* kCodexChatUrl = "codex://exec"; +inline constexpr const char* kCodexModelsUrl = "codex://models"; +inline constexpr const char* kGeminiLabel = "gemini"; +inline constexpr const char* kGeminiChatUrl = "gemini://exec"; +inline constexpr const char* kGeminiModelsUrl = "gemini://models"; +inline constexpr const char* kOllamaLabel = "ollama"; +inline constexpr const char* kOllamaBase = "http://localhost:11434/v1"; +inline constexpr int kOllamaListTtl = 600; +inline constexpr int kMuseMaxSteps = 1; +inline constexpr const char* kMuseSessionsFile = "muse_sessions.json"; +inline constexpr const char* kMuseLocksDir = "muse_locks"; +inline constexpr int kMuseWorkerTtl = 86400; +inline constexpr int kMuseReapGrace = 30; +inline constexpr const char* kOpenrouterLabel = "openrouter"; +inline constexpr const char* kOpenrouterBase = "https://openrouter.ai/api/v1"; +inline constexpr const char* kOpenrouterModel = "deepseek/deepseek-v4.1-flash"; +inline constexpr const char* kOpencodeLabel = "opencode"; +inline constexpr const char* kOpencodeChatUrl = "opencode://run"; +inline constexpr const char* kOpencodeModelsUrl = "opencode://models"; +inline constexpr const char* kOpencodeAgent = "build"; +inline constexpr int kOpencodeParseMaxScans = 200; +inline constexpr const char* kDevplaceLabel = "devplace"; +inline constexpr const char* kDevplaceBase = "https://devplace.net/openai/v1"; +inline constexpr const char* kPreferredModel = "molodetz"; +inline constexpr const char* kZenLabel = "zen"; +inline constexpr const char* kZenBase = "https://opencode.ai/zen/v1"; +inline constexpr const char* kPollinationsLabel = "pollinations"; +inline constexpr const char* kPollinationsBase = "https://text.pollinations.ai/openai/v1"; +inline constexpr const char* kPollinationsListUrl = "https://text.pollinations.ai/models"; +inline constexpr const char* kMentionAuto = "auto"; +inline constexpr const char* kOpencodeUserAgent = "opencode/1.18.29 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.15"; +inline constexpr const char* kOpencodeClientName = "cli"; +inline constexpr const char* kModelHealthFile = "model_health.json"; +inline constexpr double kModelNeutralReward = 0.5; +inline constexpr double kModelWeightPrior = 2.0; +inline constexpr double kModelSpeedRefTps = 25.0; +inline constexpr double kModelLatencyRefMs = 8000.0; +inline constexpr int kModelCircuitThreshold = 3; +inline constexpr double kModelCircuitCooldown = 300.0; +inline constexpr int kOpenrouterFreeCacheTtl = 3600; +inline constexpr const char* kRsearchBase = "https://rsearch.app.molodetz.nl"; +inline constexpr const char* kBoxName = "tai-box"; + +const std::vector& DevplaceFreeRoutes(); // ("free", "devii", "molodetz") +const std::vector& ZenFreeModels(); +const std::vector& PollinationsModels(); // ("openai-fast",) +const std::vector& ProviderLabels(); // PROVIDER_LABELS order +const std::vector& RetryBackoff(); // (2, 5, 10, 20, 30, 60) +const std::vector& CliChatUrls(); +} // namespace detz::llm diff --git a/taicpp/src/llm/helpers.cpp b/taicpp/src/llm/helpers.cpp new file mode 100644 index 0000000..ef7bdf6 --- /dev/null +++ b/taicpp/src/llm/helpers.cpp @@ -0,0 +1,158 @@ +#include "helpers.h" +#include "py.h" +#include +#include +#include +namespace detz::llm { + +std::string short_error(const std::exception& exc) { + std::string text = exc.what(); + if (text.empty()) { text = typeid(exc).name(); } + return py::head(text, 300); +} +std::string short_error_text(const std::string& text) { + return py::head(text.empty() ? std::string("Exception") : text, 300); +} +std::string truncate(const std::string& text, size_t head, size_t tail) { + size_t n = py::len(text); + if (n <= head + tail + 50) { return text; } + size_t skipped = n - head - tail; + return py::head(text, static_cast(head)) + "\n...[" + std::to_string(skipped) + " chars truncated]...\n" + + py::tail(text, static_cast(tail)); +} +int64_t estimate_tokens(const std::string& text) { + return std::max(1, static_cast(py::len(text)) / 4); +} +int64_t estimate_tokens(const json::Value& content) { + if (content.is_string()) { return estimate_tokens(content.str()); } + if (!content.truthy()) { return 1; } + // len() of a non-string truthy value (a list of parts) is its element count. + return std::max(1, static_cast(content.size()) / 4); +} +std::string human_size(int64_t num) { + if (num < 0) { num = 0; } + char buf[64]; + if (num < 1024) { std::snprintf(buf, sizeof buf, "%lld B", static_cast(num)); } + else if (num < 1048576) { std::snprintf(buf, sizeof buf, "%.1f KB", static_cast(num) / 1024); } + else { std::snprintf(buf, sizeof buf, "%.1f MB", static_cast(num) / 1048576); } + return buf; +} +std::string tool_name_clean(const json::Value& raw_name) { + std::string name = py::str_or_empty(raw_name); + size_t pos = name.find("<|"); + if (pos != std::string::npos) { name = name.substr(0, pos); } + return py::strip(name); +} +bool tool_name_valid(const std::string& name) { + // ^[a-zA-Z0-9_.-]{1,64}$ ($ also matches before a trailing newline in Python re) + std::string s = name; + if (!s.empty() && s.back() == '\n') { s.pop_back(); } + if (s.empty() || s.size() > 64) { return false; } + for (unsigned char c : s) { + if (!(std::isalnum(c) || c == '_' || c == '.' || c == '-')) { return false; } + } + return true; +} +std::string tool_args_valid(const json::Value& arguments) { + if (arguments.is_string()) { + try { + json::Value::loads(arguments.str()); + return arguments.str(); + } catch (const json::ParseError&) { + return "{}"; + } + } + return "{}"; +} +std::string tool_args_text(const json::Value& raw) { + if (!raw.truthy()) { return "{}"; } + if (raw.is_string()) { return raw.str(); } + return raw.dumps(); +} +std::string action_signature(const std::string& name, const json::Value& raw_arguments) { + std::string flat; + try { + json::DumpOptions opt; + opt.sort_keys = true; + flat = json::Value::loads(tool_args_text(raw_arguments)).dumps(opt); + } catch (const json::ParseError&) { + flat = py::str_or_empty(raw_arguments); + } + return name + " " + py::head(flat, 500); +} +std::string result_digest(const std::string& result) { return py::sha1_hex(result); } + +bool transient_failure_text(const std::string& what, int status) { + static const int permanent[] = {400, 401, 402, 403, 404, 405, 410, 413, 422}; + for (int p : permanent) { if (status == p) { return false; } } + return what.find("CLI not found") == std::string::npos; +} +bool transient_failure(const BackendError& exc) { return transient_failure_text(exc.what(), exc.status()); } + +json::Value clean_history(const json::Value& messages) { + std::vector dropped; + std::vector kept; + for (const auto& original : messages.items()) { + json::Value message = original; + const json::Value& role = message.get("role"); + json::Value calls = (role.is_string() && role.str() == "assistant") ? message.get("tool_calls") : json::Value(); + if (calls.truthy()) { + std::vector good; + for (const auto& call : calls.items()) { + const json::Value& fn = call.get("function").truthy() ? call.get("function") : json::Value(); + std::string name = py::str_or_empty(fn.get("name")); + if (tool_name_valid(name)) { good.push_back(call); } + } + for (const auto& call : calls.items()) { + if (std::find(good.begin(), good.end(), call) == good.end()) { dropped.push_back(call.get("id")); } + } + if (good.size() != calls.size()) { + json::Value goodv = json::Value::array(); + for (auto& g : good) { goodv.push(g); } + message.set("tool_calls", good.empty() ? json::Value() : goodv); + if (good.empty() && !message.get("content").truthy()) { continue; } + } + json::Value fixed = json::Value::array(); + const json::Value& current = message.get("tool_calls"); + if (current.truthy()) { + for (const auto& call : current.items()) { + const json::Value& fn = call.get("function"); + if (fn.is_object() && json::Value(tool_args_valid(fn.get("arguments"))) != fn.get("arguments")) { + json::Value newcall = call; + json::Value newfn = fn; + newfn.set("arguments", tool_args_valid(fn.get("arguments"))); + newcall.set("function", newfn); + fixed.push(newcall); + } else { + fixed.push(call); + } + } + } + if (fixed.size() > 0) { message.set("tool_calls", fixed); } + } + kept.push_back(message); + } + json::Value out = json::Value::array(); + for (auto& m : kept) { + const json::Value& r = m.get("role"); + if (r.is_string() && r.str() == "tool" && + std::find(dropped.begin(), dropped.end(), m.get("tool_call_id")) != dropped.end()) { continue; } + out.push(m); + } + return out; +} + +int64_t conversation_tokens(const json::Value& messages, const json::Value& tools) { + int64_t total = 0; + for (const auto& item : messages.items()) { + total += estimate_tokens(item.get("content")); + const json::Value& calls = item.get("tool_calls"); + if (calls.truthy()) { + for (const auto& call : calls.items()) { total += estimate_tokens(call.dumps()); } + } + } + if (tools.truthy()) { total += estimate_tokens(tools.dumps()); } + return total; +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/helpers.h b/taicpp/src/llm/helpers.h new file mode 100644 index 0000000..9016963 --- /dev/null +++ b/taicpp/src/llm/helpers.h @@ -0,0 +1,37 @@ +#pragma once +// Small shared helpers of tai.py (lines 588-1126): errors, truncation, token +// estimates, tool-call text cleanup, history cleanup. +#include "json.h" +#include +#include +#include +#include +namespace detz::llm { + +// class BackendError(Exception): message plus HTTP status (0 when none). +class BackendError : public std::runtime_error { + public: + explicit BackendError(const std::string& message, int status = 0) : std::runtime_error(message), status_(status) {} + int status() const { return status_; } + private: + int status_; +}; + +std::string short_error(const std::exception& exc); // str(exc) or type name, [:300] +std::string short_error_text(const std::string& text); // str(exc)[:300] for plain text +std::string truncate(const std::string& text, size_t head = 1500, size_t tail = 500); +int64_t estimate_tokens(const std::string& text); // max(1, len // 4) +int64_t estimate_tokens(const json::Value& content); // None/str aware (`text or ""`) +std::string human_size(int64_t num); +std::string tool_name_clean(const json::Value& raw_name); +bool tool_name_valid(const std::string& name); // TOOL_NAME_RE +std::string tool_args_valid(const json::Value& arguments);// text strict backends accept +std::string tool_args_text(const json::Value& raw); +std::string action_signature(const std::string& name, const json::Value& raw_arguments); +std::string result_digest(const std::string& result); +json::Value clean_history(const json::Value& messages); +bool transient_failure(const BackendError& exc); +bool transient_failure_text(const std::string& what, int status); +int64_t conversation_tokens(const json::Value& messages, const json::Value& tools = json::Value()); + +} // namespace detz::llm diff --git a/taicpp/src/llm/http_transport.cpp b/taicpp/src/llm/http_transport.cpp new file mode 100644 index 0000000..51b4fe0 --- /dev/null +++ b/taicpp/src/llm/http_transport.cpp @@ -0,0 +1,168 @@ +#include "http_transport.h" +#include +#include +#include +#include +namespace detz::llm { + +namespace { +struct Ctx { + std::string body; + size_t max_bytes{0}; + bool truncated{false}; + std::map* headers{nullptr}; + const LineSink* sink{nullptr}; + std::string pending; + bool stopped{false}; + CURL* handle{nullptr}; + bool status_known{false}; + bool error_status{false}; +}; + +std::string lower_ascii(std::string s) { + for (auto& c : s) { c = static_cast(std::tolower(static_cast(c))); } + return s; +} + +size_t header_cb(char* buf, size_t size, size_t n, void* user) { + auto* ctx = static_cast(user); + std::string line(buf, size * n); + if (line.rfind("HTTP/", 0) == 0) { ctx->headers->clear(); return size * n; } // new response (redirect) + size_t colon = line.find(':'); + if (colon != std::string::npos) { + std::string name = lower_ascii(line.substr(0, colon)); + std::string val = line.substr(colon + 1); + while (!val.empty() && (val.front() == ' ' || val.front() == '\t')) { val.erase(0, 1); } + while (!val.empty() && (val.back() == '\r' || val.back() == '\n')) { val.pop_back(); } + (*ctx->headers)[name] = val; + } + return size * n; +} + +size_t body_cb(char* buf, size_t size, size_t n, void* user) { + auto* ctx = static_cast(user); + size_t total = size * n; + if (!ctx->status_known && ctx->sink != nullptr) { + long st = 0; + curl_easy_getinfo(ctx->handle, CURLINFO_RESPONSE_CODE, &st); + ctx->status_known = true; + ctx->error_status = !(st >= 200 && st < 300); + } + if (ctx->sink == nullptr || ctx->error_status) { + if (ctx->max_bytes && ctx->body.size() >= ctx->max_bytes) { ctx->truncated = true; return 0; } + size_t room = ctx->max_bytes ? ctx->max_bytes - ctx->body.size() : total; + size_t take = std::min(total, room); + ctx->body.append(buf, take); + if (ctx->max_bytes && ctx->body.size() >= ctx->max_bytes && take < total) { ctx->truncated = true; return 0; } + return total; + } + ctx->pending.append(buf, total); + size_t nl; + while ((nl = ctx->pending.find('\n')) != std::string::npos) { + std::string line = ctx->pending.substr(0, nl + 1); + ctx->pending.erase(0, nl + 1); + if (!(*ctx->sink)(line)) { ctx->stopped = true; return 0; } + } + return total; +} + +std::string curl_error_text(CURLcode code) { + switch (code) { + case CURLE_COULDNT_CONNECT: return ""; + case CURLE_COULDNT_RESOLVE_HOST: return ""; + case CURLE_OPERATION_TIMEDOUT: return "timed out"; + case CURLE_SSL_CONNECT_ERROR: return ""; + default: return std::string(""; + } +} + +HttpResult perform(const HttpRequest& req, const LineSink* sink) { + static std::once_flag once; + std::call_once(once, [] { curl_global_init(CURL_GLOBAL_DEFAULT); }); + HttpResult out; + CURL* h = curl_easy_init(); + if (h == nullptr) { out.error = "curl init failed"; return out; } + Ctx ctx; + ctx.handle = h; + ctx.max_bytes = req.max_bytes; + ctx.headers = &out.headers; + ctx.sink = sink; + curl_easy_setopt(h, CURLOPT_URL, req.url.c_str()); + curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(h, CURLOPT_MAXREDIRS, 10L); + curl_easy_setopt(h, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(h, CURLOPT_ACCEPT_ENCODING, ""); + long secs = static_cast(req.timeout > 0 ? req.timeout : 60); + curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT, secs); + curl_easy_setopt(h, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(h, CURLOPT_LOW_SPEED_TIME, secs); + const std::string& m = req.method; + if (m == "GET") { curl_easy_setopt(h, CURLOPT_HTTPGET, 1L); } + else if (m == "HEAD") { curl_easy_setopt(h, CURLOPT_NOBODY, 1L); } + else if (m == "POST") { curl_easy_setopt(h, CURLOPT_POST, 1L); } + else { curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, m.c_str()); } + if (req.body.has_value()) { + if (m == "GET" || m == "HEAD") { curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, m.c_str()); } + curl_easy_setopt(h, CURLOPT_POSTFIELDS, req.body->data()); + curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(req.body->size())); + } else if (m == "POST") { + curl_easy_setopt(h, CURLOPT_POSTFIELDS, ""); + curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(0)); + } + curl_slist* list = nullptr; + bool has_ua = false; + for (const auto& kv : req.headers) { + if (lower_ascii(kv.first) == "user-agent") { has_ua = true; } + list = curl_slist_append(list, (kv.first + ": " + kv.second).c_str()); + } + // curl would add Expect/Accept defaults; urllib sends none of them. + list = curl_slist_append(list, "Expect:"); + if (!has_ua) { list = curl_slist_append(list, "User-Agent: Python-urllib/3.12"); } + curl_easy_setopt(h, CURLOPT_HTTPHEADER, list); + curl_easy_setopt(h, CURLOPT_HEADERFUNCTION, header_cb); + curl_easy_setopt(h, CURLOPT_HEADERDATA, &ctx); + curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, body_cb); + curl_easy_setopt(h, CURLOPT_WRITEDATA, &ctx); + CURLcode rc = curl_easy_perform(h); + long status = 0; + curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &status); + curl_slist_free_all(list); + curl_easy_cleanup(h); + if (!ctx.pending.empty() && sink != nullptr && !ctx.stopped && status >= 200 && status < 300) { + (*sink)(ctx.pending); + ctx.pending.clear(); + } + bool benign = ctx.truncated || ctx.stopped; + if (rc != CURLE_OK && !benign) { + if (status == 0) { + out.error = curl_error_text(rc); + return out; + } + } + if (rc != CURLE_OK && !benign) { out.stream_error = curl_error_text(rc); } + out.status = status; + out.body = ctx.body; + auto it = out.headers.find("content-type"); + if (it != out.headers.end()) { out.content_type = it->second; } + return out; +} +} // namespace + +HttpResult CurlTransport::send(const HttpRequest& req) { return perform(req, nullptr); } + +HttpResult CurlTransport::stream(const HttpRequest& req, const LineSink& sink) { + // Lines are delivered for 2xx only; on an error status the body is kept + // whole (first 2000 bytes) so the caller can quote it like HTTPError.read(n). + HttpRequest r = req; + if (r.max_bytes == 0) { r.max_bytes = 2000; } + return perform(r, &sink); +} + +namespace { +CurlTransport g_curl; +HttpTransport* g_current = nullptr; +} // namespace +HttpTransport& transport() { return g_current != nullptr ? *g_current : g_curl; } +void set_transport(HttpTransport* t) { g_current = t; } + +} // namespace detz::llm diff --git a/taicpp/src/llm/http_transport.h b/taicpp/src/llm/http_transport.h new file mode 100644 index 0000000..ca39d45 --- /dev/null +++ b/taicpp/src/llm/http_transport.h @@ -0,0 +1,57 @@ +#pragma once +// HTTP seam for the LLM layer and the web tools. tai.py uses urllib; this is +// the libcurl equivalent plus an injectable fake for tests (real network is +// never used by unit tests, integration tests use the curl transport). +#include +#include +#include +#include +#include +#include +#include +namespace detz::llm { + +struct HttpRequest { + std::string method{"GET"}; + std::string url; + std::vector> headers; + std::optional body; + double timeout{60}; // idle timeout in seconds (urllib socket timeout) + size_t max_bytes{0}; // 0 = unlimited; response.read(n) semantics otherwise +}; + +struct HttpResult { + long status{0}; // 0 when the transport failed + std::string body; + std::string content_type; // Content-Type header value, "" when absent + std::map headers; // lower-case names + std::string error; // URLError-style text when status == 0 + std::string stream_error; // transport error after a status was received + bool ok() const { return status >= 200 && status < 300; } // urlopen succeeded + bool transport_failed() const { return status == 0; } +}; + +// Called per complete line (without trailing newline stripping: raw bytes up to +// and including "\n"). Return false to stop reading. +using LineSink = std::function; + +class HttpTransport { + public: + virtual ~HttpTransport() = default; + virtual HttpResult send(const HttpRequest& req) = 0; + // Streaming read: body is delivered line by line, result.body stays empty on + // success; on HTTP error status the first 2000 bytes land in result.body. + virtual HttpResult stream(const HttpRequest& req, const LineSink& sink) = 0; +}; + +class CurlTransport : public HttpTransport { + public: + HttpResult send(const HttpRequest& req) override; + HttpResult stream(const HttpRequest& req, const LineSink& sink) override; +}; + +// Process-wide transport; defaults to CurlTransport. Tests swap it. +HttpTransport& transport(); +void set_transport(HttpTransport* t); // nullptr restores curl + +} // namespace detz::llm diff --git a/taicpp/src/llm/json.cpp b/taicpp/src/llm/json.cpp new file mode 100644 index 0000000..876a169 --- /dev/null +++ b/taicpp/src/llm/json.cpp @@ -0,0 +1,455 @@ +#include "json.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace detz::json { + +namespace { +const Value kNull; + +void put_utf8(std::string& out, uint32_t cp) { + if (cp < 0x80) { out += static_cast(cp); } + else if (cp < 0x800) { out += static_cast(0xC0 | (cp >> 6)); out += static_cast(0x80 | (cp & 0x3F)); } + else if (cp < 0x10000) { out += static_cast(0xE0 | (cp >> 12)); out += static_cast(0x80 | ((cp >> 6) & 0x3F)); out += static_cast(0x80 | (cp & 0x3F)); } + else { out += static_cast(0xF0 | (cp >> 18)); out += static_cast(0x80 | ((cp >> 12) & 0x3F)); out += static_cast(0x80 | ((cp >> 6) & 0x3F)); out += static_cast(0x80 | (cp & 0x3F)); } +} + +// Decode one code point; invalid bytes become U+FFFD like errors="replace". +uint32_t next_cp(const std::string& s, size_t& i) { + unsigned char c = static_cast(s[i]); + if (c < 0x80) { ++i; return c; } + int need = 0; uint32_t cp = 0; + if ((c & 0xE0) == 0xC0) { need = 1; cp = c & 0x1F; } + else if ((c & 0xF0) == 0xE0) { need = 2; cp = c & 0x0F; } + else if ((c & 0xF8) == 0xF0) { need = 3; cp = c & 0x07; } + else { ++i; return 0xFFFD; } + if (i + static_cast(need) >= s.size()) { ++i; return 0xFFFD; } + size_t j = i + 1; + for (int k = 0; k < need; ++k, ++j) { + unsigned char d = static_cast(s[j]); + if ((d & 0xC0) != 0x80) { ++i; return 0xFFFD; } + cp = (cp << 6) | (d & 0x3F); + } + i = j; + return cp; +} + +void hex4(std::string& out, uint32_t v) { + char buf[8]; + std::snprintf(buf, sizeof buf, "\\u%04x", v); + out += buf; +} +} // namespace + +std::string quote(const std::string& s, bool ensure_ascii) { + std::string out = "\""; + size_t i = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + if (c < 0x80) { + ++i; + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (c < 0x20 || (ensure_ascii && c == 0x7f)) { hex4(out, c); } + else { out += static_cast(c); } + } + continue; + } + size_t start = i; + uint32_t cp = next_cp(s, i); + if (!ensure_ascii) { out.append(s, start, i - start); continue; } + if (cp >= 0x10000) { + cp -= 0x10000; + hex4(out, 0xD800 | (cp >> 10)); + hex4(out, 0xDC00 | (cp & 0x3FF)); + } else { + hex4(out, cp); + } + } + out += '"'; + return out; +} + +std::string float_repr(double v) { + if (std::isnan(v)) { return "nan"; } + if (std::isinf(v)) { return v < 0 ? "-inf" : "inf"; } + if (v == 0.0) { return std::signbit(v) ? "-0.0" : "0.0"; } + char buf[64]; + auto r = std::to_chars(buf, buf + sizeof buf, v, std::chars_format::scientific); + std::string sci(buf, r.ptr); + // sci looks like "-d.ddde+XX" + bool neg = false; + size_t pos = 0; + if (sci[0] == '-') { neg = true; pos = 1; } + size_t epos = sci.find('e'); + std::string mant = sci.substr(pos, epos - pos); + int exp10 = std::atoi(sci.c_str() + epos + 1); + std::string digits; + for (char c : mant) { if (c != '.') { digits += c; } } + std::string out = neg ? "-" : ""; + if (exp10 < -4 || exp10 >= 16) { + out += digits.substr(0, 1); + if (digits.size() > 1) { out += "." + digits.substr(1); } + char eb[16]; + std::snprintf(eb, sizeof eb, "e%c%02d", exp10 < 0 ? '-' : '+', std::abs(exp10)); + out += eb; + return out; + } + if (exp10 < 0) { + out += "0." + std::string(static_cast(-exp10 - 1), '0') + digits; + } else if (static_cast(exp10) + 1 >= digits.size()) { + out += digits + std::string(static_cast(exp10) + 1 - digits.size(), '0') + ".0"; + } else { + out += digits.substr(0, static_cast(exp10) + 1) + "." + digits.substr(static_cast(exp10) + 1); + } + return out; +} + +Value Value::big_int(const std::string& digits, bool negative) { + Value v; + v.type_ = Type::Int; + v.i_ = negative ? INT64_MIN : INT64_MAX; + v.big_ = (negative ? "-" : "") + digits; + return v; +} +int64_t Value::as_int(int64_t dflt) const { + if (type_ == Type::Int || type_ == Type::Bool) { return i_; } + if (type_ == Type::Double) { return static_cast(d_); } + return dflt; +} +double Value::as_double(double dflt) const { + if (type_ == Type::Double) { return d_; } + if (type_ == Type::Int && !big_.empty()) { return std::strtod(big_.c_str(), nullptr); } + if (type_ == Type::Int || type_ == Type::Bool) { return static_cast(i_); } + return dflt; +} +bool Value::truthy() const { + switch (type_) { + case Type::Null: return false; + case Type::Bool: case Type::Int: return i_ != 0; + case Type::Double: return d_ != 0.0; + case Type::String: return !s_.empty(); + case Type::Array: return !a_.empty(); + case Type::Object: return !o_.empty(); + } + return false; +} +void Value::push(Value v) { + if (type_ == Type::Null) { type_ = Type::Array; } + a_.push_back(std::move(v)); +} +bool Value::has(const std::string& key) const { + for (const auto& m : o_) { if (m.first == key) { return true; } } + return false; +} +const Value& Value::get(const std::string& key) const { + for (const auto& m : o_) { if (m.first == key) { return m.second; } } + return kNull; +} +Value Value::get_or(const std::string& key, Value dflt) const { + for (const auto& m : o_) { if (m.first == key) { return m.second; } } + return dflt; +} +Value& Value::operator[](const std::string& key) { + if (type_ == Type::Null) { type_ = Type::Object; } + for (auto& m : o_) { if (m.first == key) { return m.second; } } + o_.emplace_back(key, Value()); + return o_.back().second; +} +void Value::set(const std::string& key, Value v) { (*this)[key] = std::move(v); } +bool Value::erase(const std::string& key) { + for (auto it = o_.begin(); it != o_.end(); ++it) { + if (it->first == key) { o_.erase(it); return true; } + } + return false; +} +bool Value::operator==(const Value& o) const { + if (type_ == Type::Int && o.type_ == Type::Int && (!big_.empty() || !o.big_.empty())) { return big_ == o.big_ && i_ == o.i_; } + if (is_number() && o.is_number()) { return as_double() == o.as_double(); } + if (type_ != o.type_) { return false; } + switch (type_) { + case Type::Null: return true; + case Type::Bool: return i_ == o.i_; + case Type::String: return s_ == o.s_; + case Type::Array: return a_ == o.a_; + case Type::Object: { + if (o_.size() != o.o_.size()) { return false; } + for (const auto& m : o_) { if (!o.has(m.first) || !(o.get(m.first) == m.second)) { return false; } } + return true; + } + default: return false; + } +} + +std::string Value::dumps() const { return dumps(DumpOptions{}); } +std::string Value::compact() const { + DumpOptions o; o.item_sep = ","; o.key_sep = ":"; + return dumps(o); +} +std::string Value::dumps(const DumpOptions& opt) const { + std::string out; + dump(out, opt, 0); + return out; +} +void Value::dump(std::string& out, const DumpOptions& opt, int depth) const { + auto newline = [&](int d) { + out += '\n'; + out.append(static_cast(d * opt.indent), ' '); + }; + switch (type_) { + case Type::Null: out += "null"; return; + case Type::Bool: out += i_ ? "true" : "false"; return; + case Type::Int: out += int_text(); return; + case Type::Double: + if (std::isnan(d_)) { out += "NaN"; } + else if (std::isinf(d_)) { out += d_ < 0 ? "-Infinity" : "Infinity"; } + else { out += float_repr(d_); } + return; + case Type::String: out += quote(s_, opt.ensure_ascii); return; + case Type::Array: { + if (a_.empty()) { out += "[]"; return; } + out += '['; + bool first = true; + for (const auto& v : a_) { + if (!first) { out += opt.item_sep; } + if (opt.indent >= 0) { newline(depth + 1); } + first = false; + v.dump(out, opt, depth + 1); + } + if (opt.indent >= 0) { newline(depth); } + out += ']'; + return; + } + case Type::Object: { + if (o_.empty()) { out += "{}"; return; } + std::vector order; + for (const auto& m : o_) { order.push_back(&m); } + if (opt.sort_keys) { + std::stable_sort(order.begin(), order.end(), [](const Member* a, const Member* b) { return a->first < b->first; }); + } + out += '{'; + bool first = true; + for (const Member* m : order) { + if (!first) { out += opt.item_sep; } + if (opt.indent >= 0) { newline(depth + 1); } + first = false; + out += quote(m->first, opt.ensure_ascii); + out += opt.key_sep; + m->second.dump(out, opt, depth + 1); + } + if (opt.indent >= 0) { newline(depth); } + out += '}'; + return; + } + } +} + +namespace { +class Parser { + public: + explicit Parser(const std::string& t) : t_(t) {} + Value run() { + if (t_.compare(0, 3, "\xEF\xBB\xBF") == 0) { fail_at("Unexpected UTF-8 BOM (decode using utf-8-sig)", 0); } + skip(); + Value v = value(0); + skip(); + if (p_ != t_.size()) { fail("Extra data"); } + return v; + } + Value run_at(size_t pos, size_t& end) { + p_ = pos; + Value v = value(0); + end = p_; + return v; + } + private: + // JSONDecodeError text: "msg: line L column C (char N)" with code point offsets. + [[noreturn]] void fail_at(const std::string& m, size_t pos) const { + size_t cp_pos = 0, line = 1, last_nl_cp = 0; + bool have_nl = false; + size_t limit = pos < t_.size() ? pos : t_.size(); + for (size_t i = 0; i < limit; ++i) { + if ((static_cast(t_[i]) & 0xC0) != 0x80) { + if (t_[i] == '\n') { ++line; last_nl_cp = cp_pos; have_nl = true; } + ++cp_pos; + } + } + if (pos > t_.size()) { cp_pos += pos - t_.size(); } + size_t col = have_nl ? cp_pos - last_nl_cp : cp_pos + 1; + throw ParseError(m + ": line " + std::to_string(line) + " column " + std::to_string(col) + " (char " + std::to_string(cp_pos) + ")"); + } + [[noreturn]] void fail(const std::string& m) const { fail_at(m, p_); } + void skip() { + while (p_ < t_.size() && (t_[p_] == ' ' || t_[p_] == '\t' || t_[p_] == '\n' || t_[p_] == '\r')) { ++p_; } + } + bool lit(const char* w) { + size_t n = std::strlen(w); + if (t_.compare(p_, n, w) == 0) { p_ += n; return true; } + return false; + } + Value value(int depth) { + if (depth > 900) { fail("recursion"); } + if (p_ >= t_.size()) { fail("Expecting value"); } + char c = t_[p_]; + if (c == '{') { return object(depth); } + if (c == '[') { return array(depth); } + if (c == '"') { return Value(string()); } + if (lit("null")) { return Value(); } + if (lit("true")) { return Value(true); } + if (lit("false")) { return Value(false); } + if (lit("NaN")) { return Value(std::numeric_limits::quiet_NaN()); } + if (lit("Infinity")) { return Value(std::numeric_limits::infinity()); } + if (lit("-Infinity")) { return Value(-std::numeric_limits::infinity()); } + return number(); + } + Value number() { + size_t s = p_; + if (p_ < t_.size() && t_[p_] == '-') { ++p_; } + if (p_ >= t_.size()) { p_ = s; fail("Expecting value"); } + if (t_[p_] == '0') { ++p_; } + else if (t_[p_] >= '1' && t_[p_] <= '9') { while (p_ < t_.size() && std::isdigit(static_cast(t_[p_]))) { ++p_; } } + else { p_ = s; fail("Expecting value"); } + bool real = false; + if (p_ + 1 < t_.size() && t_[p_] == '.' && std::isdigit(static_cast(t_[p_ + 1]))) { + real = true; ++p_; + while (p_ < t_.size() && std::isdigit(static_cast(t_[p_]))) { ++p_; } + } + if (p_ < t_.size() && (t_[p_] == 'e' || t_[p_] == 'E')) { + size_t q = p_ + 1; + if (q < t_.size() && (t_[q] == '+' || t_[q] == '-')) { ++q; } + if (q < t_.size() && std::isdigit(static_cast(t_[q]))) { + real = true; p_ = q; + while (p_ < t_.size() && std::isdigit(static_cast(t_[p_]))) { ++p_; } + } + } + std::string tok = t_.substr(s, p_ - s); + if (!real) { + errno = 0; + char* end = nullptr; + long long v = std::strtoll(tok.c_str(), &end, 10); + if (errno == 0) { return Value(v); } + bool neg = !tok.empty() && tok[0] == '-'; + return Value::big_int(neg ? tok.substr(1) : tok, neg); + } + return Value(std::strtod(tok.c_str(), nullptr)); + } + uint32_t hex(size_t at) const { + if (at + 4 > t_.size()) { fail_at("Invalid \\uXXXX escape", at - 1); } + uint32_t v = 0; + for (size_t i = at; i < at + 4; ++i) { + char c = t_[i]; v <<= 4; + if (c >= '0' && c <= '9') { v |= static_cast(c - '0'); } + else if (c >= 'a' && c <= 'f') { v |= static_cast(c - 'a' + 10); } + else if (c >= 'A' && c <= 'F') { v |= static_cast(c - 'A' + 10); } + else { fail_at("Invalid \\uXXXX escape", at - 1); } + } + return v; + } + std::string string() { + size_t begin = p_; + ++p_; // opening quote + std::string out; + for (;;) { + if (p_ >= t_.size()) { fail_at("Unterminated string starting at", begin); } + unsigned char c = static_cast(t_[p_]); + if (c == '"') { ++p_; return out; } + if (c < 0x20) { fail("Invalid control character at"); } + if (c != '\\') { out += static_cast(c); ++p_; continue; } + ++p_; + if (p_ >= t_.size()) { fail_at("Unterminated string starting at", begin); } + char e = t_[p_++]; + switch (e) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + uint32_t cp = hex(p_); p_ += 4; + if (cp >= 0xD800 && cp <= 0xDBFF && p_ + 6 <= t_.size() && t_[p_] == '\\' && t_[p_ + 1] == 'u') { + uint32_t lo = hex(p_ + 2); + if (lo >= 0xDC00 && lo <= 0xDFFF) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + p_ += 6; + } + } + put_utf8(out, cp); + break; + } + default: fail_at("Invalid \\escape", p_ - 2); + } + } + } + Value array(int depth) { + ++p_; + Value v = Value::array(); + skip(); + if (p_ < t_.size() && t_[p_] == ']') { ++p_; return v; } + for (;;) { + skip(); + v.push(value(depth + 1)); + skip(); + if (p_ >= t_.size()) { fail("Expecting ',' delimiter"); } + if (t_[p_] == ',') { + size_t comma = p_; + ++p_; + skip(); + if (p_ < t_.size() && t_[p_] == ']') { fail_at("Illegal trailing comma before end of array", comma); } + continue; + } + if (t_[p_] == ']') { ++p_; return v; } + fail("Expecting ',' delimiter"); + } + } + Value object(int depth) { + ++p_; + Value v = Value::object(); + skip(); + if (p_ < t_.size() && t_[p_] == '}') { ++p_; return v; } + for (;;) { + skip(); + if (p_ >= t_.size() || t_[p_] != '"') { fail("Expecting property name enclosed in double quotes"); } + std::string key = string(); + skip(); + if (p_ >= t_.size() || t_[p_] != ':') { fail("Expecting ':' delimiter"); } + ++p_; + skip(); + v.set(key, value(depth + 1)); + skip(); + if (p_ >= t_.size()) { fail("Expecting ',' delimiter"); } + if (t_[p_] == ',') { + size_t comma = p_; + ++p_; + skip(); + if (p_ < t_.size() && t_[p_] == '}') { fail_at("Illegal trailing comma before end of object", comma); } + continue; + } + if (t_[p_] == '}') { ++p_; return v; } + fail("Expecting ',' delimiter"); + } + } + const std::string& t_; + size_t p_{0}; +}; +} // namespace + +Value Value::loads(const std::string& text) { return Parser(text).run(); } +Value Value::raw_decode(const std::string& text, size_t pos, size_t& end) { return Parser(text).run_at(pos, end); } + +} // namespace detz::json diff --git a/taicpp/src/llm/json.h b/taicpp/src/llm/json.h new file mode 100644 index 0000000..af75857 --- /dev/null +++ b/taicpp/src/llm/json.h @@ -0,0 +1,111 @@ +#pragma once +// Python-compatible JSON value (tai.py uses json.dumps/json.loads everywhere). +// Objects keep insertion order like a Python dict. dumps() reproduces the +// default json.dumps text byte for byte (ensure_ascii, ", " and ": "). +#include +#include +#include +#include +#include +namespace detz::json { + +class ParseError : public std::runtime_error { + public: + explicit ParseError(const std::string& m) : std::runtime_error(m) {} +}; + +struct DumpOptions { + bool ensure_ascii{true}; + bool sort_keys{false}; + // separators: item_sep / key_sep; defaults are Python's (", ", ": "). + std::string item_sep{", "}; + std::string key_sep{": "}; + int indent{-1}; // -1: none +}; + +class Value { + public: + enum class Type { Null, Bool, Int, Double, String, Array, Object }; + using Member = std::pair; + + Value() = default; + Value(std::nullptr_t) {} + Value(bool b) : type_(Type::Bool), i_(b ? 1 : 0) {} + Value(int v) : type_(Type::Int), i_(v) {} + Value(long v) : type_(Type::Int), i_(v) {} + Value(long long v) : type_(Type::Int), i_(v) {} + Value(unsigned long v) : type_(Type::Int), i_(static_cast(v)) {} + Value(double v) : type_(Type::Double), d_(v) {} + Value(const char* s) : type_(Type::String), s_(s) {} + Value(std::string s) : type_(Type::String), s_(std::move(s)) {} + + // An integer beyond int64 (Python ints are unbounded): keeps the decimal text, as_int() clamps. + static Value big_int(const std::string& digits, bool negative); + bool is_big_int() const { return type_ == Type::Int && !big_.empty(); } + std::string int_text() const { return big_.empty() ? std::to_string(i_) : big_; } + static Value array() { Value v; v.type_ = Type::Array; return v; } + static Value object() { Value v; v.type_ = Type::Object; return v; } + static Value loads(const std::string& text); + // json.JSONDecoder().raw_decode(text, pos): no whitespace skip, returns end offset. + static Value raw_decode(const std::string& text, size_t pos, size_t& end); + + Type type() const { return type_; } + bool is_null() const { return type_ == Type::Null; } + bool is_bool() const { return type_ == Type::Bool; } + bool is_int() const { return type_ == Type::Int; } + bool is_double() const { return type_ == Type::Double; } + bool is_number() const { return type_ == Type::Int || type_ == Type::Double; } + bool is_string() const { return type_ == Type::String; } + bool is_array() const { return type_ == Type::Array; } + bool is_object() const { return type_ == Type::Object; } + + bool as_bool() const { return type_ == Type::Bool && i_ != 0; } + int64_t as_int(int64_t dflt = 0) const; + double as_double(double dflt = 0.0) const; + const std::string& str() const { return s_; } + std::string str_or(const std::string& dflt) const { return is_string() ? s_ : dflt; } + + // Python truthiness: None, False, 0, 0.0, "", [], {} are falsy. + bool truthy() const; + + // arrays + size_t size() const { return type_ == Type::Array ? a_.size() : (type_ == Type::Object ? o_.size() : 0); } + const std::vector& items() const { return a_; } + std::vector& items() { return a_; } + void push(Value v); + const Value& at(size_t i) const { return a_.at(i); } + + // objects + bool has(const std::string& key) const; + const Value& get(const std::string& key) const; // null Value when missing + Value get_or(const std::string& key, Value dflt) const; + Value& operator[](const std::string& key); // insert null when missing + void set(const std::string& key, Value v); + bool erase(const std::string& key); + const std::vector& members() const { return o_; } + std::vector& members() { return o_; } + + std::string dumps() const; + std::string dumps(const DumpOptions& opt) const; + std::string compact() const; // separators (",", ":") + + bool operator==(const Value& other) const; + bool operator!=(const Value& other) const { return !(*this == other); } + + private: + void dump(std::string& out, const DumpOptions& opt, int depth) const; + Type type_{Type::Null}; + int64_t i_{0}; + double d_{0.0}; + std::string s_; + std::string big_; + std::vector a_; + std::vector o_; +}; + +// Python repr of a float (shortest round trip), used by dumps and str(). +std::string float_repr(double v); +// json.dumps of a plain string (ensure_ascii=True). +std::string quote(const std::string& s, bool ensure_ascii = true); + +} // namespace detz::json diff --git a/taicpp/src/llm/llm.cpp b/taicpp/src/llm/llm.cpp deleted file mode 100644 index 8a18e83..0000000 --- a/taicpp/src/llm/llm.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "llm.h" -#include "util/http.h" -#include -#include -namespace detz::llm { -static std::string esc(const std::string& s) { - std::string o; for (char c : s) { if (c=='"') o+="\\\""; else if (c=='\n') o+="\\n"; else o+=c; } - return o; -} -static std::string req_json(const ChatRequest& r) { - std::string b = "{\"model\":\"" + esc(r.model) + "\",\"messages\":["; - bool f = true; - for (auto& m : r.messages) { if (!f) b += ","; f = false; b += "{\"role\":\"" + esc(m.role) + "\",\"content\":\"" + esc(m.content) + "\"}"; } - b += "],\"stream\":"; b += r.stream ? "true}" : "false}"; - return b; -} -static std::string grab(const std::string& j, const std::string& key) { - auto p = j.find("\"" + key + "\""); if (p == std::string::npos) return ""; - p = j.find(':', p); if (p == std::string::npos) return ""; - auto q1 = j.find('"', p); if (q1 == std::string::npos) return ""; - auto q2 = j.find('"', q1 + 1); if (q2 == std::string::npos) return ""; - return j.substr(q1 + 1, q2 - q1 - 1); -} -void feed_sse(const std::string& payload, SseSink sink) { - std::istringstream ss(payload); std::string line, data; - auto flush = [&]() { - if (data == "[DONE]") { data.clear(); return true; } - if (!data.empty()) { std::string d = data; data.clear(); return sink(d); } - return true; - }; - while (std::getline(ss, line)) { - if (!line.empty() && line.back() == '\r') line.pop_back(); - if (line.rfind("data:", 0) == 0) { - std::string v = line.substr(5); - while (!v.empty() && v.front() == ' ') v.erase(v.begin()); - data = data.empty() ? v : data + "\n" + v; - } else if (line.empty()) { if (!flush()) return; } - } - if (!data.empty()) flush(); -} -ProviderInfo provider_info(const std::string& p) { - if (p == "anthropic") return {p, "https://api.anthropic.com", "/v1/messages"}; - if (p == "google") return {p, "https://generativelanguage.googleapis.com", "/v1beta/models"}; - if (p == "deepseek") return {p, "https://api.deepseek.com", "/v1/chat/completions"}; - if (p == "xai") return {p, "https://api.x.ai", "/v1/chat/completions"}; - if (p == "openrouter") return {p, "https://openrouter.ai", "/api/v1/chat/completions"}; - return {p == "openai" ? "openai" : "openai", "https://api.openai.com", "/v1/chat/completions"}; -} -std::string provider_names() { return "openai,anthropic,google,deepseek,xai,openrouter"; } -static std::string curl_post(const std::string& url, const std::string& body, const std::map& h) { - auto r = detz::HttpPost(url, body, h, 60); - return r.body; -} -ChatClient::ChatClient(std::string provider, std::string model, HttpPost post) - : provider_(std::move(provider)), model_(std::move(model)), post_(post ? std::move(post) : HttpPost(curl_post)) { - const char* k = std::getenv("OPENAI_API_KEY"); - if (k && *k) default_headers_["Authorization"] = std::string("Bearer ")+k; -} -ChatResponse ChatClient::chat(const ChatRequest& req) { - if (!post_) throw BackendError("no transport (mock required)", -1); - auto info = provider_info(provider_); - ChatRequest r = req; if (r.model.empty()) r.model = model_; - std::string body = post_(info.base_url + info.chat_path, req_json(r), default_headers_); - if (body.empty()) throw BackendError("empty backend response", 502); - ChatResponse out; out.model = r.model; - std::string t = grab(body, "content"); if (t.empty()) t = grab(body, "text"); - if (t.empty()) t = body; - out.text = t; - return out; -} -void ChatClient::chat_stream(const ChatRequest& req, SseSink sink) { - if (!post_) throw BackendError("no transport (mock required)", -1); - auto info = provider_info(provider_); - ChatRequest r = req; r.stream = true; if (r.model.empty()) r.model = model_; - std::string payload = post_(info.base_url + info.chat_path, req_json(r), {}); - feed_sse(payload, sink); -} -void ModelHealth::record(const std::string& m, bool ok, int ms) { samples_.push_back({m, ok, ms}); } -std::optional ModelHealth::best() const { - std::optional b; - for (auto& s : samples_) if (s.ok && (!b || s.latency_ms < b->latency_ms)) b = s; - return b; -} -Opencode::Opencode(std::string base_url, HttpPost post) : base_(std::move(base_url)), post_(std::move(post)) {} -ChatResponse Opencode::complete(const std::string& model, const std::vector& msgs) { - ChatClient c("openai", model, [this](const std::string&, const std::string& body, const std::map&) { - if (!post_) throw BackendError("no transport (mock required)", -1); - return post_(base_ + "/v1/chat/completions", body, {}); - }); - return c.chat(ChatRequest{model, msgs, false}); -} -} // namespace detz::llm diff --git a/taicpp/src/llm/llm.h b/taicpp/src/llm/llm.h deleted file mode 100644 index 7f031c2..0000000 --- a/taicpp/src/llm/llm.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once -// Thin adapters over moloapp llm providers + SSE streaming. -// Mocked transport only in tests; no real network here. -#include -#include -#include -#include -#include -#include -namespace detz::llm { -struct Message { std::string role; std::string content; }; -struct ChatRequest { std::string model; std::vector messages; bool stream{false}; }; -struct ChatResponse { std::string text; std::string model; std::string finish{"stop"}; }; -class BackendError : public std::runtime_error { - public: - explicit BackendError(const std::string& m, int code = 0) : std::runtime_error(m), code_(code) {} - int code() const { return code_; } - private: - int code_; -}; -// Transport seam: real impl wraps moloapp clients; tests inject mock. -using HttpPost = std::function& headers)>; -using SseSink = std::function; -void feed_sse(const std::string& payload, SseSink sink); -struct ProviderInfo { std::string name; std::string base_url; std::string chat_path; }; -ProviderInfo provider_info(const std::string& provider); -std::string provider_names(); -class ChatClient { - public: - explicit ChatClient(std::string provider, std::string model, HttpPost post = nullptr); - ChatResponse chat(const ChatRequest& req); - void chat_stream(const ChatRequest& req, SseSink sink); - const std::string& provider() const { return provider_; } - private: - std::string provider_, model_; HttpPost post_; - std::map default_headers_; -}; -struct ModelSample { std::string model; bool ok; int latency_ms; }; -class ModelHealth { - public: - void record(const std::string& model, bool ok, int latency_ms = 0); - std::optional best() const; - std::size_t size() const { return samples_.size(); } - private: - std::vector samples_; -}; -// Opencode backend shim (OpenAI-compatible /v1/chat/completions). -class Opencode { - public: - explicit Opencode(std::string base_url = "http://localhost:4096", HttpPost post = nullptr); - ChatResponse complete(const std::string& model, const std::vector& msgs); - private: - std::string base_; HttpPost post_; - std::map default_headers_; -}; -} // namespace detz::llm diff --git a/taicpp/src/llm/model_health.cpp b/taicpp/src/llm/model_health.cpp new file mode 100644 index 0000000..7dc2eea --- /dev/null +++ b/taicpp/src/llm/model_health.cpp @@ -0,0 +1,189 @@ +#include "model_health.h" +#include "consts.h" +#include "py.h" +#include +#include +#include +#include +#include +namespace detz::llm { + +double model_speed_reward(std::optional tps) { + if (!tps.has_value() || *tps <= 0) { return kModelNeutralReward; } + return *tps / (*tps + kModelSpeedRefTps); +} +double model_latency_reward(std::optional latency_ms) { + if (!latency_ms.has_value() || *latency_ms <= 0) { return 1.0; } + return 1.0 / (1.0 + *latency_ms / kModelLatencyRefMs); +} +double model_outcome_reward(std::optional latency_ms, std::optional tps) { + return model_speed_reward(tps) * model_latency_reward(latency_ms); +} +static double num_or0(const json::Value& v) { return v.truthy() ? v.as_double() : 0.0; } +double model_weight(const json::Value& entry) { + double attempts = num_or0(entry.get("s")) + num_or0(entry.get("f")); + return (num_or0(entry.get("r")) + kModelNeutralReward * kModelWeightPrior) / (attempts + kModelWeightPrior); +} + +ModelHealth::ModelHealth(const std::string& home) { + data_ = json::Value::object(); + data_.set("models", json::Value::object()); + data_.set("lists", json::Value::object()); + if (home.empty()) { return; } + path_ = home + "/" + kModelHealthFile; + std::ifstream in(path_); + if (!in) { return; } + std::stringstream ss; + ss << in.rdbuf(); + try { + json::Value raw = json::Value::loads(ss.str()); + if (raw.is_object()) { + if (raw.get("models").is_object()) { data_.set("models", raw.get("models")); } + if (raw.get("lists").is_object()) { data_.set("lists", raw.get("lists")); } + } + } catch (const json::ParseError&) { + } +} + +void ModelHealth::save() { + if (path_.empty()) { return; } + std::string tmp = path_ + ".tmp"; + { + std::ofstream out(tmp, std::ios::trunc); + if (!out) { return; } + out << data_.dumps(); + if (!out) { return; } + } + std::rename(tmp.c_str(), path_.c_str()); +} + +json::Value ModelHealth::entry(const std::string& provider, const std::string& model) const { + std::string key = (provider.empty() ? std::string("default") : provider) + "|" + model; + const json::Value& e = data_.get("models").get(key); + return e.truthy() ? e : json::Value::object(); +} +double ModelHealth::weight(const std::string& provider, const std::string& model) const { + return model_weight(entry(provider, model)); +} +bool ModelHealth::circuit_open(const std::string& provider, const std::string& model) const { + return num_or0(entry(provider, model).get("until")) > py::now(); +} + +void ModelHealth::record(const std::string& provider, const std::string& model, bool success, + std::optional latency_ms, std::optional tps) { + if (model.empty()) { return; } + std::string key = (provider.empty() ? std::string("default") : provider) + "|" + model; + json::Value& models = data_["models"]; + if (!models.has(key)) { + json::Value e = json::Value::object(); + e.set("s", 0); e.set("f", 0); e.set("r", 0.0); e.set("lat", 0.0); e.set("tps", 0.0); + e.set("cf", 0); e.set("until", 0.0); e.set("used", 0.0); e.set("n", 0); + models.set(key, e); + } + json::Value& e = models[key]; + e.set("n", e.get("n").as_int() + 1); + e.set("used", py::now()); + if (success) { + double reward = (!latency_ms.has_value() && !tps.has_value()) ? kModelNeutralReward : model_outcome_reward(latency_ms, tps); + e.set("s", e.get("s").as_int() + 1); + e.set("cf", 0); + e.set("until", 0.0); + e.set("r", e.get("r").as_double() + reward); + if (latency_ms.has_value()) { e.set("lat", e.get("lat").as_double() + *latency_ms); } + if (tps.has_value()) { e.set("tps", e.get("tps").as_double() + *tps); } + } else { + e.set("f", e.get("f").as_int() + 1); + e.set("cf", e.get("cf").as_int() + 1); + if (e.get("cf").as_int() >= kModelCircuitThreshold) { e.set("until", py::now() + kModelCircuitCooldown); } + } + save(); +} + +std::optional> ModelHealth::cached_list(const std::string& name, double ttl) const { + const json::Value& row = data_.get("lists").get(name); + if (!row.is_object()) { return std::nullopt; } + const json::Value& models = row.get("models"); + if (!models.truthy() || (py::now() - num_or0(row.get("at"))) > ttl) { return std::nullopt; } + std::vector out; + for (const auto& item : models.items()) { + std::string s = py::str(item); + if (!py::strip(s).empty()) { out.push_back(s); } + } + return out; +} + +static std::vector cleaned(const std::vector& models) { + std::vector out; + for (const auto& m : models) { + std::string s = py::strip(m); + if (!s.empty()) { out.push_back(s); } + } + return out; +} +static json::Value list_row(const std::vector& models) { + json::Value row = json::Value::object(); + row.set("at", py::now()); + json::Value arr = json::Value::array(); + for (const auto& m : models) { arr.push(m); } + row.set("models", arr); + return row; +} +void ModelHealth::store_list(const std::string& name, const std::vector& models) { + auto c = cleaned(models); + if (c.empty()) { return; } + data_["lists"].set(name, list_row(c)); + save(); +} +ListDiff ModelHealth::replace_list(const std::string& name, const std::string& provider, const std::vector& models) { + ListDiff diff; + auto c = cleaned(models); + if (c.empty()) { return diff; } + const json::Value& row = data_.get("lists").get(name); + std::vector old; + if (row.is_object()) { + for (const auto& item : row.get("models").items()) { old.push_back(py::str(item)); } + } + std::string prefix = provider + "|"; + auto& members = data_["models"].members(); + for (auto it = members.begin(); it != members.end();) { + if (py::startswith(it->first, prefix) && + std::find(c.begin(), c.end(), it->first.substr(prefix.size())) == c.end()) { + it = members.erase(it); + } else { + ++it; + } + } + data_["lists"].set(name, list_row(c)); + save(); + for (const auto& m : c) { if (std::find(old.begin(), old.end(), m) == old.end()) { diff.added.push_back(m); } } + for (const auto& m : old) { if (std::find(c.begin(), c.end(), m) == c.end()) { diff.removed.push_back(m); } } + return diff; +} + +std::vector ModelHealth::snapshot() const { + std::vector rows; + for (const auto& kv : data_.get("models").members()) { + HealthRow r; + size_t bar = kv.first.find('|'); + r.provider = bar == std::string::npos ? kv.first : kv.first.substr(0, bar); + r.model = bar == std::string::npos ? "" : kv.first.substr(bar + 1); + const json::Value& e = kv.second; + int64_t ok = static_cast(num_or0(e.get("s"))); + r.weight = std::round(model_weight(e) * 10000.0) / 10000.0; + r.ok = ok; + r.fail = static_cast(num_or0(e.get("f"))); + if (ok && num_or0(e.get("lat")) != 0.0) { r.avg_ms = static_cast(std::nearbyint(num_or0(e.get("lat")) / static_cast(ok))); } + if (ok && num_or0(e.get("tps")) != 0.0) { r.avg_tps = std::nearbyint(num_or0(e.get("tps")) / static_cast(ok) * 10.0) / 10.0; } + r.circuit_open = num_or0(e.get("until")) > py::now(); + r.used = num_or0(e.get("used")); + rows.push_back(r); + } + std::stable_sort(rows.begin(), rows.end(), [](const HealthRow& a, const HealthRow& b) { + if (a.weight != b.weight) { return a.weight > b.weight; } + if (a.provider != b.provider) { return a.provider < b.provider; } + return a.model < b.model; + }); + return rows; +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/model_health.h b/taicpp/src/llm/model_health.h new file mode 100644 index 0000000..d1ad116 --- /dev/null +++ b/taicpp/src/llm/model_health.h @@ -0,0 +1,47 @@ +#pragma once +// ModelHealth (tai.py 7769-7900): per provider/model reward statistics, +// circuit breaker and cached model lists, persisted in model_health.json. +#include "json.h" +#include +#include +#include +namespace detz::llm { + +double model_speed_reward(std::optional tokens_per_second); +double model_latency_reward(std::optional latency_ms); +double model_outcome_reward(std::optional latency_ms, std::optional tokens_per_second); +double model_weight(const json::Value& entry); + +struct HealthRow { + std::string provider, model; + double weight{0}; + int64_t ok{0}, fail{0}; + std::optional avg_ms; + std::optional avg_tps; + bool circuit_open{false}; + double used{0}; +}; + +struct ListDiff { std::vector added, removed; }; + +class ModelHealth { + public: + explicit ModelHealth(const std::string& home); + json::Value entry(const std::string& provider, const std::string& model) const; + double weight(const std::string& provider, const std::string& model) const; + bool circuit_open(const std::string& provider, const std::string& model) const; + void record(const std::string& provider, const std::string& model, bool success, + std::optional latency_ms = std::nullopt, std::optional tokens_per_second = std::nullopt); + // nullopt when absent or older than ttl; ttl defaults to OPENROUTER_FREE_CACHE_TTL. + std::optional> cached_list(const std::string& name, double ttl = 3600) const; + void store_list(const std::string& name, const std::vector& models); + ListDiff replace_list(const std::string& name, const std::string& provider, const std::vector& models); + std::vector snapshot() const; + const json::Value& data() const { return data_; } + private: + void save(); + std::string path_; + json::Value data_; +}; + +} // namespace detz::llm diff --git a/taicpp/src/llm/models.cpp b/taicpp/src/llm/models.cpp new file mode 100644 index 0000000..e1abcc1 --- /dev/null +++ b/taicpp/src/llm/models.cpp @@ -0,0 +1,141 @@ +#include "models.h" +#include "consts.h" +#include "http_transport.h" +#include "proc.h" +#include "py.h" +#include +namespace detz::llm { + +std::vector split_models(const std::string& spec) { + std::vector out; + for (const auto& item : py::split(py::replace(spec, "\n", ","), ",")) { + std::string s = py::strip(item); + if (!s.empty()) { out.push_back(s); } + } + return out; +} + +std::vector parse_backends(const std::string& spec) { + std::vector found; + for (const auto& entry : py::split(py::replace(spec, "\n", ";"), ";")) { + std::vector fields; + for (const auto& f : py::split(entry, "|")) { fields.push_back(py::strip(f)); } + if (fields.size() < 3) { continue; } + BackendSpec b; + b.label = fields[0]; + b.base = py::rstrip_chars(fields[1], "/"); + b.key = fields[2]; + if (!b.key.empty() && b.key[0] == '$') { b.key = env_get(b.key.substr(1)); } + b.model = fields.size() > 3 ? py::strip(fields[3]) : ""; + b.profile = fields.size() > 4 ? py::lower(py::strip(fields[4])) : ""; + if (b.profile != "" && b.profile != "opencode") { b.profile = ""; } + if (b.label.empty() || b.base.empty()) { continue; } + found.push_back(b); + } + return found; +} + +bool is_free_model(const std::string& model) { + std::string m = py::lower(model); + return py::endswith(m, ":free") || py::endswith(m, "-free"); +} +bool is_free_openrouter(const std::string& model) { + return py::endswith(model, ":free") || model == "openrouter/free"; +} +std::vector prefer_muse(std::vector models) { + std::stable_sort(models.begin(), models.end(), [](const std::string& a, const std::string& b) { + int ka = py::lower(a).find(kMuseLabel) != std::string::npos ? 0 : 1; + int kb = py::lower(b).find(kMuseLabel) != std::string::npos ? 0 : 1; + return ka < kb; + }); + return models; +} + +std::optional> fetch_model_ids( + const std::string& models_url, const std::string& key, + const std::vector>& extra_headers, double timeout) { + HttpRequest req; + req.method = "GET"; + req.url = models_url; + req.headers = extra_headers; + if (!key.empty()) { + req.headers.erase(std::remove_if(req.headers.begin(), req.headers.end(), + [](const auto& kv) { return kv.first == "Authorization"; }), req.headers.end()); + req.headers.emplace_back("Authorization", "Bearer " + key); + } + req.timeout = timeout; + HttpResult res = transport().send(req); + if (!res.ok()) { return std::nullopt; } + json::Value payload; + try { + payload = json::Value::loads(res.body); + } catch (const json::ParseError&) { + return std::nullopt; + } + if (!payload.is_object() || !payload.get("data").is_array()) { return std::nullopt; } + std::vector ids; + for (const auto& row : payload.get("data").items()) { + if (!row.is_object()) { continue; } + std::string id = py::strip(py::str_or_empty(row.get("id"))); + if (!id.empty()) { ids.push_back(py::strip(py::str(row.get("id")))); } + } + if (ids.empty()) { return std::nullopt; } + return ids; +} + +std::optional> fetch_pollinations_free() { + HttpRequest req; + req.method = "GET"; + req.url = kPollinationsListUrl; + req.timeout = kModelListTimeout; + HttpResult res = transport().send(req); + if (!res.ok()) { return std::nullopt; } + json::Value rows; + try { + rows = json::Value::loads(res.body); + } catch (const json::ParseError&) { + return std::nullopt; + } + if (!rows.is_array()) { return std::nullopt; } + std::vector names; + for (const auto& row : rows.items()) { + if (!row.is_object()) { continue; } + const json::Value& tier = row.get("tier"); + if (!(tier.is_string() && tier.str() == "anonymous")) { continue; } + if (!row.get("name").truthy()) { continue; } + json::Value mods = row.get("output_modalities"); + if (!mods.truthy()) { mods = json::Value::array(); mods.push("text"); } + json::Value want = json::Value::array(); want.push("text"); + if (!(mods == want)) { continue; } + names.push_back(py::strip(py::str(row.get("name")))); + } + if (names.empty()) { return std::nullopt; } + return names; +} + +std::string embedded_failure_message(const json::Value& data) { + if (!data.is_object()) { return ""; } + const json::Value& error = data.get("error"); + if (error.is_object()) { + json::Value message = error.get("message").truthy() ? error.get("message") : error.get("type"); + return py::strip(py::str_or_empty(message)); + } + if (error.is_string() && !py::strip(error.str()).empty()) { return py::strip(error.str()); } + const json::Value& type = data.get("type"); + if (type.is_string() && type.str() == "error") { + json::Value m = data.get("message").truthy() ? data.get("message") : json::Value("upstream error"); + return py::strip(py::str(m)); + } + return ""; +} + +int openrouter_free_tier(const Candidate& cand) { + std::string model = py::lower(cand.model); + int muse = model.find(kMuseLabel) != std::string::npos ? 0 : 1; + if (cand.provider == kOpencodeLabel) { return muse; } + if (cand.provider == kDevplaceLabel) { return 2 + muse; } + if (cand.provider == kOpenrouterLabel && is_free_model(model)) { return 4 + muse; } + return 6; +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/models.h b/taicpp/src/llm/models.h new file mode 100644 index 0000000..c9e03bf --- /dev/null +++ b/taicpp/src/llm/models.h @@ -0,0 +1,34 @@ +#pragma once +// Model catalog helpers of tai.py (4710-4730, 7902-7970). +#include "json.h" +#include "ports.h" +#include +#include +#include +namespace detz::llm { + +std::vector split_models(const std::string& spec); +std::vector parse_backends(const std::string& spec); + +bool is_free_model(const std::string& model); +bool is_free_openrouter(const std::string& model); +std::vector prefer_muse(std::vector models); + +// GET models_url (Bearer key, extra headers), the OpenAI-style {"data":[{"id":..}]} id list. +// nullopt when unreachable, malformed or empty. +std::optional> fetch_model_ids( + const std::string& models_url, const std::string& key = "", + const std::vector>& extra_headers = {}, double timeout = 8); +std::optional> fetch_pollinations_free(); +// error.message / error string / type=="error" message of an upstream body, "" when none. +std::string embedded_failure_message(const json::Value& data); + +// One backend candidate (ChatClient.candidates() dict). +struct Candidate { + std::string provider, chat_url, models_url, key, model, profile; + std::string runner; // "" or "box" + std::string engine; +}; +int openrouter_free_tier(const Candidate& cand); + +} // namespace detz::llm diff --git a/taicpp/src/llm/muse.cpp b/taicpp/src/llm/muse.cpp new file mode 100644 index 0000000..035944e --- /dev/null +++ b/taicpp/src/llm/muse.cpp @@ -0,0 +1,345 @@ +#include "muse.h" +#include "cli_backends.h" +#include "consts.h" +#include "proc.h" +#include "py.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; +namespace detz::llm { + +void muse_write_file(const std::string& path, const std::string& text) { + int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) { throw BackendError(std::string("cannot write ") + path + ": " + std::strerror(errno)); } + size_t off = 0; + while (off < text.size()) { + ssize_t w = ::write(fd, text.data() + off, text.size() - off); + if (w < 0) { if (errno == EINTR) { continue; } break; } + off += static_cast(w); + } + fsync(fd); + close(fd); +} + +std::string muse_message_hash(const json::Value& message) { + json::DumpOptions o; + o.sort_keys = true; + o.item_sep = ","; + o.key_sep = ":"; + return py::sha256_hex(message.dumps(o)); +} + +std::string muse_map_path(const std::string& home) { return home + "/" + kMuseSessionsFile; } + +void quarantine(const std::string& path) { + std::error_code ec; + if (fs::exists(path, ec)) { + std::string dest = path + ".corrupt-" + std::to_string(static_cast(py::now())); + fs::rename(path, dest, ec); + } +} + +void muse_save_map(const std::string& home, const json::Value& data) { + json::DumpOptions o; + o.sort_keys = true; + muse_write_file(muse_map_path(home), data.dumps(o) + "\n"); +} + +json::Value muse_load_map(const std::string& home) { + std::string path = muse_map_path(home); + std::ifstream in(path); + if (!in) { + std::error_code ec; + if (!fs::exists(path, ec)) { return json::Value::object(); } + quarantine(path); + return json::Value::object(); + } + std::stringstream ss; + ss << in.rdbuf(); + json::Value data; + try { + data = json::Value::loads(ss.str()); + } catch (const json::ParseError&) { + in.close(); + quarantine(path); + return json::Value::object(); + } + if (!data.is_object()) { + in.close(); + quarantine(path); + return json::Value::object(); + } + double now = py::now(); + bool pruned = false; + auto& members = data.members(); + for (auto it = members.begin(); it != members.end();) { + const json::Value& entry = it->second; + bool drop = false; + if (!entry.is_object() || !entry.get("id").is_string() || !entry.get("hashes").is_array()) { + drop = true; + } else if (py::startswith(it->first, "worker/")) { + double stamp = 0; + try { + stamp = py::to_epoch(py::fromisoformat(py::str_or_empty(entry.get("updated")))); + } catch (const std::invalid_argument&) { + stamp = 0; + } + if (now - stamp > kMuseWorkerTtl) { drop = true; } + } + if (drop) { it = members.erase(it); pruned = true; } else { ++it; } + } + if (pruned) { muse_save_map(home, data); } + return data; +} + +namespace { +std::mutex g_guard; +std::map> g_thread_locks; +} // namespace + +std::function muse_acquire(const std::string& home, const std::string& name) { + std::binary_semaphore* sem; + { + std::lock_guard lock(g_guard); + auto& slot = g_thread_locks[name]; + if (!slot) { slot = std::make_unique(1); } + sem = slot.get(); + } + sem->acquire(); + int fd = -1; + std::error_code ec; + std::string folder = home + "/" + kMuseLocksDir; + fs::create_directories(folder, ec); + if (!ec) { + std::string lock_path = folder + "/" + py::sha256_hex(name) + ".lock"; + fd = ::open(lock_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd >= 0 && flock(fd, LOCK_EX) != 0) { close(fd); fd = -1; } + } + auto released = std::make_shared>(false); + return [sem, fd, released]() { + if (released->exchange(true)) { return; } + if (fd >= 0) { + flock(fd, LOCK_UN); + close(fd); + } + sem->release(); + }; +} + +void muse_transact(const std::string& home, const std::function& func) { + auto release = muse_acquire(home, "map"); + try { + json::Value data = muse_load_map(home); + func(data); + muse_save_map(home, data); + } catch (...) { + release(); + throw; + } + release(); +} + +std::string muse_session_root() { + std::string base = env_get("XDG_DATA_HOME"); + if (base.empty()) { + std::string home = env_get("HOME"); + base = home + "/.local/share"; + } + return base + "/muse/sessions"; +} + +bool muse_session_alive(const std::string& session_id) { + try { + std::error_code ec; + fs::path root = muse_session_root(); + if (!fs::is_directory(root, ec)) { return false; } + for (const auto& a : fs::directory_iterator(root, ec)) { + if (!a.is_directory(ec)) { continue; } + for (const auto& b : fs::directory_iterator(a.path(), ec)) { + if (!b.is_directory(ec)) { continue; } + for (const auto& c : fs::directory_iterator(b.path(), ec)) { + if (!c.is_directory(ec)) { continue; } + if (fs::exists(c.path() / session_id / "session.jsonl", ec)) { return true; } + } + } + } + return false; + } catch (const std::exception&) { + return true; + } +} + +std::optional> muse_session_split(const json::Value& messages, const json::Value& hashes) { + if (hashes.size() > messages.size()) { return std::nullopt; } + for (size_t pos = 0; pos < hashes.size(); ++pos) { + if (!hashes.at(pos).is_string() || muse_message_hash(messages.at(pos)) != hashes.at(pos).str()) { return std::nullopt; } + } + std::vector rest(messages.items().begin() + static_cast(hashes.size()), messages.items().end()); + return rest; +} + +json::Value muse_valid_answer(const std::string& text) { + json::Value data; + try { + data = json::Value::loads(text); + } catch (const json::ParseError&) { + return json::Value(); + } + if (!data.is_object()) { return json::Value(); } + if (!data.get("content").is_string() || !data.get("tool_calls").is_array()) { return json::Value(); } + for (const auto& call : data.get("tool_calls").items()) { + if (!call.is_object() || !call.get("name").is_string() || !call.get("arguments").is_string()) { return json::Value(); } + } + return data; +} + +std::vector muse_argv(const std::string& explicit_model, const std::string& schema_path, + const std::string& prompt_path, const std::optional& session_id) { + std::vector argv = {muse_binary(), "exec", "--json", "--max-model-steps", std::to_string(kMuseMaxSteps), + "--disable-web-tools", "--disable-reminders", "--disable-shell", "--disable-write", + "--output-schema", schema_path, "--prompt-file", prompt_path}; + if (!session_id.has_value()) { + argv.push_back("--no-session-log"); + } else { + argv.push_back("--allow-workspace-switch"); + argv.push_back("--session-id"); + argv.push_back(*session_id); + } + if (!explicit_model.empty()) { + argv.push_back("--model"); + argv.push_back(explicit_model); + } + return argv; +} + +namespace { +struct Event { + std::mutex m; + std::condition_variable cv; + bool set{false}; + void fire() { { std::lock_guard l(m); set = true; } cv.notify_all(); } + bool wait_for(double secs) { + std::unique_lock l(m); + return cv.wait_for(l, std::chrono::duration(secs), [&] { return set; }); + } +}; + +void reap(std::shared_ptr proc, std::shared_ptr stderr_done, std::function on_reap) { + std::thread([proc, stderr_done, on_reap]() { + bool ok = false; + try { + std::string line; + while (proc->read_line(line)) {} + proc->close_pipes(); + if (proc->wait(kMuseReapGrace)) { + ok = proc->returncode() == 0; + } else { + proc->kill(); + proc->wait(10); + } + stderr_done->wait_for(10); + } catch (...) { + } + if (on_reap) { + try { on_reap(ok); } catch (...) {} + } + }).detach(); +} +} // namespace + +json::Value muse_run(const std::vector& argv, const std::string& cwd, const StreamSink* sink, + double timeout, const std::shared_ptr& gate) { + std::string err; + std::unique_ptr owned = Popen::spawn(argv, cwd, err); + if (!owned) { throw BackendError("muse exec failed: " + short_error_text(err)); } + std::shared_ptr proc(std::move(owned)); + if (gate) { gate->armed = true; } + auto stderr_parts = std::make_shared(); + auto stderr_done = std::make_shared(); + auto stderr_mutex = std::make_shared(); + std::thread([proc, stderr_parts, stderr_done, stderr_mutex]() { + std::string line; + while (proc->read_err_line(line)) { + std::lock_guard l(*stderr_mutex); + stderr_parts->append(line); + } + stderr_done->fire(); + }).detach(); + auto finished = std::make_shared(); + auto expired = std::make_shared>(false); + std::thread([proc, finished, expired, timeout]() { + if (!finished->wait_for(timeout)) { + expired->store(true); + proc->kill(); + } + }).detach(); + json::Value answer; + json::Value terminal; + std::string buffer; + try { + std::string raw; + while (proc->read_line(raw)) { + std::string text = py::strip(raw); + if (!py::startswith(text, "{")) { continue; } + json::Value event; + try { + event = json::Value::loads(text); + } catch (const json::ParseError&) { + continue; + } + std::string kind = event.is_object() ? py::str_or_empty(event.get("payload_type")) : ""; + json::Value payload = (event.is_object() && event.get("payload").truthy()) ? event.get("payload") : json::Value::object(); + if (kind == "run.output.delta" && payload.get("text").truthy()) { + std::string piece = py::str(payload.get("text")); + if (sink != nullptr) { (*sink)(piece); } + buffer += piece; + if (answer.is_null()) { + answer = muse_valid_answer(buffer); + if (!answer.is_null()) { break; } + } + } else if (py::startswith(kind, "run.terminal.")) { + terminal = payload; + break; + } + } + } catch (...) { + finished->fire(); + reap(proc, stderr_done, gate ? gate->on_reap : std::function()); + throw; + } + finished->fire(); + reap(proc, stderr_done, gate ? gate->on_reap : std::function()); + if (!terminal.is_null()) { + const json::Value& t = terminal.get("terminal"); + if (!(t.is_string() && t.str() == "completed")) { + std::string what = t.truthy() ? py::str(t) : "failed"; + std::string reason = terminal.get("reason").truthy() ? py::str(terminal.get("reason")) : "unknown"; + throw BackendError("muse " + what + ": " + py::head(reason, 300)); + } + json::Value parsed = muse_valid_answer(py::str_or_empty(terminal.get("text"))); + if (parsed.is_null()) { throw BackendError("muse returned invalid structured answer"); } + return parsed; + } + if (!answer.is_null()) { return answer; } + if (expired->load()) { throw BackendError("muse timed out after " + std::to_string(static_cast(timeout)) + "s"); } + std::string code = proc->reaped() ? std::to_string(proc->returncode()) : "None"; + std::string tailtext; + { + std::lock_guard l(*stderr_mutex); + tailtext = py::tail(py::strip(*stderr_parts), 300); + } + throw BackendError("muse exit " + code + ": " + (tailtext.empty() ? std::string("no terminal event") : tailtext)); +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/muse.h b/taicpp/src/llm/muse.h new file mode 100644 index 0000000..2aae11d --- /dev/null +++ b/taicpp/src/llm/muse.h @@ -0,0 +1,41 @@ +#pragma once +// The muse CLI backend (tai.py 7406-7674): session map file, file+thread +// locks, structured-answer validation and the streaming `muse exec` runner. +#include "helpers.h" +#include "json.h" +#include "ports.h" +#include +#include +#include +#include +#include +#include +namespace detz::llm { + +void muse_write_file(const std::string& path, const std::string& text); // write + fsync +std::string muse_message_hash(const json::Value& message); +std::string muse_map_path(const std::string& home); +json::Value muse_load_map(const std::string& home); +void muse_save_map(const std::string& home, const json::Value& data); +// Returns a release function; callable from any thread, exactly once. +std::function muse_acquire(const std::string& home, const std::string& name); +// Load, let func mutate, save, under the "map" lock. +void muse_transact(const std::string& home, const std::function& func); +std::string muse_session_root(); +bool muse_session_alive(const std::string& session_id); +// nullopt when the hashes are not a prefix of messages. +std::optional> muse_session_split(const json::Value& messages, const json::Value& hashes); +json::Value muse_valid_answer(const std::string& text); // null Value when invalid + +struct MuseGate { + std::function on_reap; + std::atomic armed{false}; +}; +using StreamSink = std::function; +json::Value muse_run(const std::vector& argv, const std::string& cwd, const StreamSink* sink, + double timeout, const std::shared_ptr& gate = nullptr); +std::vector muse_argv(const std::string& explicit_model, const std::string& schema_path, + const std::string& prompt_path, const std::optional& session_id = std::nullopt); +void quarantine(const std::string& path); + +} // namespace detz::llm diff --git a/taicpp/src/llm/ports.h b/taicpp/src/llm/ports.h new file mode 100644 index 0000000..9c0c561 --- /dev/null +++ b/taicpp/src/llm/ports.h @@ -0,0 +1,72 @@ +#pragma once +// Narrow interfaces the LLM layer needs from the rest of the program. +// Everything outside src/llm and src/agent is reached only through these. +#include "proc.h" +#include +#include +#include "json.h" +namespace detz::llm { + +// One TAI_BACKENDS entry (tai.py parse_backends row). +struct BackendSpec { + std::string label, base, key, model, profile; +}; + +// The Config attributes ChatClient / update_models read. Defaults are the ones +// tai.py Config.__init__ gives when no environment variable is set. +struct LlmConfig { + std::string home; + std::string model{"muse"}; + std::string explicit_model; + std::vector backends; + std::string openrouter_key; + std::vector openrouter_models; + double retry_seconds{0.0}; + bool opencode_enabled{true}; + std::vector opencode_models; + std::string devplace_key; + std::string devplace_base{"https://devplace.net/openai/v1"}; + std::vector devplace_models; + bool openrouter_enabled{true}; + bool pollinations_enabled{true}; + std::vector pollinations_models; + bool zen_enabled{false}; + std::vector zen_models; + bool ollama_enabled{true}; + std::vector ollama_models; +}; + +// DebugLog.event(level, area, kind, message, profile=, bot=, meta=) and the +// `LOG.enabled` flag. The store/logging module implements it (logging.md). +class LogSink { + public: + virtual ~LogSink() = default; + virtual bool enabled() const = 0; + virtual void event(const std::string& level, const std::string& area, const std::string& kind, + const std::string& message, const std::string& profile, const std::string& bot, + const json::Value& meta) = 0; +}; +LogSink* log_sink(); +void set_log_sink(LogSink* sink); // nullptr silences logging +// Convenience wrappers used throughout (no-ops without a sink). +bool log_enabled(); +void log_event(const std::string& level, const std::string& area, const std::string& kind, + const std::string& message, const std::string& profile = "", const std::string& bot = "", + const json::Value& meta = json::Value()); + +// Container sandbox (Sandbox / box_* of tai.py), owned by src/shell. +class BoxPort { + public: + virtual ~BoxPort() = default; + virtual std::string container_engine() = 0; // "" when no engine + virtual std::string box_state(const std::string& engine) = 0; // "missing", "running", ... + virtual std::string box_opencode_binary(const std::string& engine) = 0; // "" when absent + virtual ProcResult box_exec(const std::string& engine, const std::vector& argv, double timeout_s) = 0; + // box_opencode_run: returns the joined text events or throws BackendError. + virtual std::string box_opencode_run(const std::string& engine, const std::string& model, + const std::string& prompt, double timeout_s) = 0; +}; +BoxPort* box_port(); +void set_box_port(BoxPort* port); // nullptr: no container engine available + +} // namespace detz::llm diff --git a/taicpp/src/llm/proc.cpp b/taicpp/src/llm/proc.cpp new file mode 100644 index 0000000..2f657d3 --- /dev/null +++ b/taicpp/src/llm/proc.cpp @@ -0,0 +1,269 @@ +#include "proc.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern char** environ; +namespace detz::llm { + +namespace { +double mono() { + using namespace std::chrono; + return duration(steady_clock::now().time_since_epoch()).count(); +} +void set_cloexec(int fd) { fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); } + +struct Spawned { + pid_t pid{-1}; + int in_fd{-1}; + int out_fd{-1}; + int err_fd{-1}; + int error{0}; +}; + +// stdin_mode: 0 inherit, 1 pipe, 2 devnull +Spawned spawn_child(const std::vector& argv, int stdin_mode, const std::string& cwd, + const std::map* env) { + Spawned s; + int in_p[2] = {-1, -1}, out_p[2], err_p[2]; + if (stdin_mode == 1 && pipe(in_p) != 0) { s.error = errno; return s; } + if (pipe(out_p) != 0) { s.error = errno; return s; } + if (pipe(err_p) != 0) { s.error = errno; return s; } + posix_spawn_file_actions_t fa; + posix_spawn_file_actions_init(&fa); + if (stdin_mode == 1) { + posix_spawn_file_actions_adddup2(&fa, in_p[0], 0); + posix_spawn_file_actions_addclose(&fa, in_p[0]); + posix_spawn_file_actions_addclose(&fa, in_p[1]); + } else if (stdin_mode == 2) { + posix_spawn_file_actions_addopen(&fa, 0, "/dev/null", O_RDONLY, 0); + } + posix_spawn_file_actions_adddup2(&fa, out_p[1], 1); + posix_spawn_file_actions_adddup2(&fa, err_p[1], 2); + posix_spawn_file_actions_addclose(&fa, out_p[0]); + posix_spawn_file_actions_addclose(&fa, out_p[1]); + posix_spawn_file_actions_addclose(&fa, err_p[0]); + posix_spawn_file_actions_addclose(&fa, err_p[1]); + if (!cwd.empty()) { posix_spawn_file_actions_addchdir_np(&fa, cwd.c_str()); } + std::vector args; + for (const auto& a : argv) { args.push_back(const_cast(a.c_str())); } + args.push_back(nullptr); + std::vector env_store; + std::vector envp; + if (env != nullptr) { + for (const auto& kv : *env) { env_store.push_back(kv.first + "=" + kv.second); } + for (auto& e : env_store) { envp.push_back(e.data()); } + envp.push_back(nullptr); + } + pid_t pid = -1; + int rc = posix_spawnp(&pid, argv[0].c_str(), &fa, nullptr, args.data(), env != nullptr ? envp.data() : environ); + posix_spawn_file_actions_destroy(&fa); + close(out_p[1]); + close(err_p[1]); + if (stdin_mode == 1) { close(in_p[0]); } + if (rc != 0) { + s.error = rc; + close(out_p[0]); close(err_p[0]); + if (stdin_mode == 1) { close(in_p[1]); } + return s; + } + s.pid = pid; + s.in_fd = stdin_mode == 1 ? in_p[1] : -1; + s.out_fd = out_p[0]; + s.err_fd = err_p[0]; + set_cloexec(s.out_fd); + set_cloexec(s.err_fd); + if (s.in_fd >= 0) { set_cloexec(s.in_fd); } + return s; +} + +int decode_status(int status) { + if (WIFEXITED(status)) { return WEXITSTATUS(status); } + if (WIFSIGNALED(status)) { return -WTERMSIG(status); } + return -1; +} +} // namespace + +ProcResult run_process(const std::vector& argv, const std::optional& input, + double timeout_s, const std::string& cwd, bool devnull_stdin, + const std::map* env) { + ProcResult r; + int mode = input.has_value() ? 1 : (devnull_stdin ? 2 : 0); + Spawned s = spawn_child(argv, mode, cwd, env); + if (s.pid < 0) { + r.spawn_failed = true; + r.spawn_errno = s.error; + r.spawn_error = std::string(std::strerror(s.error)); + return r; + } + signal(SIGPIPE, SIG_IGN); + double deadline = timeout_s > 0 ? mono() + timeout_s : -1; + size_t written = 0; + int in_fd = s.in_fd, out_fd = s.out_fd, err_fd = s.err_fd; + if (in_fd >= 0) { + fcntl(in_fd, F_SETFL, fcntl(in_fd, F_GETFL) | O_NONBLOCK); + if (input->empty()) { close(in_fd); in_fd = -1; } + } + char buf[65536]; + while (out_fd >= 0 || err_fd >= 0 || in_fd >= 0) { + struct pollfd pfds[3]; + int n = 0; + int idx_in = -1, idx_out = -1, idx_err = -1; + if (in_fd >= 0) { idx_in = n; pfds[n++] = {in_fd, POLLOUT, 0}; } + if (out_fd >= 0) { idx_out = n; pfds[n++] = {out_fd, POLLIN, 0}; } + if (err_fd >= 0) { idx_err = n; pfds[n++] = {err_fd, POLLIN, 0}; } + int wait_ms = 200; + if (deadline > 0) { + double left = deadline - mono(); + if (left <= 0) { r.timed_out = true; break; } + wait_ms = static_cast(std::min(left * 1000.0, 200.0)) + 1; + } + int pr = poll(pfds, static_cast(n), wait_ms); + if (pr < 0 && errno != EINTR) { break; } + if (pr <= 0) { continue; } + if (idx_in >= 0 && (pfds[idx_in].revents & (POLLOUT | POLLERR | POLLHUP))) { + ssize_t w = write(in_fd, input->data() + written, input->size() - written); + if (w > 0) { written += static_cast(w); } + if ((w < 0 && errno != EAGAIN && errno != EINTR) || written >= input->size()) { close(in_fd); in_fd = -1; } + } + if (idx_out >= 0 && (pfds[idx_out].revents & (POLLIN | POLLHUP | POLLERR))) { + ssize_t c = read(out_fd, buf, sizeof buf); + if (c > 0) { r.out.append(buf, static_cast(c)); } + else if (c == 0 || (c < 0 && errno != EINTR && errno != EAGAIN)) { close(out_fd); out_fd = -1; } + } + if (idx_err >= 0 && (pfds[idx_err].revents & (POLLIN | POLLHUP | POLLERR))) { + ssize_t c = read(err_fd, buf, sizeof buf); + if (c > 0) { r.err.append(buf, static_cast(c)); } + else if (c == 0 || (c < 0 && errno != EINTR && errno != EAGAIN)) { close(err_fd); err_fd = -1; } + } + } + if (r.timed_out) { kill(s.pid, SIGKILL); } + if (in_fd >= 0) { close(in_fd); } + if (out_fd >= 0) { close(out_fd); } + if (err_fd >= 0) { close(err_fd); } + int status = 0; + while (waitpid(s.pid, &status, 0) < 0 && errno == EINTR) {} + r.returncode = decode_status(status); + return r; +} + +// ------------------------------------------------------------------ Popen +std::unique_ptr Popen::spawn(const std::vector& argv, const std::string& cwd, std::string& error) { + Spawned s = spawn_child(argv, 0, cwd, nullptr); + if (s.pid < 0) { + error = std::strerror(s.error); + return nullptr; + } + std::unique_ptr p(new Popen()); + p->pid_ = s.pid; + p->out_fd_ = s.out_fd; + p->err_fd_ = s.err_fd; + return p; +} +Popen::~Popen() { + close_pipes(); + if (!reaped_ && pid_ > 0) { + kill(); + int status = 0; + while (waitpid(pid_, &status, 0) < 0 && errno == EINTR) {} + } +} +void Popen::close_pipes() { + std::lock_guard lock(guard_); + if (out_fd_ >= 0) { close(out_fd_); out_fd_ = -1; } + if (err_fd_ >= 0) { close(err_fd_); err_fd_ = -1; } +} +bool Popen::read_line_fd(int& fd, std::string& buf, std::string& line) { + for (;;) { + size_t nl = buf.find('\n'); + if (nl != std::string::npos) { + line = buf.substr(0, nl + 1); + buf.erase(0, nl + 1); + return true; + } + if (fd < 0) { + if (buf.empty()) { return false; } + line = buf; + buf.clear(); + return true; + } + char chunk[8192]; + ssize_t c = read(fd, chunk, sizeof chunk); + if (c > 0) { buf.append(chunk, static_cast(c)); continue; } + if (c < 0 && errno == EINTR) { continue; } + std::lock_guard lock(guard_); + if (fd >= 0) { close(fd); fd = -1; } + } +} +bool Popen::read_line(std::string& line) { return read_line_fd(out_fd_, out_buf_, line); } +bool Popen::read_err_line(std::string& line) { return read_line_fd(err_fd_, err_buf_, line); } +void Popen::kill() { + if (pid_ > 0 && !reaped_) { ::kill(pid_, SIGKILL); } +} +bool Popen::wait(double timeout_s) { + if (reaped_) { return true; } + double deadline = mono() + timeout_s; + for (;;) { + int status = 0; + pid_t w = waitpid(pid_, &status, WNOHANG); + if (w == pid_) { + returncode_ = decode_status(status); + reaped_ = true; + return true; + } + if (w < 0 && errno != EINTR) { reaped_ = true; return true; } + if (mono() >= deadline) { return false; } + usleep(20000); + } +} + +std::string oserror_text(const ProcResult& r, const std::string& filename) { + return "[Errno " + std::to_string(r.spawn_errno) + "] " + r.spawn_error + ": '" + filename + "'"; +} +std::string timeout_text(const std::vector& argv, double timeout_s) { + std::string list = "["; + for (size_t i = 0; i < argv.size(); ++i) { + if (i) { list += ", "; } + list += "'" + argv[i] + "'"; + } + list += "]"; + char buf[32]; + std::snprintf(buf, sizeof buf, "%g", timeout_s); + return "Command '" + list + "' timed out after " + buf + " seconds"; +} + +std::string env_get(const std::string& name) { + const char* v = std::getenv(name.c_str()); + return v == nullptr ? std::string() : std::string(v); +} +bool env_has(const std::string& name) { return std::getenv(name.c_str()) != nullptr; } + +std::string which(const std::string& name) { + if (name.empty()) { return ""; } + if (name.find('/') != std::string::npos) { + return access(name.c_str(), X_OK) == 0 ? name : ""; + } + std::string path = env_get("PATH"); + if (path.empty() && !env_has("PATH")) { path = "/usr/local/bin:/usr/bin:/bin"; } + size_t pos = 0; + while (pos <= path.size()) { + size_t e = path.find(':', pos); + std::string dir = path.substr(pos, e == std::string::npos ? std::string::npos : e - pos); + if (dir.empty()) { dir = "."; } + std::string cand = dir + "/" + name; + struct stat st; + if (stat(cand.c_str(), &st) == 0 && S_ISREG(st.st_mode) && access(cand.c_str(), X_OK) == 0) { return cand; } + if (e == std::string::npos) { break; } + pos = e + 1; + } + return ""; +} + +} // namespace detz::llm diff --git a/taicpp/src/llm/proc.h b/taicpp/src/llm/proc.h new file mode 100644 index 0000000..650c10e --- /dev/null +++ b/taicpp/src/llm/proc.h @@ -0,0 +1,67 @@ +#pragma once +// Subprocess layer for the CLI model backends (muse, grok, claude, codex, +// gemini, opencode). Mirrors the subprocess.run / subprocess.Popen calls of +// tai.py. Agent commands (shell tool) do NOT use this: they go through the +// shell module (run_live), see docs/agent.md. +#include +#include +#include +#include +#include +#include +#include +namespace detz::llm { + +struct ProcResult { + int returncode{-1}; + std::string out; + std::string err; + bool timed_out{false}; + bool spawn_failed{false}; + std::string spawn_error; // strerror text + int spawn_errno{0}; +}; + +// subprocess.run(argv, input=..., stdout=PIPE, stderr=PIPE, cwd=..., timeout=...) +// stdin: `input` piped when set, /dev/null when devnull_stdin, inherited otherwise. +ProcResult run_process(const std::vector& argv, const std::optional& input, + double timeout_s, const std::string& cwd = "", bool devnull_stdin = false, + const std::map* env = nullptr); + +// subprocess.Popen(argv, stdout=PIPE, stderr=PIPE, cwd=cwd) with line readers. +class Popen { + public: + ~Popen(); + // Returns nullptr and fills error (OSError text) when the spawn fails. + static std::unique_ptr spawn(const std::vector& argv, const std::string& cwd, std::string& error); + bool read_line(std::string& line); // false at EOF (stdout) + bool read_err_line(std::string& line); // false at EOF (stderr) + void kill(); + // wait with timeout; returns true when the process exited (code in returncode()). + bool wait(double timeout_s); + int returncode() const { return returncode_; } + bool reaped() const { return reaped_; } + void close_pipes(); + private: + Popen() = default; + bool read_line_fd(int& fd, std::string& buf, std::string& line); + int pid_{-1}; + int out_fd_{-1}; + int err_fd_{-1}; + std::string out_buf_, err_buf_; + std::atomic returncode_{-1}; + std::atomic reaped_{false}; + std::mutex guard_; +}; + +// str(OSError) of a failed spawn: "[Errno 2] No such file or directory: 'name'". +std::string oserror_text(const ProcResult& r, const std::string& filename); +// str(subprocess.TimeoutExpired) for argv. +std::string timeout_text(const std::vector& argv, double timeout_s); +// shutil.which +std::string which(const std::string& name); +// os.environ.get(name) +std::string env_get(const std::string& name); +bool env_has(const std::string& name); + +} // namespace detz::llm diff --git a/taicpp/src/llm/py.cpp b/taicpp/src/llm/py.cpp new file mode 100644 index 0000000..5a3272b --- /dev/null +++ b/taicpp/src/llm/py.cpp @@ -0,0 +1,792 @@ +#include "py.h" +#include "unicode_data.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace detz::py { + +// ---------------------------------------------------------------- utf-8 +std::vector decode(const std::string& s) { + std::vector out; + out.reserve(s.size()); + size_t i = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + if (c < 0x80) { out.push_back(c); ++i; continue; } + int need = 0; uint32_t cp = 0; + if ((c & 0xE0) == 0xC0) { need = 1; cp = c & 0x1F; } + else if ((c & 0xF0) == 0xE0) { need = 2; cp = c & 0x0F; } + else if ((c & 0xF8) == 0xF0) { need = 3; cp = c & 0x07; } + else { out.push_back(0xFFFD); ++i; continue; } + if (i + static_cast(need) >= s.size()) { out.push_back(0xFFFD); ++i; continue; } + bool ok = true; + for (int k = 1; k <= need; ++k) { + unsigned char d = static_cast(s[i + static_cast(k)]); + if ((d & 0xC0) != 0x80) { ok = false; break; } + cp = (cp << 6) | (d & 0x3F); + } + if (!ok) { out.push_back(0xFFFD); ++i; continue; } + out.push_back(cp); + i += static_cast(need) + 1; + } + return out; +} +std::string utf8_replace(const std::string& s) { + std::string out; + out.reserve(s.size()); + size_t i = 0, n = s.size(); + auto cont = [&](size_t k, unsigned lo, unsigned hi) { + return k < n && static_cast(s[k]) >= lo && static_cast(s[k]) <= hi; + }; + while (i < n) { + unsigned char c = static_cast(s[i]); + if (c < 0x80) { out += static_cast(c); ++i; continue; } + if (c >= 0xC2 && c <= 0xDF) { + if (cont(i + 1, 0x80, 0xBF)) { out.append(s, i, 2); i += 2; } else { out += "\xEF\xBF\xBD"; ++i; } + continue; + } + if (c >= 0xE0 && c <= 0xEF) { + unsigned lo = c == 0xE0 ? 0xA0 : 0x80, hi = c == 0xED ? 0x9F : 0xBF; + if (cont(i + 1, lo, hi)) { + if (cont(i + 2, 0x80, 0xBF)) { out.append(s, i, 3); i += 3; } + else { out += "\xEF\xBF\xBD"; i += 2; } + } else { out += "\xEF\xBF\xBD"; ++i; } + continue; + } + if (c >= 0xF0 && c <= 0xF4) { + unsigned lo = c == 0xF0 ? 0x90 : 0x80, hi = c == 0xF4 ? 0x8F : 0xBF; + if (cont(i + 1, lo, hi)) { + if (cont(i + 2, 0x80, 0xBF)) { + if (cont(i + 3, 0x80, 0xBF)) { out.append(s, i, 4); i += 4; } + else { out += "\xEF\xBF\xBD"; i += 3; } + } else { out += "\xEF\xBF\xBD"; i += 2; } + } else { out += "\xEF\xBF\xBD"; ++i; } + continue; + } + out += "\xEF\xBF\xBD"; + ++i; + } + return out; +} +std::string universal_newlines(const std::string& text) { + if (text.find('\r') == std::string::npos) { return text; } + std::string out; + out.reserve(text.size()); + for (size_t i = 0; i < text.size(); ++i) { + if (text[i] == '\r') { + out += '\n'; + if (i + 1 < text.size() && text[i + 1] == '\n') { ++i; } + } else { + out += text[i]; + } + } + return out; +} +void put_utf8(std::string& out, uint32_t cp) { + if (cp < 0x80) { out += static_cast(cp); } + else if (cp < 0x800) { out += static_cast(0xC0 | (cp >> 6)); out += static_cast(0x80 | (cp & 0x3F)); } + else if (cp < 0x10000) { out += static_cast(0xE0 | (cp >> 12)); out += static_cast(0x80 | ((cp >> 6) & 0x3F)); out += static_cast(0x80 | (cp & 0x3F)); } + else { out += static_cast(0xF0 | (cp >> 18)); out += static_cast(0x80 | ((cp >> 12) & 0x3F)); out += static_cast(0x80 | ((cp >> 6) & 0x3F)); out += static_cast(0x80 | (cp & 0x3F)); } +} +std::string encode(const std::vector& cps) { + std::string out; + for (uint32_t cp : cps) { put_utf8(out, cp); } + return out; +} + +static bool is_ascii(const std::string& s) { + for (unsigned char c : s) { if (c >= 0x80) { return false; } } + return true; +} + +size_t len(const std::string& s) { + size_t n = 0; + for (unsigned char c : s) { if ((c & 0xC0) != 0x80) { ++n; } } + return n; +} + +std::string slice(const std::string& s, std::optional start, std::optional stop) { + if (is_ascii(s)) { + long n = static_cast(s.size()); + long a = start.value_or(0), b = stop.value_or(n); + if (a < 0) { a += n; if (a < 0) { a = 0; } } + if (b < 0) { b += n; if (b < 0) { b = 0; } } + if (a > n) { a = n; } + if (b > n) { b = n; } + if (b <= a) { return ""; } + return s.substr(static_cast(a), static_cast(b - a)); + } + auto cps = decode(s); + long n = static_cast(cps.size()); + long a = start.value_or(0), b = stop.value_or(n); + if (a < 0) { a += n; if (a < 0) { a = 0; } } + if (b < 0) { b += n; if (b < 0) { b = 0; } } + if (a > n) { a = n; } + if (b > n) { b = n; } + if (b <= a) { return ""; } + return encode(std::vector(cps.begin() + a, cps.begin() + b)); +} + +static bool in_ranges(const std::vector& r, uint32_t cp); +bool is_space_cp(uint32_t cp) { + if (cp < 0x80) { return cp == ' ' || (cp >= 9 && cp <= 13) || (cp >= 0x1c && cp <= 0x1f); } + return in_ranges(ucd::SpaceRanges(), cp); +} + +std::string strip(const std::string& s) { return rstrip(lstrip(s)); } +std::string lstrip(const std::string& s) { + if (is_ascii(s)) { + size_t i = 0; + while (i < s.size() && is_space_cp(static_cast(s[i]))) { ++i; } + return s.substr(i); + } + auto cps = decode(s); + size_t i = 0; + while (i < cps.size() && is_space_cp(cps[i])) { ++i; } + return encode(std::vector(cps.begin() + static_cast(i), cps.end())); +} +std::string rstrip(const std::string& s) { + if (is_ascii(s)) { + size_t e = s.size(); + while (e > 0 && is_space_cp(static_cast(s[e - 1]))) { --e; } + return s.substr(0, e); + } + auto cps = decode(s); + size_t e = cps.size(); + while (e > 0 && is_space_cp(cps[e - 1])) { --e; } + return encode(std::vector(cps.begin(), cps.begin() + static_cast(e))); +} +std::string lstrip_chars(const std::string& s, const std::string& chars) { + auto set = decode(chars); + auto cps = decode(s); + size_t i = 0; + while (i < cps.size() && std::find(set.begin(), set.end(), cps[i]) != set.end()) { ++i; } + return encode(std::vector(cps.begin() + static_cast(i), cps.end())); +} +std::string rstrip_chars(const std::string& s, const std::string& chars) { + auto set = decode(chars); + auto cps = decode(s); + size_t e = cps.size(); + while (e > 0 && std::find(set.begin(), set.end(), cps[e - 1]) != set.end()) { --e; } + return encode(std::vector(cps.begin(), cps.begin() + static_cast(e))); +} +std::string strip_chars(const std::string& s, const std::string& chars) { + return rstrip_chars(lstrip_chars(s, chars), chars); +} + +static bool in_ranges(const std::vector& r, uint32_t cp) { + size_t lo = 0, hi = r.size(); + while (lo < hi) { + size_t mid = (lo + hi) / 2; + if (cp < r[mid].first) { hi = mid; } + else if (cp > r[mid].last) { lo = mid + 1; } + else { return true; } + } + return false; +} +static const char* map_lookup(const std::vector& t, uint32_t cp) { + size_t lo = 0, hi = t.size(); + while (lo < hi) { + size_t mid = (lo + hi) / 2; + if (cp < t[mid].cp) { hi = mid; } + else if (cp > t[mid].cp) { lo = mid + 1; } + else { return t[mid].text; } + } + return nullptr; +} +// Final_Sigma context of CPython's handle_capital_sigma. +static bool final_sigma(const std::vector& cps, size_t i) { + const auto& ign = ucd::SigmaIgnorableRanges(); + const auto& cased = ucd::SigmaCasedRanges(); + long j = static_cast(i) - 1; + while (j >= 0 && in_ranges(ign, cps[static_cast(j)])) { --j; } + bool final_s = j >= 0 && in_ranges(cased, cps[static_cast(j)]); + if (final_s) { + size_t k = i + 1; + while (k < cps.size() && in_ranges(ign, cps[k])) { ++k; } + final_s = k == cps.size() || !in_ranges(cased, cps[k]); + } + return final_s; +} +std::string lower(const std::string& s) { + if (is_ascii(s)) { + std::string o = s; + for (auto& ch : o) { if (ch >= 'A' && ch <= 'Z') { ch = static_cast(ch + 32); } } + return o; + } + auto cps = decode(s); + std::string out; + for (size_t i = 0; i < cps.size(); ++i) { + uint32_t c = cps[i]; + if (c == 0x3A3) { put_utf8(out, final_sigma(cps, i) ? 0x3C2 : 0x3C3); continue; } + if (c < 0x80) { put_utf8(out, (c >= 'A' && c <= 'Z') ? c + 32 : c); continue; } + const char* m = map_lookup(ucd::LowerMap(), c); + if (m != nullptr) { out += m; } else { put_utf8(out, c); } + } + return out; +} +std::string upper(const std::string& s) { + std::string out; + for (uint32_t c : decode(s)) { + if (c < 0x80) { put_utf8(out, (c >= 'a' && c <= 'z') ? c - 32 : c); continue; } + const char* m = map_lookup(ucd::UpperMap(), c); + if (m != nullptr) { out += m; } else { put_utf8(out, c); } + } + return out; +} + +std::vector split_ws(const std::string& s) { + std::vector out; + auto cps = decode(s); + std::vector cur; + for (uint32_t c : cps) { + if (is_space_cp(c)) { + if (!cur.empty()) { out.push_back(encode(cur)); cur.clear(); } + } else { + cur.push_back(c); + } + } + if (!cur.empty()) { out.push_back(encode(cur)); } + return out; +} +std::vector split(const std::string& s, const std::string& sep, long maxsplit) { + std::vector out; + size_t pos = 0; + long done = 0; + for (;;) { + if (maxsplit >= 0 && done >= maxsplit) { break; } + size_t f = s.find(sep, pos); + if (f == std::string::npos) { break; } + out.push_back(s.substr(pos, f - pos)); + pos = f + sep.size(); + ++done; + } + out.push_back(s.substr(pos)); + return out; +} +std::vector splitlines(const std::string& s) { + std::vector out; + size_t i = 0, start = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + size_t brk = 0; + if (c == '\n' || c == '\v' || c == '\f' || c == 0x1c || c == 0x1d || c == 0x1e) { brk = 1; } + else if (c == '\r') { brk = (i + 1 < s.size() && s[i + 1] == '\n') ? 2 : 1; } + else if (c == 0xC2 && i + 1 < s.size() && static_cast(s[i + 1]) == 0x85) { brk = 2; } + else if (c == 0xE2 && i + 2 < s.size() && static_cast(s[i + 1]) == 0x80 && + (static_cast(s[i + 2]) == 0xA8 || static_cast(s[i + 2]) == 0xA9)) { brk = 3; } + if (brk) { + out.push_back(s.substr(start, i - start)); + i += brk; + start = i; + } else { + ++i; + } + } + if (start < s.size()) { out.push_back(s.substr(start)); } + return out; +} +std::string join(const std::vector& parts, const std::string& sep) { + std::string out; + for (size_t i = 0; i < parts.size(); ++i) { + if (i) { out += sep; } + out += parts[i]; + } + return out; +} +bool startswith(const std::string& s, const std::string& p) { return s.size() >= p.size() && s.compare(0, p.size(), p) == 0; } +bool endswith(const std::string& s, const std::string& p) { return s.size() >= p.size() && s.compare(s.size() - p.size(), p.size(), p) == 0; } +std::string replace(const std::string& s, const std::string& from, const std::string& to) { + if (from.empty()) { + // Python inserts `to` between every code point and at both ends. + auto cps = decode(s); + std::string out = to; + for (uint32_t c : cps) { put_utf8(out, c); out += to; } + return out; + } + std::string out; + size_t pos = 0; + for (;;) { + size_t f = s.find(from, pos); + if (f == std::string::npos) { out += s.substr(pos); break; } + out += s.substr(pos, f - pos); + out += to; + pos = f + from.size(); + } + return out; +} +long count(const std::string& s, const std::string& sub) { + if (sub.empty()) { return static_cast(len(s)) + 1; } + long n = 0; + size_t pos = 0; + for (;;) { + size_t f = s.find(sub, pos); + if (f == std::string::npos) { break; } + ++n; + pos = f + sub.size(); + } + return n; +} +bool isdigit_ascii(const std::string& s) { + if (s.empty()) { return false; } + for (unsigned char c : s) { if (c < '0' || c > '9') { return false; } } + return true; +} +std::string ljust_cut(const std::string& s, size_t n) { return slice(s, 0, static_cast(n)); } + +// ---------------------------------------------------------------- repr / str +static bool printable_cp(uint32_t c) { + if (c < 0x20 || c == 0x7f) { return false; } + if (c >= 0x80 && c < 0xA1) { return false; } + if (c == 0xAD) { return false; } + if (c == 0xA0 || c == 0x1680 || (c >= 0x2000 && c <= 0x200F) || (c >= 0x2028 && c <= 0x202F) || + c == 0x205F || (c >= 0x2060 && c <= 0x206F) || c == 0x3000 || c == 0xFEFF) { return false; } + if (c >= 0xD800 && c <= 0xDFFF) { return false; } + if (c >= 0xE000 && c <= 0xF8FF) { return false; } + return true; +} +std::string repr_str(const std::string& s) { + bool has_single = s.find('\'') != std::string::npos; + bool has_double = s.find('"') != std::string::npos; + char q = (has_single && !has_double) ? '"' : '\''; + std::string out(1, q); + for (uint32_t c : decode(s)) { + if (c == static_cast(q) || c == '\\') { out += '\\'; out += static_cast(c); } + else if (c == '\n') { out += "\\n"; } + else if (c == '\r') { out += "\\r"; } + else if (c == '\t') { out += "\\t"; } + else if (!printable_cp(c)) { + char buf[16]; + if (c < 0x100) { std::snprintf(buf, sizeof buf, "\\x%02x", c); } + else if (c < 0x10000) { std::snprintf(buf, sizeof buf, "\\u%04x", c); } + else { std::snprintf(buf, sizeof buf, "\\U%08x", c); } + out += buf; + } else { + put_utf8(out, c); + } + } + out += q; + return out; +} +std::string repr(const json::Value& v) { + switch (v.type()) { + case json::Value::Type::Null: return "None"; + case json::Value::Type::Bool: return v.as_bool() ? "True" : "False"; + case json::Value::Type::Int: return v.int_text(); + case json::Value::Type::Double: { + double d = v.as_double(); + if (std::isnan(d)) { return "nan"; } + if (std::isinf(d)) { return d < 0 ? "-inf" : "inf"; } + return json::float_repr(d); + } + case json::Value::Type::String: return repr_str(v.str()); + case json::Value::Type::Array: { + std::string out = "["; + bool first = true; + for (const auto& x : v.items()) { if (!first) { out += ", "; } first = false; out += repr(x); } + return out + "]"; + } + case json::Value::Type::Object: { + std::string out = "{"; + bool first = true; + for (const auto& m : v.members()) { + if (!first) { out += ", "; } + first = false; + out += repr_str(m.first) + ": " + repr(m.second); + } + return out + "}"; + } + } + return ""; +} +std::string str(const json::Value& v) { + if (v.is_string()) { return v.str(); } + return repr(v); +} +std::string str_or_empty(const json::Value& v) { + if (!v.truthy()) { return ""; } + return str(v); +} +const json::Value& or_value(const json::Value& v, const json::Value& dflt) { return v.truthy() ? v : dflt; } + +std::optional parse_int(const std::string& raw) { + std::string s = strip(raw); + if (s.empty()) { return std::nullopt; } + std::vector cps = decode(s); + size_t i = 0; + bool neg = false; + if (cps[i] == '+' || cps[i] == '-') { neg = cps[i] == '-'; ++i; } + if (i >= cps.size()) { return std::nullopt; } + int64_t v = 0; + bool saturated = false; + bool prev_digit = false; + auto digit_of = [](uint32_t c) -> int { + if (c >= '0' && c <= '9') { return static_cast(c - '0'); } + if (c < 0x80) { return -1; } + for (const auto& r : ucd::DecimalRanges()) { + if (c >= r.first && c <= r.last) { return static_cast((c - r.first) % 10); } + } + return -1; + }; + for (; i < cps.size(); ++i) { + uint32_t c = cps[i]; + if (c == '_') { + if (!prev_digit || i + 1 >= cps.size() || digit_of(cps[i + 1]) < 0) { return std::nullopt; } + prev_digit = false; + continue; + } + int d = digit_of(c); + if (d < 0) { return std::nullopt; } + if (!saturated) { + if (v > (INT64_MAX - d) / 10) { saturated = true; } + else { v = v * 10 + d; } + } + prev_digit = true; + } + if (saturated) { return neg ? INT64_MIN : INT64_MAX; } // beyond int64: clamp (Python has bigints) + return neg ? -v : v; +} +std::optional to_int(const json::Value& v) { + switch (v.type()) { + case json::Value::Type::Int: return v.as_int(); + case json::Value::Type::Bool: return v.as_bool() ? 1 : 0; + case json::Value::Type::Double: { + double d = v.as_double(); + if (std::isnan(d) || std::isinf(d)) { return std::nullopt; } + if (d >= 9.2233720368547758e18) { return INT64_MAX; } // beyond int64: clamp (Python has bigints) + if (d <= -9.2233720368547758e18) { return INT64_MIN; } + return static_cast(d); + } + case json::Value::Type::String: return parse_int(v.str()); + default: return std::nullopt; + } +} +std::optional to_float(const json::Value& v) { + switch (v.type()) { + case json::Value::Type::Int: return static_cast(v.as_int()); + case json::Value::Type::Bool: return v.as_bool() ? 1.0 : 0.0; + case json::Value::Type::Double: return v.as_double(); + case json::Value::Type::String: { + std::string s = strip(v.str()); + if (s.empty()) { return std::nullopt; } + std::string l = lower(s); + if (l == "nan" || l == "+nan" || l == "-nan") { return std::nan(""); } + if (l == "inf" || l == "+inf" || l == "infinity" || l == "+infinity") { return INFINITY; } + if (l == "-inf" || l == "-infinity") { return -INFINITY; } + std::string t; + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == '_') { + if (i == 0 || i + 1 >= s.size() || !std::isdigit(static_cast(s[i - 1])) || + !std::isdigit(static_cast(s[i + 1]))) { return std::nullopt; } + continue; + } + t += s[i]; + } + char* end = nullptr; + double d = std::strtod(t.c_str(), &end); + if (end == t.c_str() || *end != '\0') { return std::nullopt; } + // reject hex floats, which strtod accepts and Python does not + if (t.find('x') != std::string::npos || t.find('X') != std::string::npos) { return std::nullopt; } + return d; + } + default: return std::nullopt; + } +} + +// ---------------------------------------------------------------- time +double now() { + using namespace std::chrono; + return duration(system_clock::now().time_since_epoch()).count(); +} +static void civil_from_epoch(int64_t secs, int& y, int& m, int& d, int& H, int& M, int& S) { + int64_t days = secs >= 0 ? secs / 86400 : -((-secs + 86399) / 86400); + int64_t rem = secs - days * 86400; + H = static_cast(rem / 3600); M = static_cast((rem % 3600) / 60); S = static_cast(rem % 60); + days += 719468; + int64_t era = (days >= 0 ? days : days - 146096) / 146097; + int64_t doe = days - era * 146097; + int64_t yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + int64_t yy = yoe + era * 400; + int64_t doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + int64_t mp = (5 * doy + 2) / 153; + d = static_cast(doy - (153 * mp + 2) / 5 + 1); + m = static_cast(mp < 10 ? mp + 3 : mp - 9); + y = static_cast(yy + (m <= 2)); +} +static int64_t days_from_civil(int y, int m, int d) { + y -= m <= 2; + int64_t era = (y >= 0 ? y : y - 399) / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} +std::string utc_iso(double epoch, bool with_micro) { + double fl = std::floor(epoch); + int64_t secs = static_cast(fl); + int micro = static_cast(std::floor((epoch - fl) * 1e6)); + if (micro >= 1000000) { micro = 999999; } + int y, m, d, H, M, S; + civil_from_epoch(secs, y, m, d, H, M, S); + char buf[64]; + if (with_micro && micro != 0) { + std::snprintf(buf, sizeof buf, "%04d-%02d-%02dT%02d:%02d:%02d.%06d+00:00", y, m, d, H, M, S, micro); + } else { + std::snprintf(buf, sizeof buf, "%04d-%02d-%02dT%02d:%02d:%02d+00:00", y, m, d, H, M, S); + } + return buf; +} +std::string now_iso() { return utc_iso(now(), false); } +std::string utc_now_iso() { return utc_iso(now(), true); } + +static bool digits_at(const std::string& s, size_t pos, size_t n) { + if (pos + n > s.size()) { return false; } + for (size_t i = 0; i < n; ++i) { if (s[pos + i] < '0' || s[pos + i] > '9') { return false; } } + return true; +} +static int num_at(const std::string& s, size_t pos, size_t n) { return std::stoi(s.substr(pos, n)); } +static bool valid_date(int y, int m, int d) { + if (y < 1 || m < 1 || m > 12 || d < 1) { return false; } + static const int dm[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + int max = dm[m - 1]; + if (m == 2 && ((y % 4 == 0 && y % 100 != 0) || y % 400 == 0)) { max = 29; } + return d <= max; +} +DateTime fromisoformat(const std::string& text) { + auto bad = []() { return std::invalid_argument("invalid datetime, use ISO like 2026-10-08T09:00"); }; + std::string raw = strip(text); + if (raw.empty()) { throw std::invalid_argument("empty datetime"); } + if (raw.back() == 'Z' || raw.back() == 'z') { raw = raw.substr(0, raw.size() - 1) + "+00:00"; } + DateTime dt; + size_t p = 0; + // date part: YYYY-MM-DD or YYYYMMDD + if (digits_at(raw, 0, 4) && raw.size() > 4 && raw[4] == '-' && digits_at(raw, 5, 2) && raw.size() > 7 && raw[7] == '-' && digits_at(raw, 8, 2)) { + dt.year = num_at(raw, 0, 4); dt.month = num_at(raw, 5, 2); dt.day = num_at(raw, 8, 2); p = 10; + } else if (digits_at(raw, 0, 8)) { + dt.year = num_at(raw, 0, 4); dt.month = num_at(raw, 4, 2); dt.day = num_at(raw, 6, 2); p = 8; + } else { + throw bad(); + } + if (!valid_date(dt.year, dt.month, dt.day)) { throw bad(); } + if (p < raw.size()) { + char sep = raw[p]; + if (sep != 'T' && sep != 't' && sep != ' ') { throw bad(); } + ++p; + if (!digits_at(raw, p, 2)) { throw bad(); } + dt.hour = num_at(raw, p, 2); p += 2; + bool colon = p < raw.size() && raw[p] == ':'; + if (colon || digits_at(raw, p, 2)) { + if (colon) { ++p; } + if (!digits_at(raw, p, 2)) { throw bad(); } + dt.minute = num_at(raw, p, 2); p += 2; + bool c2 = p < raw.size() && raw[p] == ':'; + if (colon ? c2 : digits_at(raw, p, 2)) { + if (colon) { ++p; } + if (!digits_at(raw, p, 2)) { throw bad(); } + dt.second = num_at(raw, p, 2); p += 2; + if (p < raw.size() && (raw[p] == '.' || raw[p] == ',')) { + ++p; + size_t s = p; + while (p < raw.size() && raw[p] >= '0' && raw[p] <= '9') { ++p; } + if (p == s) { throw bad(); } + std::string frac = raw.substr(s, std::min(6, p - s)); + while (frac.size() < 6) { frac += '0'; } + dt.micro = std::stoi(frac); + } + } + } + if (dt.hour > 23 || dt.minute > 59 || dt.second > 59) { throw bad(); } + if (p < raw.size()) { + char sign = raw[p]; + if (sign != '+' && sign != '-') { throw bad(); } + ++p; + if (!digits_at(raw, p, 2)) { throw bad(); } + int oh = num_at(raw, p, 2); p += 2; + int om = 0, os = 0; + if (p < raw.size() && raw[p] == ':') { ++p; } + if (digits_at(raw, p, 2)) { + om = num_at(raw, p, 2); p += 2; + if (p < raw.size() && raw[p] == ':') { ++p; } + if (digits_at(raw, p, 2)) { os = num_at(raw, p, 2); p += 2; } + } + if (p != raw.size()) { throw bad(); } + if (oh > 23 || om > 59 || os > 59) { throw bad(); } + dt.aware = true; + dt.offset_seconds = (sign == '-' ? -1 : 1) * (oh * 3600 + om * 60 + os); + } + } + return dt; +} +static int64_t local_as_utc_seconds(int64_t u) { + time_t t = static_cast(u); + std::tm tmv{}; + localtime_r(&t, &tmv); + return days_from_civil(tmv.tm_year + 1900, tmv.tm_mon + 1, tmv.tm_mday) * 86400 + tmv.tm_hour * 3600 + tmv.tm_min * 60 + tmv.tm_sec; +} +// datetime(...).timestamp() / astimezone() for a naive value: t is the naive wall clock as seconds +// since 1970 "as if UTC"; the result is the instant, resolving folds and gaps like CPython (fold=0). +static int64_t naive_local_to_epoch(int64_t t) { + const int64_t max_fold = 24 * 3600; + int64_t a = local_as_utc_seconds(t) - t; + int64_t u1 = t - a; + int64_t t1 = local_as_utc_seconds(u1); + int64_t b; + if (t1 == t) { + int64_t u2 = u1 - max_fold; + b = local_as_utc_seconds(u2) - u2; + if (a == b) { return u1; } + } else { + b = t1 - u1; + } + int64_t u2 = t - b; + int64_t t2 = local_as_utc_seconds(u2); + if (t2 == t) { return u2; } + if (t1 == t) { return u1; } + return std::max(u1, u2); +} +double to_epoch(const DateTime& dt) { + int64_t days = days_from_civil(dt.year, dt.month, dt.day); + int64_t secs = days * 86400 + dt.hour * 3600 + dt.minute * 60 + dt.second; + double frac = dt.micro / 1e6; + if (dt.aware) { return static_cast(secs - dt.offset_seconds) + frac; } + // naive: CPython's local_to_seconds with fold=0 + return static_cast(naive_local_to_epoch(secs)) + frac; +} +std::string isoformat(const DateTime& dt) { + char buf[96]; + int n = std::snprintf(buf, sizeof buf, "%04d-%02d-%02dT%02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + std::string out(buf, static_cast(n)); + if (dt.micro) { std::snprintf(buf, sizeof buf, ".%06d", dt.micro); out += buf; } + if (dt.aware) { + int off = dt.offset_seconds; + char sign = off < 0 ? '-' : '+'; + off = std::abs(off); + std::snprintf(buf, sizeof buf, "%c%02d:%02d", sign, off / 3600, (off % 3600) / 60); + out += buf; + if (off % 60) { std::snprintf(buf, sizeof buf, ":%02d", off % 60); out += buf; } + } + return out; +} +std::string to_utc_iso(const DateTime& dt) { + double e = to_epoch(dt); + double fl = std::floor(e); + int64_t secs = static_cast(fl); + int y, m, d, H, M, S; + civil_from_epoch(secs, y, m, d, H, M, S); + DateTime u; + u.year = y; u.month = m; u.day = d; u.hour = H; u.minute = M; u.second = S; u.micro = dt.micro; + u.aware = true; u.offset_seconds = 0; + return isoformat(u); +} +std::string local_minutes(double epoch) { + time_t t = static_cast(std::floor(epoch)); + std::tm tmv{}; + localtime_r(&t, &tmv); + char buf[64]; + std::strftime(buf, sizeof buf, "%Y-%m-%d %H:%M", &tmv); + return buf; +} + +// ---------------------------------------------------------------- hashes +static uint32_t rol(uint32_t v, int n) { return (v << n) | (v >> (32 - n)); } +static uint32_t ror(uint32_t v, int n) { return (v >> n) | (v << (32 - n)); } +static std::string to_hex(const uint8_t* data, size_t n) { + static const char* h = "0123456789abcdef"; + std::string out; + for (size_t i = 0; i < n; ++i) { out += h[data[i] >> 4]; out += h[data[i] & 15]; } + return out; +} +std::string sha1_hex(const std::string& data) { + uint32_t h[5] = {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; + std::string msg = data; + uint64_t bits = static_cast(data.size()) * 8; + msg += static_cast(0x80); + while (msg.size() % 64 != 56) { msg += '\0'; } + for (int i = 7; i >= 0; --i) { msg += static_cast((bits >> (i * 8)) & 0xff); } + for (size_t off = 0; off < msg.size(); off += 64) { + uint32_t w[80]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(static_cast(msg[off + i * 4])) << 24) | + (static_cast(static_cast(msg[off + i * 4 + 1])) << 16) | + (static_cast(static_cast(msg[off + i * 4 + 2])) << 8) | + static_cast(static_cast(msg[off + i * 4 + 3])); + } + for (int i = 16; i < 80; ++i) { w[i] = rol(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4]; + for (int i = 0; i < 80; ++i) { + uint32_t f, k; + if (i < 20) { f = (b & c) | (~b & d); k = 0x5A827999; } + else if (i < 40) { f = b ^ c ^ d; k = 0x6ED9EBA1; } + else if (i < 60) { f = (b & c) | (b & d) | (c & d); k = 0x8F1BBCDC; } + else { f = b ^ c ^ d; k = 0xCA62C1D6; } + uint32_t t = rol(a, 5) + f + e + k + w[i]; + e = d; d = c; c = rol(b, 30); b = a; a = t; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; + } + uint8_t out[20]; + for (int i = 0; i < 5; ++i) { for (int j = 0; j < 4; ++j) { out[i * 4 + j] = static_cast(h[i] >> (24 - 8 * j)); } } + return to_hex(out, 20); +} +std::string sha256_hex(const std::string& data) { + static const uint32_t K[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2}; + uint32_t h[8] = {0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19}; + std::string msg = data; + uint64_t bits = static_cast(data.size()) * 8; + msg += static_cast(0x80); + while (msg.size() % 64 != 56) { msg += '\0'; } + for (int i = 7; i >= 0; --i) { msg += static_cast((bits >> (i * 8)) & 0xff); } + for (size_t off = 0; off < msg.size(); off += 64) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(static_cast(msg[off + i * 4])) << 24) | + (static_cast(static_cast(msg[off + i * 4 + 1])) << 16) | + (static_cast(static_cast(msg[off + i * 4 + 2])) << 8) | + static_cast(static_cast(msg[off + i * 4 + 3])); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = ror(w[i - 15], 7) ^ ror(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ror(w[i - 2], 17) ^ ror(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + uint32_t S1 = ror(e, 6) ^ ror(e, 11) ^ ror(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + S1 + ch + K[i] + w[i]; + uint32_t S0 = ror(a, 2) ^ ror(a, 13) ^ ror(a, 22); + uint32_t mj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + mj; + hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh; + } + uint8_t out[32]; + for (int i = 0; i < 8; ++i) { for (int j = 0; j < 4; ++j) { out[i * 4 + j] = static_cast(h[i] >> (24 - 8 * j)); } } + return to_hex(out, 32); +} +std::string uuid4_hex() { + static std::mt19937_64 rng{std::random_device{}()}; + uint64_t a = rng(), b = rng(); + a = (a & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + b = (b & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + char buf[40]; + std::snprintf(buf, sizeof buf, "%016llx%016llx", static_cast(a), static_cast(b)); + return buf; +} +std::string uuid4() { + std::string h = uuid4_hex(); + return h.substr(0, 8) + "-" + h.substr(8, 4) + "-" + h.substr(12, 4) + "-" + h.substr(16, 4) + "-" + h.substr(20); +} + +} // namespace detz::py diff --git a/taicpp/src/llm/py.h b/taicpp/src/llm/py.h new file mode 100644 index 0000000..d152b04 --- /dev/null +++ b/taicpp/src/llm/py.h @@ -0,0 +1,86 @@ +#pragma once +// Python semantics used all over tai.py, kept in one place so a literal port +// stays literal: str lengths and slices count code points (not bytes), +// str()/repr()/int() of JSON values, strip/split/lower, datetime isoformat. +#include "json.h" +#include +#include +#include +#include +namespace detz::py { + +// ---- str (code point semantics) ---- +size_t len(const std::string& s); +// s[start:stop] with Python rules (negative indices, clamping). +std::string slice(const std::string& s, std::optional start, std::optional stop); +inline std::string head(const std::string& s, long n) { return slice(s, std::nullopt, n); } +inline std::string tail(const std::string& s, long n) { return slice(s, -n, std::nullopt); } +std::vector decode(const std::string& s); +std::string encode(const std::vector& cps); +void put_utf8(std::string& out, uint32_t cp); + +// bytes.decode("utf-8", "replace") with CPython's maximal-subpart rule; the result is +// valid UTF-8 (surrogates and overlongs become U+FFFD). Use at every ingest of external bytes. +std::string utf8_replace(const std::string& bytes); +// text-mode newline translation of open(path, "r"): \r\n and \r become \n. +std::string universal_newlines(const std::string& text); + +bool is_space_cp(uint32_t cp); +std::string strip(const std::string& s); +std::string lstrip(const std::string& s); +std::string rstrip(const std::string& s); +std::string strip_chars(const std::string& s, const std::string& chars); +std::string rstrip_chars(const std::string& s, const std::string& chars); +std::string lstrip_chars(const std::string& s, const std::string& chars); +std::string lower(const std::string& s); +std::string upper(const std::string& s); +std::vector split_ws(const std::string& s); // s.split() +std::vector split(const std::string& s, const std::string& sep, long maxsplit = -1); +std::vector splitlines(const std::string& s); +std::string join(const std::vector& parts, const std::string& sep); +bool startswith(const std::string& s, const std::string& prefix); +bool endswith(const std::string& s, const std::string& suffix); +std::string replace(const std::string& s, const std::string& from, const std::string& to); +long count(const std::string& s, const std::string& sub); +bool isdigit_ascii(const std::string& s); // str.isdigit() for ASCII input (non-empty) +std::string ljust_cut(const std::string& s, size_t n); + +// ---- values ---- +std::string str(const json::Value& v); // str(x) +std::string repr(const json::Value& v); // repr(x) +std::string repr_str(const std::string& s); +// int(x) / float(x) as tai.py uses them under `except (TypeError, ValueError)`. +std::optional to_int(const json::Value& v); +std::optional to_float(const json::Value& v); +std::optional parse_int(const std::string& s); // int("...") + +// `str(args.get(key) or "")` +std::string str_or_empty(const json::Value& v); +// `x or dflt` for ints: dflt when falsy. +const json::Value& or_value(const json::Value& v, const json::Value& dflt); + +// ---- time ---- +struct DateTime { + int year{1970}, month{1}, day{1}, hour{0}, minute{0}, second{0}, micro{0}; + bool aware{false}; + int offset_seconds{0}; +}; +double now(); // time.time() +std::string now_iso(); // datetime.now(utc).isoformat(timespec="seconds") +std::string utc_iso(double epoch, bool with_micro); // aware UTC isoformat +std::string utc_now_iso(); // datetime.now(timezone.utc).isoformat() +// _fromisoformat; throws std::invalid_argument("invalid datetime...") on failure. +DateTime fromisoformat(const std::string& text); +double to_epoch(const DateTime& dt); // naive means local time +std::string isoformat(const DateTime& dt); +// moment.astimezone(utc).isoformat() +std::string to_utc_iso(const DateTime& dt); +// astimezone().strftime("%Y-%m-%d %H:%M") in local time +std::string local_minutes(double epoch); + +// ---- misc ---- +std::string sha1_hex(const std::string& data); +std::string sha256_hex(const std::string& data); +std::string uuid4_hex(); +std::string uuid4(); +} // namespace detz::py diff --git a/taicpp/src/llm/term.cpp b/taicpp/src/llm/term.cpp new file mode 100644 index 0000000..be9c983 --- /dev/null +++ b/taicpp/src/llm/term.cpp @@ -0,0 +1,68 @@ +#include "term.h" +#include "ports.h" +#include "consts.h" +#include +#include +#include +namespace detz::llm { + +std::string paint(const std::string& text, const std::string& c1, const std::string& c2) { + const char* nc = std::getenv("NO_COLOR"); + if ((nc != nullptr && *nc != '\0') || !isatty(1)) { return text; } + return c1 + c2 + text + Ansi::RESET; +} +void print_out(const std::string& line) { + std::fputs(line.c_str(), stdout); + std::fputc('\n', stdout); + std::fflush(stdout); +} +void print_err(const std::string& line) { + std::fputs(line.c_str(), stderr); + std::fputc('\n', stderr); + std::fflush(stderr); +} +bool stdin_isatty() { return isatty(0) != 0; } +bool stdout_isatty() { return isatty(1) != 0; } +bool stderr_isatty() { return isatty(2) != 0; } + +// ---- log + box ports (process-wide seams) +namespace { +LogSink* g_log = nullptr; +BoxPort* g_box = nullptr; +} // namespace +LogSink* log_sink() { return g_log; } +void set_log_sink(LogSink* sink) { g_log = sink; } +bool log_enabled() { return g_log != nullptr && g_log->enabled(); } +void log_event(const std::string& level, const std::string& area, const std::string& kind, + const std::string& message, const std::string& profile, const std::string& bot, + const json::Value& meta) { + if (g_log != nullptr) { g_log->event(level, area, kind, message, profile, bot, meta); } +} +BoxPort* box_port() { return g_box; } +void set_box_port(BoxPort* port) { g_box = port; } + +const std::vector& DevplaceFreeRoutes() { + static const std::vector v = {"free", "devii", "molodetz"}; + return v; +} +const std::vector& ZenFreeModels() { + static const std::vector v = {"mimo-v2.6-flash-free", "nemotron-3.5-lightning-free", "muse-spark-1.3-contributor-free", "muse-spark-1.2-contributor-free", "jev-1.13-free", "step-5-preview-free", "space-bunny-free", "longcat-2.5-preview-free", "ling-3.1-flash-free"}; + return v; +} +const std::vector& PollinationsModels() { + static const std::vector v = {"openai-fast"}; + return v; +} +const std::vector& ProviderLabels() { + static const std::vector v = {kMuseLabel, kGrokLabel, kClaudeLabel, kCodexLabel, kGeminiLabel, kOpencodeLabel, kOllamaLabel, kOpenrouterLabel, kPollinationsLabel, kZenLabel, kDevplaceLabel}; + return v; +} +const std::vector& RetryBackoff() { + static const std::vector v = {2, 5, 10, 20, 30, 60}; + return v; +} +const std::vector& CliChatUrls() { + static const std::vector v = {kMuseChatUrl, kGrokChatUrl, kClaudeChatUrl, kCodexChatUrl, kGeminiChatUrl}; + return v; +} +} // namespace detz::llm diff --git a/taicpp/src/llm/term.h b/taicpp/src/llm/term.h new file mode 100644 index 0000000..d2c3c6c --- /dev/null +++ b/taicpp/src/llm/term.h @@ -0,0 +1,27 @@ +#pragma once +// Ansi + paint + stderr/stdout helpers matching tai.py's Ansi class and paint(). +#include +namespace detz::llm { +struct Ansi { + static constexpr const char* RESET = "\033[0m"; + static constexpr const char* BOLD = "\033[1m"; + static constexpr const char* DIM = "\033[2m"; + static constexpr const char* UNDERLINE = "\033[4m"; + static constexpr const char* STRIKE = "\033[9m"; + static constexpr const char* RED = "\033[31m"; + static constexpr const char* GREEN = "\033[32m"; + static constexpr const char* YELLOW = "\033[33m"; + static constexpr const char* BLUE = "\033[34m"; + static constexpr const char* MAGENTA = "\033[35m"; + static constexpr const char* CYAN = "\033[36m"; + static constexpr const char* GRAY = "\033[90m"; +}; +// paint(text, *codes): plain text when NO_COLOR is set or stdout is not a tty. +std::string paint(const std::string& text, const std::string& c1 = "", const std::string& c2 = ""); +// print(...) equivalents. +void print_out(const std::string& line); +void print_err(const std::string& line); +bool stdin_isatty(); +bool stdout_isatty(); +bool stderr_isatty(); +} // namespace detz::llm diff --git a/taicpp/src/llm/unicode_data.cpp b/taicpp/src/llm/unicode_data.cpp new file mode 100644 index 0000000..d7bdd82 --- /dev/null +++ b/taicpp/src/llm/unicode_data.cpp @@ -0,0 +1,3731 @@ +// GENERATED by src/agent/gen/gen_data.py unicode from Python 3.14.4 (unicodedata 16.0.0). Do not edit. +#include "unicode_data.h" +namespace detz::py::ucd { +const std::vector& LowerMap() { + static const std::vector v = { + {0x41, "a"}, + {0x42, "b"}, + {0x43, "c"}, + {0x44, "d"}, + {0x45, "e"}, + {0x46, "f"}, + {0x47, "g"}, + {0x48, "h"}, + {0x49, "i"}, + {0x4A, "j"}, + {0x4B, "k"}, + {0x4C, "l"}, + {0x4D, "m"}, + {0x4E, "n"}, + {0x4F, "o"}, + {0x50, "p"}, + {0x51, "q"}, + {0x52, "r"}, + {0x53, "s"}, + {0x54, "t"}, + {0x55, "u"}, + {0x56, "v"}, + {0x57, "w"}, + {0x58, "x"}, + {0x59, "y"}, + {0x5A, "z"}, + {0xC0, "\303\240"}, + {0xC1, "\303\241"}, + {0xC2, "\303\242"}, + {0xC3, "\303\243"}, + {0xC4, "\303\244"}, + {0xC5, "\303\245"}, + {0xC6, "\303\246"}, + {0xC7, "\303\247"}, + {0xC8, "\303\250"}, + {0xC9, "\303\251"}, + {0xCA, "\303\252"}, + {0xCB, "\303\253"}, + {0xCC, "\303\254"}, + {0xCD, "\303\255"}, + {0xCE, "\303\256"}, + {0xCF, "\303\257"}, + {0xD0, "\303\260"}, + {0xD1, "\303\261"}, + {0xD2, "\303\262"}, + {0xD3, "\303\263"}, + {0xD4, "\303\264"}, + {0xD5, "\303\265"}, + {0xD6, "\303\266"}, + {0xD8, "\303\270"}, + {0xD9, "\303\271"}, + {0xDA, "\303\272"}, + {0xDB, "\303\273"}, + {0xDC, "\303\274"}, + {0xDD, "\303\275"}, + {0xDE, "\303\276"}, + {0x100, "\304\201"}, + {0x102, "\304\203"}, + {0x104, "\304\205"}, + {0x106, "\304\207"}, + {0x108, "\304\211"}, + {0x10A, "\304\213"}, + {0x10C, "\304\215"}, + {0x10E, "\304\217"}, + {0x110, "\304\221"}, + {0x112, "\304\223"}, + {0x114, "\304\225"}, + {0x116, "\304\227"}, + {0x118, "\304\231"}, + {0x11A, "\304\233"}, + {0x11C, "\304\235"}, + {0x11E, "\304\237"}, + {0x120, "\304\241"}, + {0x122, "\304\243"}, + {0x124, "\304\245"}, + {0x126, "\304\247"}, + {0x128, "\304\251"}, + {0x12A, "\304\253"}, + {0x12C, "\304\255"}, + {0x12E, "\304\257"}, + {0x130, "i\314\207"}, + {0x132, "\304\263"}, + {0x134, "\304\265"}, + {0x136, "\304\267"}, + {0x139, "\304\272"}, + {0x13B, "\304\274"}, + {0x13D, "\304\276"}, + {0x13F, "\305\200"}, + {0x141, "\305\202"}, + {0x143, "\305\204"}, + {0x145, "\305\206"}, + {0x147, "\305\210"}, + {0x14A, "\305\213"}, + {0x14C, "\305\215"}, + {0x14E, "\305\217"}, + {0x150, "\305\221"}, + {0x152, "\305\223"}, + {0x154, "\305\225"}, + {0x156, "\305\227"}, + {0x158, "\305\231"}, + {0x15A, "\305\233"}, + {0x15C, "\305\235"}, + {0x15E, "\305\237"}, + {0x160, "\305\241"}, + {0x162, "\305\243"}, + {0x164, "\305\245"}, + {0x166, "\305\247"}, + {0x168, "\305\251"}, + {0x16A, "\305\253"}, + {0x16C, "\305\255"}, + {0x16E, "\305\257"}, + {0x170, "\305\261"}, + {0x172, "\305\263"}, + {0x174, "\305\265"}, + {0x176, "\305\267"}, + {0x178, "\303\277"}, + {0x179, "\305\272"}, + {0x17B, "\305\274"}, + {0x17D, "\305\276"}, + {0x181, "\311\223"}, + {0x182, "\306\203"}, + {0x184, "\306\205"}, + {0x186, "\311\224"}, + {0x187, "\306\210"}, + {0x189, "\311\226"}, + {0x18A, "\311\227"}, + {0x18B, "\306\214"}, + {0x18E, "\307\235"}, + {0x18F, "\311\231"}, + {0x190, "\311\233"}, + {0x191, "\306\222"}, + {0x193, "\311\240"}, + {0x194, "\311\243"}, + {0x196, "\311\251"}, + {0x197, "\311\250"}, + {0x198, "\306\231"}, + {0x19C, "\311\257"}, + {0x19D, "\311\262"}, + {0x19F, "\311\265"}, + {0x1A0, "\306\241"}, + {0x1A2, "\306\243"}, + {0x1A4, "\306\245"}, + {0x1A6, "\312\200"}, + {0x1A7, "\306\250"}, + {0x1A9, "\312\203"}, + {0x1AC, "\306\255"}, + {0x1AE, "\312\210"}, + {0x1AF, "\306\260"}, + {0x1B1, "\312\212"}, + {0x1B2, "\312\213"}, + {0x1B3, "\306\264"}, + {0x1B5, "\306\266"}, + {0x1B7, "\312\222"}, + {0x1B8, "\306\271"}, + {0x1BC, "\306\275"}, + {0x1C4, "\307\206"}, + {0x1C5, "\307\206"}, + {0x1C7, "\307\211"}, + {0x1C8, "\307\211"}, + {0x1CA, "\307\214"}, + {0x1CB, "\307\214"}, + {0x1CD, "\307\216"}, + {0x1CF, "\307\220"}, + {0x1D1, "\307\222"}, + {0x1D3, "\307\224"}, + {0x1D5, "\307\226"}, + {0x1D7, "\307\230"}, + {0x1D9, "\307\232"}, + {0x1DB, "\307\234"}, + {0x1DE, "\307\237"}, + {0x1E0, "\307\241"}, + {0x1E2, "\307\243"}, + {0x1E4, "\307\245"}, + {0x1E6, "\307\247"}, + {0x1E8, "\307\251"}, + {0x1EA, "\307\253"}, + {0x1EC, "\307\255"}, + {0x1EE, "\307\257"}, + {0x1F1, "\307\263"}, + {0x1F2, "\307\263"}, + {0x1F4, "\307\265"}, + {0x1F6, "\306\225"}, + {0x1F7, "\306\277"}, + {0x1F8, "\307\271"}, + {0x1FA, "\307\273"}, + {0x1FC, "\307\275"}, + {0x1FE, "\307\277"}, + {0x200, "\310\201"}, + {0x202, "\310\203"}, + {0x204, "\310\205"}, + {0x206, "\310\207"}, + {0x208, "\310\211"}, + {0x20A, "\310\213"}, + {0x20C, "\310\215"}, + {0x20E, "\310\217"}, + {0x210, "\310\221"}, + {0x212, "\310\223"}, + {0x214, "\310\225"}, + {0x216, "\310\227"}, + {0x218, "\310\231"}, + {0x21A, "\310\233"}, + {0x21C, "\310\235"}, + {0x21E, "\310\237"}, + {0x220, "\306\236"}, + {0x222, "\310\243"}, + {0x224, "\310\245"}, + {0x226, "\310\247"}, + {0x228, "\310\251"}, + {0x22A, "\310\253"}, + {0x22C, "\310\255"}, + {0x22E, "\310\257"}, + {0x230, "\310\261"}, + {0x232, "\310\263"}, + {0x23A, "\342\261\245"}, + {0x23B, "\310\274"}, + {0x23D, "\306\232"}, + {0x23E, "\342\261\246"}, + {0x241, "\311\202"}, + {0x243, "\306\200"}, + {0x244, "\312\211"}, + {0x245, "\312\214"}, + {0x246, "\311\207"}, + {0x248, "\311\211"}, + {0x24A, "\311\213"}, + {0x24C, "\311\215"}, + {0x24E, "\311\217"}, + {0x370, "\315\261"}, + {0x372, "\315\263"}, + {0x376, "\315\267"}, + {0x37F, "\317\263"}, + {0x386, "\316\254"}, + {0x388, "\316\255"}, + {0x389, "\316\256"}, + {0x38A, "\316\257"}, + {0x38C, "\317\214"}, + {0x38E, "\317\215"}, + {0x38F, "\317\216"}, + {0x391, "\316\261"}, + {0x392, "\316\262"}, + {0x393, "\316\263"}, + {0x394, "\316\264"}, + {0x395, "\316\265"}, + {0x396, "\316\266"}, + {0x397, "\316\267"}, + {0x398, "\316\270"}, + {0x399, "\316\271"}, + {0x39A, "\316\272"}, + {0x39B, "\316\273"}, + {0x39C, "\316\274"}, + {0x39D, "\316\275"}, + {0x39E, "\316\276"}, + {0x39F, "\316\277"}, + {0x3A0, "\317\200"}, + {0x3A1, "\317\201"}, + {0x3A3, "\317\203"}, + {0x3A4, "\317\204"}, + {0x3A5, "\317\205"}, + {0x3A6, "\317\206"}, + {0x3A7, "\317\207"}, + {0x3A8, "\317\210"}, + {0x3A9, "\317\211"}, + {0x3AA, "\317\212"}, + {0x3AB, "\317\213"}, + {0x3CF, "\317\227"}, + {0x3D8, "\317\231"}, + {0x3DA, "\317\233"}, + {0x3DC, "\317\235"}, + {0x3DE, "\317\237"}, + {0x3E0, "\317\241"}, + {0x3E2, "\317\243"}, + {0x3E4, "\317\245"}, + {0x3E6, "\317\247"}, + {0x3E8, "\317\251"}, + {0x3EA, "\317\253"}, + {0x3EC, "\317\255"}, + {0x3EE, "\317\257"}, + {0x3F4, "\316\270"}, + {0x3F7, "\317\270"}, + {0x3F9, "\317\262"}, + {0x3FA, "\317\273"}, + {0x3FD, "\315\273"}, + {0x3FE, "\315\274"}, + {0x3FF, "\315\275"}, + {0x400, "\321\220"}, + {0x401, "\321\221"}, + {0x402, "\321\222"}, + {0x403, "\321\223"}, + {0x404, "\321\224"}, + {0x405, "\321\225"}, + {0x406, "\321\226"}, + {0x407, "\321\227"}, + {0x408, "\321\230"}, + {0x409, "\321\231"}, + {0x40A, "\321\232"}, + {0x40B, "\321\233"}, + {0x40C, "\321\234"}, + {0x40D, "\321\235"}, + {0x40E, "\321\236"}, + {0x40F, "\321\237"}, + {0x410, "\320\260"}, + {0x411, "\320\261"}, + {0x412, "\320\262"}, + {0x413, "\320\263"}, + {0x414, "\320\264"}, + {0x415, "\320\265"}, + {0x416, "\320\266"}, + {0x417, "\320\267"}, + {0x418, "\320\270"}, + {0x419, "\320\271"}, + {0x41A, "\320\272"}, + {0x41B, "\320\273"}, + {0x41C, "\320\274"}, + {0x41D, "\320\275"}, + {0x41E, "\320\276"}, + {0x41F, "\320\277"}, + {0x420, "\321\200"}, + {0x421, "\321\201"}, + {0x422, "\321\202"}, + {0x423, "\321\203"}, + {0x424, "\321\204"}, + {0x425, "\321\205"}, + {0x426, "\321\206"}, + {0x427, "\321\207"}, + {0x428, "\321\210"}, + {0x429, "\321\211"}, + {0x42A, "\321\212"}, + {0x42B, "\321\213"}, + {0x42C, "\321\214"}, + {0x42D, "\321\215"}, + {0x42E, "\321\216"}, + {0x42F, "\321\217"}, + {0x460, "\321\241"}, + {0x462, "\321\243"}, + {0x464, "\321\245"}, + {0x466, "\321\247"}, + {0x468, "\321\251"}, + {0x46A, "\321\253"}, + {0x46C, "\321\255"}, + {0x46E, "\321\257"}, + {0x470, "\321\261"}, + {0x472, "\321\263"}, + {0x474, "\321\265"}, + {0x476, "\321\267"}, + {0x478, "\321\271"}, + {0x47A, "\321\273"}, + {0x47C, "\321\275"}, + {0x47E, "\321\277"}, + {0x480, "\322\201"}, + {0x48A, "\322\213"}, + {0x48C, "\322\215"}, + {0x48E, "\322\217"}, + {0x490, "\322\221"}, + {0x492, "\322\223"}, + {0x494, "\322\225"}, + {0x496, "\322\227"}, + {0x498, "\322\231"}, + {0x49A, "\322\233"}, + {0x49C, "\322\235"}, + {0x49E, "\322\237"}, + {0x4A0, "\322\241"}, + {0x4A2, "\322\243"}, + {0x4A4, "\322\245"}, + {0x4A6, "\322\247"}, + {0x4A8, "\322\251"}, + {0x4AA, "\322\253"}, + {0x4AC, "\322\255"}, + {0x4AE, "\322\257"}, + {0x4B0, "\322\261"}, + {0x4B2, "\322\263"}, + {0x4B4, "\322\265"}, + {0x4B6, "\322\267"}, + {0x4B8, "\322\271"}, + {0x4BA, "\322\273"}, + {0x4BC, "\322\275"}, + {0x4BE, "\322\277"}, + {0x4C0, "\323\217"}, + {0x4C1, "\323\202"}, + {0x4C3, "\323\204"}, + {0x4C5, "\323\206"}, + {0x4C7, "\323\210"}, + {0x4C9, "\323\212"}, + {0x4CB, "\323\214"}, + {0x4CD, "\323\216"}, + {0x4D0, "\323\221"}, + {0x4D2, "\323\223"}, + {0x4D4, "\323\225"}, + {0x4D6, "\323\227"}, + {0x4D8, "\323\231"}, + {0x4DA, "\323\233"}, + {0x4DC, "\323\235"}, + {0x4DE, "\323\237"}, + {0x4E0, "\323\241"}, + {0x4E2, "\323\243"}, + {0x4E4, "\323\245"}, + {0x4E6, "\323\247"}, + {0x4E8, "\323\251"}, + {0x4EA, "\323\253"}, + {0x4EC, "\323\255"}, + {0x4EE, "\323\257"}, + {0x4F0, "\323\261"}, + {0x4F2, "\323\263"}, + {0x4F4, "\323\265"}, + {0x4F6, "\323\267"}, + {0x4F8, "\323\271"}, + {0x4FA, "\323\273"}, + {0x4FC, "\323\275"}, + {0x4FE, "\323\277"}, + {0x500, "\324\201"}, + {0x502, "\324\203"}, + {0x504, "\324\205"}, + {0x506, "\324\207"}, + {0x508, "\324\211"}, + {0x50A, "\324\213"}, + {0x50C, "\324\215"}, + {0x50E, "\324\217"}, + {0x510, "\324\221"}, + {0x512, "\324\223"}, + {0x514, "\324\225"}, + {0x516, "\324\227"}, + {0x518, "\324\231"}, + {0x51A, "\324\233"}, + {0x51C, "\324\235"}, + {0x51E, "\324\237"}, + {0x520, "\324\241"}, + {0x522, "\324\243"}, + {0x524, "\324\245"}, + {0x526, "\324\247"}, + {0x528, "\324\251"}, + {0x52A, "\324\253"}, + {0x52C, "\324\255"}, + {0x52E, "\324\257"}, + {0x531, "\325\241"}, + {0x532, "\325\242"}, + {0x533, "\325\243"}, + {0x534, "\325\244"}, + {0x535, "\325\245"}, + {0x536, "\325\246"}, + {0x537, "\325\247"}, + {0x538, "\325\250"}, + {0x539, "\325\251"}, + {0x53A, "\325\252"}, + {0x53B, "\325\253"}, + {0x53C, "\325\254"}, + {0x53D, "\325\255"}, + {0x53E, "\325\256"}, + {0x53F, "\325\257"}, + {0x540, "\325\260"}, + {0x541, "\325\261"}, + {0x542, "\325\262"}, + {0x543, "\325\263"}, + {0x544, "\325\264"}, + {0x545, "\325\265"}, + {0x546, "\325\266"}, + {0x547, "\325\267"}, + {0x548, "\325\270"}, + {0x549, "\325\271"}, + {0x54A, "\325\272"}, + {0x54B, "\325\273"}, + {0x54C, "\325\274"}, + {0x54D, "\325\275"}, + {0x54E, "\325\276"}, + {0x54F, "\325\277"}, + {0x550, "\326\200"}, + {0x551, "\326\201"}, + {0x552, "\326\202"}, + {0x553, "\326\203"}, + {0x554, "\326\204"}, + {0x555, "\326\205"}, + {0x556, "\326\206"}, + {0x10A0, "\342\264\200"}, + {0x10A1, "\342\264\201"}, + {0x10A2, "\342\264\202"}, + {0x10A3, "\342\264\203"}, + {0x10A4, "\342\264\204"}, + {0x10A5, "\342\264\205"}, + {0x10A6, "\342\264\206"}, + {0x10A7, "\342\264\207"}, + {0x10A8, "\342\264\210"}, + {0x10A9, "\342\264\211"}, + {0x10AA, "\342\264\212"}, + {0x10AB, "\342\264\213"}, + {0x10AC, "\342\264\214"}, + {0x10AD, "\342\264\215"}, + {0x10AE, "\342\264\216"}, + {0x10AF, "\342\264\217"}, + {0x10B0, "\342\264\220"}, + {0x10B1, "\342\264\221"}, + {0x10B2, "\342\264\222"}, + {0x10B3, "\342\264\223"}, + {0x10B4, "\342\264\224"}, + {0x10B5, "\342\264\225"}, + {0x10B6, "\342\264\226"}, + {0x10B7, "\342\264\227"}, + {0x10B8, "\342\264\230"}, + {0x10B9, "\342\264\231"}, + {0x10BA, "\342\264\232"}, + {0x10BB, "\342\264\233"}, + {0x10BC, "\342\264\234"}, + {0x10BD, "\342\264\235"}, + {0x10BE, "\342\264\236"}, + {0x10BF, "\342\264\237"}, + {0x10C0, "\342\264\240"}, + {0x10C1, "\342\264\241"}, + {0x10C2, "\342\264\242"}, + {0x10C3, "\342\264\243"}, + {0x10C4, "\342\264\244"}, + {0x10C5, "\342\264\245"}, + {0x10C7, "\342\264\247"}, + {0x10CD, "\342\264\255"}, + {0x13A0, "\352\255\260"}, + {0x13A1, "\352\255\261"}, + {0x13A2, "\352\255\262"}, + {0x13A3, "\352\255\263"}, + {0x13A4, "\352\255\264"}, + {0x13A5, "\352\255\265"}, + {0x13A6, "\352\255\266"}, + {0x13A7, "\352\255\267"}, + {0x13A8, "\352\255\270"}, + {0x13A9, "\352\255\271"}, + {0x13AA, "\352\255\272"}, + {0x13AB, "\352\255\273"}, + {0x13AC, "\352\255\274"}, + {0x13AD, "\352\255\275"}, + {0x13AE, "\352\255\276"}, + {0x13AF, "\352\255\277"}, + {0x13B0, "\352\256\200"}, + {0x13B1, "\352\256\201"}, + {0x13B2, "\352\256\202"}, + {0x13B3, "\352\256\203"}, + {0x13B4, "\352\256\204"}, + {0x13B5, "\352\256\205"}, + {0x13B6, "\352\256\206"}, + {0x13B7, "\352\256\207"}, + {0x13B8, "\352\256\210"}, + {0x13B9, "\352\256\211"}, + {0x13BA, "\352\256\212"}, + {0x13BB, "\352\256\213"}, + {0x13BC, "\352\256\214"}, + {0x13BD, "\352\256\215"}, + {0x13BE, "\352\256\216"}, + {0x13BF, "\352\256\217"}, + {0x13C0, "\352\256\220"}, + {0x13C1, "\352\256\221"}, + {0x13C2, "\352\256\222"}, + {0x13C3, "\352\256\223"}, + {0x13C4, "\352\256\224"}, + {0x13C5, "\352\256\225"}, + {0x13C6, "\352\256\226"}, + {0x13C7, "\352\256\227"}, + {0x13C8, "\352\256\230"}, + {0x13C9, "\352\256\231"}, + {0x13CA, "\352\256\232"}, + {0x13CB, "\352\256\233"}, + {0x13CC, "\352\256\234"}, + {0x13CD, "\352\256\235"}, + {0x13CE, "\352\256\236"}, + {0x13CF, "\352\256\237"}, + {0x13D0, "\352\256\240"}, + {0x13D1, "\352\256\241"}, + {0x13D2, "\352\256\242"}, + {0x13D3, "\352\256\243"}, + {0x13D4, "\352\256\244"}, + {0x13D5, "\352\256\245"}, + {0x13D6, "\352\256\246"}, + {0x13D7, "\352\256\247"}, + {0x13D8, "\352\256\250"}, + {0x13D9, "\352\256\251"}, + {0x13DA, "\352\256\252"}, + {0x13DB, "\352\256\253"}, + {0x13DC, "\352\256\254"}, + {0x13DD, "\352\256\255"}, + {0x13DE, "\352\256\256"}, + {0x13DF, "\352\256\257"}, + {0x13E0, "\352\256\260"}, + {0x13E1, "\352\256\261"}, + {0x13E2, "\352\256\262"}, + {0x13E3, "\352\256\263"}, + {0x13E4, "\352\256\264"}, + {0x13E5, "\352\256\265"}, + {0x13E6, "\352\256\266"}, + {0x13E7, "\352\256\267"}, + {0x13E8, "\352\256\270"}, + {0x13E9, "\352\256\271"}, + {0x13EA, "\352\256\272"}, + {0x13EB, "\352\256\273"}, + {0x13EC, "\352\256\274"}, + {0x13ED, "\352\256\275"}, + {0x13EE, "\352\256\276"}, + {0x13EF, "\352\256\277"}, + {0x13F0, "\341\217\270"}, + {0x13F1, "\341\217\271"}, + {0x13F2, "\341\217\272"}, + {0x13F3, "\341\217\273"}, + {0x13F4, "\341\217\274"}, + {0x13F5, "\341\217\275"}, + {0x1C89, "\341\262\212"}, + {0x1C90, "\341\203\220"}, + {0x1C91, "\341\203\221"}, + {0x1C92, "\341\203\222"}, + {0x1C93, "\341\203\223"}, + {0x1C94, "\341\203\224"}, + {0x1C95, "\341\203\225"}, + {0x1C96, "\341\203\226"}, + {0x1C97, "\341\203\227"}, + {0x1C98, "\341\203\230"}, + {0x1C99, "\341\203\231"}, + {0x1C9A, "\341\203\232"}, + {0x1C9B, "\341\203\233"}, + {0x1C9C, "\341\203\234"}, + {0x1C9D, "\341\203\235"}, + {0x1C9E, "\341\203\236"}, + {0x1C9F, "\341\203\237"}, + {0x1CA0, "\341\203\240"}, + {0x1CA1, "\341\203\241"}, + {0x1CA2, "\341\203\242"}, + {0x1CA3, "\341\203\243"}, + {0x1CA4, "\341\203\244"}, + {0x1CA5, "\341\203\245"}, + {0x1CA6, "\341\203\246"}, + {0x1CA7, "\341\203\247"}, + {0x1CA8, "\341\203\250"}, + {0x1CA9, "\341\203\251"}, + {0x1CAA, "\341\203\252"}, + {0x1CAB, "\341\203\253"}, + {0x1CAC, "\341\203\254"}, + {0x1CAD, "\341\203\255"}, + {0x1CAE, "\341\203\256"}, + {0x1CAF, "\341\203\257"}, + {0x1CB0, "\341\203\260"}, + {0x1CB1, "\341\203\261"}, + {0x1CB2, "\341\203\262"}, + {0x1CB3, "\341\203\263"}, + {0x1CB4, "\341\203\264"}, + {0x1CB5, "\341\203\265"}, + {0x1CB6, "\341\203\266"}, + {0x1CB7, "\341\203\267"}, + {0x1CB8, "\341\203\270"}, + {0x1CB9, "\341\203\271"}, + {0x1CBA, "\341\203\272"}, + {0x1CBD, "\341\203\275"}, + {0x1CBE, "\341\203\276"}, + {0x1CBF, "\341\203\277"}, + {0x1E00, "\341\270\201"}, + {0x1E02, "\341\270\203"}, + {0x1E04, "\341\270\205"}, + {0x1E06, "\341\270\207"}, + {0x1E08, "\341\270\211"}, + {0x1E0A, "\341\270\213"}, + {0x1E0C, "\341\270\215"}, + {0x1E0E, "\341\270\217"}, + {0x1E10, "\341\270\221"}, + {0x1E12, "\341\270\223"}, + {0x1E14, "\341\270\225"}, + {0x1E16, "\341\270\227"}, + {0x1E18, "\341\270\231"}, + {0x1E1A, "\341\270\233"}, + {0x1E1C, "\341\270\235"}, + {0x1E1E, "\341\270\237"}, + {0x1E20, "\341\270\241"}, + {0x1E22, "\341\270\243"}, + {0x1E24, "\341\270\245"}, + {0x1E26, "\341\270\247"}, + {0x1E28, "\341\270\251"}, + {0x1E2A, "\341\270\253"}, + {0x1E2C, "\341\270\255"}, + {0x1E2E, "\341\270\257"}, + {0x1E30, "\341\270\261"}, + {0x1E32, "\341\270\263"}, + {0x1E34, "\341\270\265"}, + {0x1E36, "\341\270\267"}, + {0x1E38, "\341\270\271"}, + {0x1E3A, "\341\270\273"}, + {0x1E3C, "\341\270\275"}, + {0x1E3E, "\341\270\277"}, + {0x1E40, "\341\271\201"}, + {0x1E42, "\341\271\203"}, + {0x1E44, "\341\271\205"}, + {0x1E46, "\341\271\207"}, + {0x1E48, "\341\271\211"}, + {0x1E4A, "\341\271\213"}, + {0x1E4C, "\341\271\215"}, + {0x1E4E, "\341\271\217"}, + {0x1E50, "\341\271\221"}, + {0x1E52, "\341\271\223"}, + {0x1E54, "\341\271\225"}, + {0x1E56, "\341\271\227"}, + {0x1E58, "\341\271\231"}, + {0x1E5A, "\341\271\233"}, + {0x1E5C, "\341\271\235"}, + {0x1E5E, "\341\271\237"}, + {0x1E60, "\341\271\241"}, + {0x1E62, "\341\271\243"}, + {0x1E64, "\341\271\245"}, + {0x1E66, "\341\271\247"}, + {0x1E68, "\341\271\251"}, + {0x1E6A, "\341\271\253"}, + {0x1E6C, "\341\271\255"}, + {0x1E6E, "\341\271\257"}, + {0x1E70, "\341\271\261"}, + {0x1E72, "\341\271\263"}, + {0x1E74, "\341\271\265"}, + {0x1E76, "\341\271\267"}, + {0x1E78, "\341\271\271"}, + {0x1E7A, "\341\271\273"}, + {0x1E7C, "\341\271\275"}, + {0x1E7E, "\341\271\277"}, + {0x1E80, "\341\272\201"}, + {0x1E82, "\341\272\203"}, + {0x1E84, "\341\272\205"}, + {0x1E86, "\341\272\207"}, + {0x1E88, "\341\272\211"}, + {0x1E8A, "\341\272\213"}, + {0x1E8C, "\341\272\215"}, + {0x1E8E, "\341\272\217"}, + {0x1E90, "\341\272\221"}, + {0x1E92, "\341\272\223"}, + {0x1E94, "\341\272\225"}, + {0x1E9E, "\303\237"}, + {0x1EA0, "\341\272\241"}, + {0x1EA2, "\341\272\243"}, + {0x1EA4, "\341\272\245"}, + {0x1EA6, "\341\272\247"}, + {0x1EA8, "\341\272\251"}, + {0x1EAA, "\341\272\253"}, + {0x1EAC, "\341\272\255"}, + {0x1EAE, "\341\272\257"}, + {0x1EB0, "\341\272\261"}, + {0x1EB2, "\341\272\263"}, + {0x1EB4, "\341\272\265"}, + {0x1EB6, "\341\272\267"}, + {0x1EB8, "\341\272\271"}, + {0x1EBA, "\341\272\273"}, + {0x1EBC, "\341\272\275"}, + {0x1EBE, "\341\272\277"}, + {0x1EC0, "\341\273\201"}, + {0x1EC2, "\341\273\203"}, + {0x1EC4, "\341\273\205"}, + {0x1EC6, "\341\273\207"}, + {0x1EC8, "\341\273\211"}, + {0x1ECA, "\341\273\213"}, + {0x1ECC, "\341\273\215"}, + {0x1ECE, "\341\273\217"}, + {0x1ED0, "\341\273\221"}, + {0x1ED2, "\341\273\223"}, + {0x1ED4, "\341\273\225"}, + {0x1ED6, "\341\273\227"}, + {0x1ED8, "\341\273\231"}, + {0x1EDA, "\341\273\233"}, + {0x1EDC, "\341\273\235"}, + {0x1EDE, "\341\273\237"}, + {0x1EE0, "\341\273\241"}, + {0x1EE2, "\341\273\243"}, + {0x1EE4, "\341\273\245"}, + {0x1EE6, "\341\273\247"}, + {0x1EE8, "\341\273\251"}, + {0x1EEA, "\341\273\253"}, + {0x1EEC, "\341\273\255"}, + {0x1EEE, "\341\273\257"}, + {0x1EF0, "\341\273\261"}, + {0x1EF2, "\341\273\263"}, + {0x1EF4, "\341\273\265"}, + {0x1EF6, "\341\273\267"}, + {0x1EF8, "\341\273\271"}, + {0x1EFA, "\341\273\273"}, + {0x1EFC, "\341\273\275"}, + {0x1EFE, "\341\273\277"}, + {0x1F08, "\341\274\200"}, + {0x1F09, "\341\274\201"}, + {0x1F0A, "\341\274\202"}, + {0x1F0B, "\341\274\203"}, + {0x1F0C, "\341\274\204"}, + {0x1F0D, "\341\274\205"}, + {0x1F0E, "\341\274\206"}, + {0x1F0F, "\341\274\207"}, + {0x1F18, "\341\274\220"}, + {0x1F19, "\341\274\221"}, + {0x1F1A, "\341\274\222"}, + {0x1F1B, "\341\274\223"}, + {0x1F1C, "\341\274\224"}, + {0x1F1D, "\341\274\225"}, + {0x1F28, "\341\274\240"}, + {0x1F29, "\341\274\241"}, + {0x1F2A, "\341\274\242"}, + {0x1F2B, "\341\274\243"}, + {0x1F2C, "\341\274\244"}, + {0x1F2D, "\341\274\245"}, + {0x1F2E, "\341\274\246"}, + {0x1F2F, "\341\274\247"}, + {0x1F38, "\341\274\260"}, + {0x1F39, "\341\274\261"}, + {0x1F3A, "\341\274\262"}, + {0x1F3B, "\341\274\263"}, + {0x1F3C, "\341\274\264"}, + {0x1F3D, "\341\274\265"}, + {0x1F3E, "\341\274\266"}, + {0x1F3F, "\341\274\267"}, + {0x1F48, "\341\275\200"}, + {0x1F49, "\341\275\201"}, + {0x1F4A, "\341\275\202"}, + {0x1F4B, "\341\275\203"}, + {0x1F4C, "\341\275\204"}, + {0x1F4D, "\341\275\205"}, + {0x1F59, "\341\275\221"}, + {0x1F5B, "\341\275\223"}, + {0x1F5D, "\341\275\225"}, + {0x1F5F, "\341\275\227"}, + {0x1F68, "\341\275\240"}, + {0x1F69, "\341\275\241"}, + {0x1F6A, "\341\275\242"}, + {0x1F6B, "\341\275\243"}, + {0x1F6C, "\341\275\244"}, + {0x1F6D, "\341\275\245"}, + {0x1F6E, "\341\275\246"}, + {0x1F6F, "\341\275\247"}, + {0x1F88, "\341\276\200"}, + {0x1F89, "\341\276\201"}, + {0x1F8A, "\341\276\202"}, + {0x1F8B, "\341\276\203"}, + {0x1F8C, "\341\276\204"}, + {0x1F8D, "\341\276\205"}, + {0x1F8E, "\341\276\206"}, + {0x1F8F, "\341\276\207"}, + {0x1F98, "\341\276\220"}, + {0x1F99, "\341\276\221"}, + {0x1F9A, "\341\276\222"}, + {0x1F9B, "\341\276\223"}, + {0x1F9C, "\341\276\224"}, + {0x1F9D, "\341\276\225"}, + {0x1F9E, "\341\276\226"}, + {0x1F9F, "\341\276\227"}, + {0x1FA8, "\341\276\240"}, + {0x1FA9, "\341\276\241"}, + {0x1FAA, "\341\276\242"}, + {0x1FAB, "\341\276\243"}, + {0x1FAC, "\341\276\244"}, + {0x1FAD, "\341\276\245"}, + {0x1FAE, "\341\276\246"}, + {0x1FAF, "\341\276\247"}, + {0x1FB8, "\341\276\260"}, + {0x1FB9, "\341\276\261"}, + {0x1FBA, "\341\275\260"}, + {0x1FBB, "\341\275\261"}, + {0x1FBC, "\341\276\263"}, + {0x1FC8, "\341\275\262"}, + {0x1FC9, "\341\275\263"}, + {0x1FCA, "\341\275\264"}, + {0x1FCB, "\341\275\265"}, + {0x1FCC, "\341\277\203"}, + {0x1FD8, "\341\277\220"}, + {0x1FD9, "\341\277\221"}, + {0x1FDA, "\341\275\266"}, + {0x1FDB, "\341\275\267"}, + {0x1FE8, "\341\277\240"}, + {0x1FE9, "\341\277\241"}, + {0x1FEA, "\341\275\272"}, + {0x1FEB, "\341\275\273"}, + {0x1FEC, "\341\277\245"}, + {0x1FF8, "\341\275\270"}, + {0x1FF9, "\341\275\271"}, + {0x1FFA, "\341\275\274"}, + {0x1FFB, "\341\275\275"}, + {0x1FFC, "\341\277\263"}, + {0x2126, "\317\211"}, + {0x212A, "k"}, + {0x212B, "\303\245"}, + {0x2132, "\342\205\216"}, + {0x2160, "\342\205\260"}, + {0x2161, "\342\205\261"}, + {0x2162, "\342\205\262"}, + {0x2163, "\342\205\263"}, + {0x2164, "\342\205\264"}, + {0x2165, "\342\205\265"}, + {0x2166, "\342\205\266"}, + {0x2167, "\342\205\267"}, + {0x2168, "\342\205\270"}, + {0x2169, "\342\205\271"}, + {0x216A, "\342\205\272"}, + {0x216B, "\342\205\273"}, + {0x216C, "\342\205\274"}, + {0x216D, "\342\205\275"}, + {0x216E, "\342\205\276"}, + {0x216F, "\342\205\277"}, + {0x2183, "\342\206\204"}, + {0x24B6, "\342\223\220"}, + {0x24B7, "\342\223\221"}, + {0x24B8, "\342\223\222"}, + {0x24B9, "\342\223\223"}, + {0x24BA, "\342\223\224"}, + {0x24BB, "\342\223\225"}, + {0x24BC, "\342\223\226"}, + {0x24BD, "\342\223\227"}, + {0x24BE, "\342\223\230"}, + {0x24BF, "\342\223\231"}, + {0x24C0, "\342\223\232"}, + {0x24C1, "\342\223\233"}, + {0x24C2, "\342\223\234"}, + {0x24C3, "\342\223\235"}, + {0x24C4, "\342\223\236"}, + {0x24C5, "\342\223\237"}, + {0x24C6, "\342\223\240"}, + {0x24C7, "\342\223\241"}, + {0x24C8, "\342\223\242"}, + {0x24C9, "\342\223\243"}, + {0x24CA, "\342\223\244"}, + {0x24CB, "\342\223\245"}, + {0x24CC, "\342\223\246"}, + {0x24CD, "\342\223\247"}, + {0x24CE, "\342\223\250"}, + {0x24CF, "\342\223\251"}, + {0x2C00, "\342\260\260"}, + {0x2C01, "\342\260\261"}, + {0x2C02, "\342\260\262"}, + {0x2C03, "\342\260\263"}, + {0x2C04, "\342\260\264"}, + {0x2C05, "\342\260\265"}, + {0x2C06, "\342\260\266"}, + {0x2C07, "\342\260\267"}, + {0x2C08, "\342\260\270"}, + {0x2C09, "\342\260\271"}, + {0x2C0A, "\342\260\272"}, + {0x2C0B, "\342\260\273"}, + {0x2C0C, "\342\260\274"}, + {0x2C0D, "\342\260\275"}, + {0x2C0E, "\342\260\276"}, + {0x2C0F, "\342\260\277"}, + {0x2C10, "\342\261\200"}, + {0x2C11, "\342\261\201"}, + {0x2C12, "\342\261\202"}, + {0x2C13, "\342\261\203"}, + {0x2C14, "\342\261\204"}, + {0x2C15, "\342\261\205"}, + {0x2C16, "\342\261\206"}, + {0x2C17, "\342\261\207"}, + {0x2C18, "\342\261\210"}, + {0x2C19, "\342\261\211"}, + {0x2C1A, "\342\261\212"}, + {0x2C1B, "\342\261\213"}, + {0x2C1C, "\342\261\214"}, + {0x2C1D, "\342\261\215"}, + {0x2C1E, "\342\261\216"}, + {0x2C1F, "\342\261\217"}, + {0x2C20, "\342\261\220"}, + {0x2C21, "\342\261\221"}, + {0x2C22, "\342\261\222"}, + {0x2C23, "\342\261\223"}, + {0x2C24, "\342\261\224"}, + {0x2C25, "\342\261\225"}, + {0x2C26, "\342\261\226"}, + {0x2C27, "\342\261\227"}, + {0x2C28, "\342\261\230"}, + {0x2C29, "\342\261\231"}, + {0x2C2A, "\342\261\232"}, + {0x2C2B, "\342\261\233"}, + {0x2C2C, "\342\261\234"}, + {0x2C2D, "\342\261\235"}, + {0x2C2E, "\342\261\236"}, + {0x2C2F, "\342\261\237"}, + {0x2C60, "\342\261\241"}, + {0x2C62, "\311\253"}, + {0x2C63, "\341\265\275"}, + {0x2C64, "\311\275"}, + {0x2C67, "\342\261\250"}, + {0x2C69, "\342\261\252"}, + {0x2C6B, "\342\261\254"}, + {0x2C6D, "\311\221"}, + {0x2C6E, "\311\261"}, + {0x2C6F, "\311\220"}, + {0x2C70, "\311\222"}, + {0x2C72, "\342\261\263"}, + {0x2C75, "\342\261\266"}, + {0x2C7E, "\310\277"}, + {0x2C7F, "\311\200"}, + {0x2C80, "\342\262\201"}, + {0x2C82, "\342\262\203"}, + {0x2C84, "\342\262\205"}, + {0x2C86, "\342\262\207"}, + {0x2C88, "\342\262\211"}, + {0x2C8A, "\342\262\213"}, + {0x2C8C, "\342\262\215"}, + {0x2C8E, "\342\262\217"}, + {0x2C90, "\342\262\221"}, + {0x2C92, "\342\262\223"}, + {0x2C94, "\342\262\225"}, + {0x2C96, "\342\262\227"}, + {0x2C98, "\342\262\231"}, + {0x2C9A, "\342\262\233"}, + {0x2C9C, "\342\262\235"}, + {0x2C9E, "\342\262\237"}, + {0x2CA0, "\342\262\241"}, + {0x2CA2, "\342\262\243"}, + {0x2CA4, "\342\262\245"}, + {0x2CA6, "\342\262\247"}, + {0x2CA8, "\342\262\251"}, + {0x2CAA, "\342\262\253"}, + {0x2CAC, "\342\262\255"}, + {0x2CAE, "\342\262\257"}, + {0x2CB0, "\342\262\261"}, + {0x2CB2, "\342\262\263"}, + {0x2CB4, "\342\262\265"}, + {0x2CB6, "\342\262\267"}, + {0x2CB8, "\342\262\271"}, + {0x2CBA, "\342\262\273"}, + {0x2CBC, "\342\262\275"}, + {0x2CBE, "\342\262\277"}, + {0x2CC0, "\342\263\201"}, + {0x2CC2, "\342\263\203"}, + {0x2CC4, "\342\263\205"}, + {0x2CC6, "\342\263\207"}, + {0x2CC8, "\342\263\211"}, + {0x2CCA, "\342\263\213"}, + {0x2CCC, "\342\263\215"}, + {0x2CCE, "\342\263\217"}, + {0x2CD0, "\342\263\221"}, + {0x2CD2, "\342\263\223"}, + {0x2CD4, "\342\263\225"}, + {0x2CD6, "\342\263\227"}, + {0x2CD8, "\342\263\231"}, + {0x2CDA, "\342\263\233"}, + {0x2CDC, "\342\263\235"}, + {0x2CDE, "\342\263\237"}, + {0x2CE0, "\342\263\241"}, + {0x2CE2, "\342\263\243"}, + {0x2CEB, "\342\263\254"}, + {0x2CED, "\342\263\256"}, + {0x2CF2, "\342\263\263"}, + {0xA640, "\352\231\201"}, + {0xA642, "\352\231\203"}, + {0xA644, "\352\231\205"}, + {0xA646, "\352\231\207"}, + {0xA648, "\352\231\211"}, + {0xA64A, "\352\231\213"}, + {0xA64C, "\352\231\215"}, + {0xA64E, "\352\231\217"}, + {0xA650, "\352\231\221"}, + {0xA652, "\352\231\223"}, + {0xA654, "\352\231\225"}, + {0xA656, "\352\231\227"}, + {0xA658, "\352\231\231"}, + {0xA65A, "\352\231\233"}, + {0xA65C, "\352\231\235"}, + {0xA65E, "\352\231\237"}, + {0xA660, "\352\231\241"}, + {0xA662, "\352\231\243"}, + {0xA664, "\352\231\245"}, + {0xA666, "\352\231\247"}, + {0xA668, "\352\231\251"}, + {0xA66A, "\352\231\253"}, + {0xA66C, "\352\231\255"}, + {0xA680, "\352\232\201"}, + {0xA682, "\352\232\203"}, + {0xA684, "\352\232\205"}, + {0xA686, "\352\232\207"}, + {0xA688, "\352\232\211"}, + {0xA68A, "\352\232\213"}, + {0xA68C, "\352\232\215"}, + {0xA68E, "\352\232\217"}, + {0xA690, "\352\232\221"}, + {0xA692, "\352\232\223"}, + {0xA694, "\352\232\225"}, + {0xA696, "\352\232\227"}, + {0xA698, "\352\232\231"}, + {0xA69A, "\352\232\233"}, + {0xA722, "\352\234\243"}, + {0xA724, "\352\234\245"}, + {0xA726, "\352\234\247"}, + {0xA728, "\352\234\251"}, + {0xA72A, "\352\234\253"}, + {0xA72C, "\352\234\255"}, + {0xA72E, "\352\234\257"}, + {0xA732, "\352\234\263"}, + {0xA734, "\352\234\265"}, + {0xA736, "\352\234\267"}, + {0xA738, "\352\234\271"}, + {0xA73A, "\352\234\273"}, + {0xA73C, "\352\234\275"}, + {0xA73E, "\352\234\277"}, + {0xA740, "\352\235\201"}, + {0xA742, "\352\235\203"}, + {0xA744, "\352\235\205"}, + {0xA746, "\352\235\207"}, + {0xA748, "\352\235\211"}, + {0xA74A, "\352\235\213"}, + {0xA74C, "\352\235\215"}, + {0xA74E, "\352\235\217"}, + {0xA750, "\352\235\221"}, + {0xA752, "\352\235\223"}, + {0xA754, "\352\235\225"}, + {0xA756, "\352\235\227"}, + {0xA758, "\352\235\231"}, + {0xA75A, "\352\235\233"}, + {0xA75C, "\352\235\235"}, + {0xA75E, "\352\235\237"}, + {0xA760, "\352\235\241"}, + {0xA762, "\352\235\243"}, + {0xA764, "\352\235\245"}, + {0xA766, "\352\235\247"}, + {0xA768, "\352\235\251"}, + {0xA76A, "\352\235\253"}, + {0xA76C, "\352\235\255"}, + {0xA76E, "\352\235\257"}, + {0xA779, "\352\235\272"}, + {0xA77B, "\352\235\274"}, + {0xA77D, "\341\265\271"}, + {0xA77E, "\352\235\277"}, + {0xA780, "\352\236\201"}, + {0xA782, "\352\236\203"}, + {0xA784, "\352\236\205"}, + {0xA786, "\352\236\207"}, + {0xA78B, "\352\236\214"}, + {0xA78D, "\311\245"}, + {0xA790, "\352\236\221"}, + {0xA792, "\352\236\223"}, + {0xA796, "\352\236\227"}, + {0xA798, "\352\236\231"}, + {0xA79A, "\352\236\233"}, + {0xA79C, "\352\236\235"}, + {0xA79E, "\352\236\237"}, + {0xA7A0, "\352\236\241"}, + {0xA7A2, "\352\236\243"}, + {0xA7A4, "\352\236\245"}, + {0xA7A6, "\352\236\247"}, + {0xA7A8, "\352\236\251"}, + {0xA7AA, "\311\246"}, + {0xA7AB, "\311\234"}, + {0xA7AC, "\311\241"}, + {0xA7AD, "\311\254"}, + {0xA7AE, "\311\252"}, + {0xA7B0, "\312\236"}, + {0xA7B1, "\312\207"}, + {0xA7B2, "\312\235"}, + {0xA7B3, "\352\255\223"}, + {0xA7B4, "\352\236\265"}, + {0xA7B6, "\352\236\267"}, + {0xA7B8, "\352\236\271"}, + {0xA7BA, "\352\236\273"}, + {0xA7BC, "\352\236\275"}, + {0xA7BE, "\352\236\277"}, + {0xA7C0, "\352\237\201"}, + {0xA7C2, "\352\237\203"}, + {0xA7C4, "\352\236\224"}, + {0xA7C5, "\312\202"}, + {0xA7C6, "\341\266\216"}, + {0xA7C7, "\352\237\210"}, + {0xA7C9, "\352\237\212"}, + {0xA7CB, "\311\244"}, + {0xA7CC, "\352\237\215"}, + {0xA7D0, "\352\237\221"}, + {0xA7D6, "\352\237\227"}, + {0xA7D8, "\352\237\231"}, + {0xA7DA, "\352\237\233"}, + {0xA7DC, "\306\233"}, + {0xA7F5, "\352\237\266"}, + {0xFF21, "\357\275\201"}, + {0xFF22, "\357\275\202"}, + {0xFF23, "\357\275\203"}, + {0xFF24, "\357\275\204"}, + {0xFF25, "\357\275\205"}, + {0xFF26, "\357\275\206"}, + {0xFF27, "\357\275\207"}, + {0xFF28, "\357\275\210"}, + {0xFF29, "\357\275\211"}, + {0xFF2A, "\357\275\212"}, + {0xFF2B, "\357\275\213"}, + {0xFF2C, "\357\275\214"}, + {0xFF2D, "\357\275\215"}, + {0xFF2E, "\357\275\216"}, + {0xFF2F, "\357\275\217"}, + {0xFF30, "\357\275\220"}, + {0xFF31, "\357\275\221"}, + {0xFF32, "\357\275\222"}, + {0xFF33, "\357\275\223"}, + {0xFF34, "\357\275\224"}, + {0xFF35, "\357\275\225"}, + {0xFF36, "\357\275\226"}, + {0xFF37, "\357\275\227"}, + {0xFF38, "\357\275\230"}, + {0xFF39, "\357\275\231"}, + {0xFF3A, "\357\275\232"}, + {0x10400, "\360\220\220\250"}, + {0x10401, "\360\220\220\251"}, + {0x10402, "\360\220\220\252"}, + {0x10403, "\360\220\220\253"}, + {0x10404, "\360\220\220\254"}, + {0x10405, "\360\220\220\255"}, + {0x10406, "\360\220\220\256"}, + {0x10407, "\360\220\220\257"}, + {0x10408, "\360\220\220\260"}, + {0x10409, "\360\220\220\261"}, + {0x1040A, "\360\220\220\262"}, + {0x1040B, "\360\220\220\263"}, + {0x1040C, "\360\220\220\264"}, + {0x1040D, "\360\220\220\265"}, + {0x1040E, "\360\220\220\266"}, + {0x1040F, "\360\220\220\267"}, + {0x10410, "\360\220\220\270"}, + {0x10411, "\360\220\220\271"}, + {0x10412, "\360\220\220\272"}, + {0x10413, "\360\220\220\273"}, + {0x10414, "\360\220\220\274"}, + {0x10415, "\360\220\220\275"}, + {0x10416, "\360\220\220\276"}, + {0x10417, "\360\220\220\277"}, + {0x10418, "\360\220\221\200"}, + {0x10419, "\360\220\221\201"}, + {0x1041A, "\360\220\221\202"}, + {0x1041B, "\360\220\221\203"}, + {0x1041C, "\360\220\221\204"}, + {0x1041D, "\360\220\221\205"}, + {0x1041E, "\360\220\221\206"}, + {0x1041F, "\360\220\221\207"}, + {0x10420, "\360\220\221\210"}, + {0x10421, "\360\220\221\211"}, + {0x10422, "\360\220\221\212"}, + {0x10423, "\360\220\221\213"}, + {0x10424, "\360\220\221\214"}, + {0x10425, "\360\220\221\215"}, + {0x10426, "\360\220\221\216"}, + {0x10427, "\360\220\221\217"}, + {0x104B0, "\360\220\223\230"}, + {0x104B1, "\360\220\223\231"}, + {0x104B2, "\360\220\223\232"}, + {0x104B3, "\360\220\223\233"}, + {0x104B4, "\360\220\223\234"}, + {0x104B5, "\360\220\223\235"}, + {0x104B6, "\360\220\223\236"}, + {0x104B7, "\360\220\223\237"}, + {0x104B8, "\360\220\223\240"}, + {0x104B9, "\360\220\223\241"}, + {0x104BA, "\360\220\223\242"}, + {0x104BB, "\360\220\223\243"}, + {0x104BC, "\360\220\223\244"}, + {0x104BD, "\360\220\223\245"}, + {0x104BE, "\360\220\223\246"}, + {0x104BF, "\360\220\223\247"}, + {0x104C0, "\360\220\223\250"}, + {0x104C1, "\360\220\223\251"}, + {0x104C2, "\360\220\223\252"}, + {0x104C3, "\360\220\223\253"}, + {0x104C4, "\360\220\223\254"}, + {0x104C5, "\360\220\223\255"}, + {0x104C6, "\360\220\223\256"}, + {0x104C7, "\360\220\223\257"}, + {0x104C8, "\360\220\223\260"}, + {0x104C9, "\360\220\223\261"}, + {0x104CA, "\360\220\223\262"}, + {0x104CB, "\360\220\223\263"}, + {0x104CC, "\360\220\223\264"}, + {0x104CD, "\360\220\223\265"}, + {0x104CE, "\360\220\223\266"}, + {0x104CF, "\360\220\223\267"}, + {0x104D0, "\360\220\223\270"}, + {0x104D1, "\360\220\223\271"}, + {0x104D2, "\360\220\223\272"}, + {0x104D3, "\360\220\223\273"}, + {0x10570, "\360\220\226\227"}, + {0x10571, "\360\220\226\230"}, + {0x10572, "\360\220\226\231"}, + {0x10573, "\360\220\226\232"}, + {0x10574, "\360\220\226\233"}, + {0x10575, "\360\220\226\234"}, + {0x10576, "\360\220\226\235"}, + {0x10577, "\360\220\226\236"}, + {0x10578, "\360\220\226\237"}, + {0x10579, "\360\220\226\240"}, + {0x1057A, "\360\220\226\241"}, + {0x1057C, "\360\220\226\243"}, + {0x1057D, "\360\220\226\244"}, + {0x1057E, "\360\220\226\245"}, + {0x1057F, "\360\220\226\246"}, + {0x10580, "\360\220\226\247"}, + {0x10581, "\360\220\226\250"}, + {0x10582, "\360\220\226\251"}, + {0x10583, "\360\220\226\252"}, + {0x10584, "\360\220\226\253"}, + {0x10585, "\360\220\226\254"}, + {0x10586, "\360\220\226\255"}, + {0x10587, "\360\220\226\256"}, + {0x10588, "\360\220\226\257"}, + {0x10589, "\360\220\226\260"}, + {0x1058A, "\360\220\226\261"}, + {0x1058C, "\360\220\226\263"}, + {0x1058D, "\360\220\226\264"}, + {0x1058E, "\360\220\226\265"}, + {0x1058F, "\360\220\226\266"}, + {0x10590, "\360\220\226\267"}, + {0x10591, "\360\220\226\270"}, + {0x10592, "\360\220\226\271"}, + {0x10594, "\360\220\226\273"}, + {0x10595, "\360\220\226\274"}, + {0x10C80, "\360\220\263\200"}, + {0x10C81, "\360\220\263\201"}, + {0x10C82, "\360\220\263\202"}, + {0x10C83, "\360\220\263\203"}, + {0x10C84, "\360\220\263\204"}, + {0x10C85, "\360\220\263\205"}, + {0x10C86, "\360\220\263\206"}, + {0x10C87, "\360\220\263\207"}, + {0x10C88, "\360\220\263\210"}, + {0x10C89, "\360\220\263\211"}, + {0x10C8A, "\360\220\263\212"}, + {0x10C8B, "\360\220\263\213"}, + {0x10C8C, "\360\220\263\214"}, + {0x10C8D, "\360\220\263\215"}, + {0x10C8E, "\360\220\263\216"}, + {0x10C8F, "\360\220\263\217"}, + {0x10C90, "\360\220\263\220"}, + {0x10C91, "\360\220\263\221"}, + {0x10C92, "\360\220\263\222"}, + {0x10C93, "\360\220\263\223"}, + {0x10C94, "\360\220\263\224"}, + {0x10C95, "\360\220\263\225"}, + {0x10C96, "\360\220\263\226"}, + {0x10C97, "\360\220\263\227"}, + {0x10C98, "\360\220\263\230"}, + {0x10C99, "\360\220\263\231"}, + {0x10C9A, "\360\220\263\232"}, + {0x10C9B, "\360\220\263\233"}, + {0x10C9C, "\360\220\263\234"}, + {0x10C9D, "\360\220\263\235"}, + {0x10C9E, "\360\220\263\236"}, + {0x10C9F, "\360\220\263\237"}, + {0x10CA0, "\360\220\263\240"}, + {0x10CA1, "\360\220\263\241"}, + {0x10CA2, "\360\220\263\242"}, + {0x10CA3, "\360\220\263\243"}, + {0x10CA4, "\360\220\263\244"}, + {0x10CA5, "\360\220\263\245"}, + {0x10CA6, "\360\220\263\246"}, + {0x10CA7, "\360\220\263\247"}, + {0x10CA8, "\360\220\263\250"}, + {0x10CA9, "\360\220\263\251"}, + {0x10CAA, "\360\220\263\252"}, + {0x10CAB, "\360\220\263\253"}, + {0x10CAC, "\360\220\263\254"}, + {0x10CAD, "\360\220\263\255"}, + {0x10CAE, "\360\220\263\256"}, + {0x10CAF, "\360\220\263\257"}, + {0x10CB0, "\360\220\263\260"}, + {0x10CB1, "\360\220\263\261"}, + {0x10CB2, "\360\220\263\262"}, + {0x10D50, "\360\220\265\260"}, + {0x10D51, "\360\220\265\261"}, + {0x10D52, "\360\220\265\262"}, + {0x10D53, "\360\220\265\263"}, + {0x10D54, "\360\220\265\264"}, + {0x10D55, "\360\220\265\265"}, + {0x10D56, "\360\220\265\266"}, + {0x10D57, "\360\220\265\267"}, + {0x10D58, "\360\220\265\270"}, + {0x10D59, "\360\220\265\271"}, + {0x10D5A, "\360\220\265\272"}, + {0x10D5B, "\360\220\265\273"}, + {0x10D5C, "\360\220\265\274"}, + {0x10D5D, "\360\220\265\275"}, + {0x10D5E, "\360\220\265\276"}, + {0x10D5F, "\360\220\265\277"}, + {0x10D60, "\360\220\266\200"}, + {0x10D61, "\360\220\266\201"}, + {0x10D62, "\360\220\266\202"}, + {0x10D63, "\360\220\266\203"}, + {0x10D64, "\360\220\266\204"}, + {0x10D65, "\360\220\266\205"}, + {0x118A0, "\360\221\243\200"}, + {0x118A1, "\360\221\243\201"}, + {0x118A2, "\360\221\243\202"}, + {0x118A3, "\360\221\243\203"}, + {0x118A4, "\360\221\243\204"}, + {0x118A5, "\360\221\243\205"}, + {0x118A6, "\360\221\243\206"}, + {0x118A7, "\360\221\243\207"}, + {0x118A8, "\360\221\243\210"}, + {0x118A9, "\360\221\243\211"}, + {0x118AA, "\360\221\243\212"}, + {0x118AB, "\360\221\243\213"}, + {0x118AC, "\360\221\243\214"}, + {0x118AD, "\360\221\243\215"}, + {0x118AE, "\360\221\243\216"}, + {0x118AF, "\360\221\243\217"}, + {0x118B0, "\360\221\243\220"}, + {0x118B1, "\360\221\243\221"}, + {0x118B2, "\360\221\243\222"}, + {0x118B3, "\360\221\243\223"}, + {0x118B4, "\360\221\243\224"}, + {0x118B5, "\360\221\243\225"}, + {0x118B6, "\360\221\243\226"}, + {0x118B7, "\360\221\243\227"}, + {0x118B8, "\360\221\243\230"}, + {0x118B9, "\360\221\243\231"}, + {0x118BA, "\360\221\243\232"}, + {0x118BB, "\360\221\243\233"}, + {0x118BC, "\360\221\243\234"}, + {0x118BD, "\360\221\243\235"}, + {0x118BE, "\360\221\243\236"}, + {0x118BF, "\360\221\243\237"}, + {0x16E40, "\360\226\271\240"}, + {0x16E41, "\360\226\271\241"}, + {0x16E42, "\360\226\271\242"}, + {0x16E43, "\360\226\271\243"}, + {0x16E44, "\360\226\271\244"}, + {0x16E45, "\360\226\271\245"}, + {0x16E46, "\360\226\271\246"}, + {0x16E47, "\360\226\271\247"}, + {0x16E48, "\360\226\271\250"}, + {0x16E49, "\360\226\271\251"}, + {0x16E4A, "\360\226\271\252"}, + {0x16E4B, "\360\226\271\253"}, + {0x16E4C, "\360\226\271\254"}, + {0x16E4D, "\360\226\271\255"}, + {0x16E4E, "\360\226\271\256"}, + {0x16E4F, "\360\226\271\257"}, + {0x16E50, "\360\226\271\260"}, + {0x16E51, "\360\226\271\261"}, + {0x16E52, "\360\226\271\262"}, + {0x16E53, "\360\226\271\263"}, + {0x16E54, "\360\226\271\264"}, + {0x16E55, "\360\226\271\265"}, + {0x16E56, "\360\226\271\266"}, + {0x16E57, "\360\226\271\267"}, + {0x16E58, "\360\226\271\270"}, + {0x16E59, "\360\226\271\271"}, + {0x16E5A, "\360\226\271\272"}, + {0x16E5B, "\360\226\271\273"}, + {0x16E5C, "\360\226\271\274"}, + {0x16E5D, "\360\226\271\275"}, + {0x16E5E, "\360\226\271\276"}, + {0x16E5F, "\360\226\271\277"}, + {0x1E900, "\360\236\244\242"}, + {0x1E901, "\360\236\244\243"}, + {0x1E902, "\360\236\244\244"}, + {0x1E903, "\360\236\244\245"}, + {0x1E904, "\360\236\244\246"}, + {0x1E905, "\360\236\244\247"}, + {0x1E906, "\360\236\244\250"}, + {0x1E907, "\360\236\244\251"}, + {0x1E908, "\360\236\244\252"}, + {0x1E909, "\360\236\244\253"}, + {0x1E90A, "\360\236\244\254"}, + {0x1E90B, "\360\236\244\255"}, + {0x1E90C, "\360\236\244\256"}, + {0x1E90D, "\360\236\244\257"}, + {0x1E90E, "\360\236\244\260"}, + {0x1E90F, "\360\236\244\261"}, + {0x1E910, "\360\236\244\262"}, + {0x1E911, "\360\236\244\263"}, + {0x1E912, "\360\236\244\264"}, + {0x1E913, "\360\236\244\265"}, + {0x1E914, "\360\236\244\266"}, + {0x1E915, "\360\236\244\267"}, + {0x1E916, "\360\236\244\270"}, + {0x1E917, "\360\236\244\271"}, + {0x1E918, "\360\236\244\272"}, + {0x1E919, "\360\236\244\273"}, + {0x1E91A, "\360\236\244\274"}, + {0x1E91B, "\360\236\244\275"}, + {0x1E91C, "\360\236\244\276"}, + {0x1E91D, "\360\236\244\277"}, + {0x1E91E, "\360\236\245\200"}, + {0x1E91F, "\360\236\245\201"}, + {0x1E920, "\360\236\245\202"}, + {0x1E921, "\360\236\245\203"}, + }; + return v; +} +const std::vector& UpperMap() { + static const std::vector v = { + {0x61, "A"}, + {0x62, "B"}, + {0x63, "C"}, + {0x64, "D"}, + {0x65, "E"}, + {0x66, "F"}, + {0x67, "G"}, + {0x68, "H"}, + {0x69, "I"}, + {0x6A, "J"}, + {0x6B, "K"}, + {0x6C, "L"}, + {0x6D, "M"}, + {0x6E, "N"}, + {0x6F, "O"}, + {0x70, "P"}, + {0x71, "Q"}, + {0x72, "R"}, + {0x73, "S"}, + {0x74, "T"}, + {0x75, "U"}, + {0x76, "V"}, + {0x77, "W"}, + {0x78, "X"}, + {0x79, "Y"}, + {0x7A, "Z"}, + {0xB5, "\316\234"}, + {0xDF, "SS"}, + {0xE0, "\303\200"}, + {0xE1, "\303\201"}, + {0xE2, "\303\202"}, + {0xE3, "\303\203"}, + {0xE4, "\303\204"}, + {0xE5, "\303\205"}, + {0xE6, "\303\206"}, + {0xE7, "\303\207"}, + {0xE8, "\303\210"}, + {0xE9, "\303\211"}, + {0xEA, "\303\212"}, + {0xEB, "\303\213"}, + {0xEC, "\303\214"}, + {0xED, "\303\215"}, + {0xEE, "\303\216"}, + {0xEF, "\303\217"}, + {0xF0, "\303\220"}, + {0xF1, "\303\221"}, + {0xF2, "\303\222"}, + {0xF3, "\303\223"}, + {0xF4, "\303\224"}, + {0xF5, "\303\225"}, + {0xF6, "\303\226"}, + {0xF8, "\303\230"}, + {0xF9, "\303\231"}, + {0xFA, "\303\232"}, + {0xFB, "\303\233"}, + {0xFC, "\303\234"}, + {0xFD, "\303\235"}, + {0xFE, "\303\236"}, + {0xFF, "\305\270"}, + {0x101, "\304\200"}, + {0x103, "\304\202"}, + {0x105, "\304\204"}, + {0x107, "\304\206"}, + {0x109, "\304\210"}, + {0x10B, "\304\212"}, + {0x10D, "\304\214"}, + {0x10F, "\304\216"}, + {0x111, "\304\220"}, + {0x113, "\304\222"}, + {0x115, "\304\224"}, + {0x117, "\304\226"}, + {0x119, "\304\230"}, + {0x11B, "\304\232"}, + {0x11D, "\304\234"}, + {0x11F, "\304\236"}, + {0x121, "\304\240"}, + {0x123, "\304\242"}, + {0x125, "\304\244"}, + {0x127, "\304\246"}, + {0x129, "\304\250"}, + {0x12B, "\304\252"}, + {0x12D, "\304\254"}, + {0x12F, "\304\256"}, + {0x131, "I"}, + {0x133, "\304\262"}, + {0x135, "\304\264"}, + {0x137, "\304\266"}, + {0x13A, "\304\271"}, + {0x13C, "\304\273"}, + {0x13E, "\304\275"}, + {0x140, "\304\277"}, + {0x142, "\305\201"}, + {0x144, "\305\203"}, + {0x146, "\305\205"}, + {0x148, "\305\207"}, + {0x149, "\312\274N"}, + {0x14B, "\305\212"}, + {0x14D, "\305\214"}, + {0x14F, "\305\216"}, + {0x151, "\305\220"}, + {0x153, "\305\222"}, + {0x155, "\305\224"}, + {0x157, "\305\226"}, + {0x159, "\305\230"}, + {0x15B, "\305\232"}, + {0x15D, "\305\234"}, + {0x15F, "\305\236"}, + {0x161, "\305\240"}, + {0x163, "\305\242"}, + {0x165, "\305\244"}, + {0x167, "\305\246"}, + {0x169, "\305\250"}, + {0x16B, "\305\252"}, + {0x16D, "\305\254"}, + {0x16F, "\305\256"}, + {0x171, "\305\260"}, + {0x173, "\305\262"}, + {0x175, "\305\264"}, + {0x177, "\305\266"}, + {0x17A, "\305\271"}, + {0x17C, "\305\273"}, + {0x17E, "\305\275"}, + {0x17F, "S"}, + {0x180, "\311\203"}, + {0x183, "\306\202"}, + {0x185, "\306\204"}, + {0x188, "\306\207"}, + {0x18C, "\306\213"}, + {0x192, "\306\221"}, + {0x195, "\307\266"}, + {0x199, "\306\230"}, + {0x19A, "\310\275"}, + {0x19B, "\352\237\234"}, + {0x19E, "\310\240"}, + {0x1A1, "\306\240"}, + {0x1A3, "\306\242"}, + {0x1A5, "\306\244"}, + {0x1A8, "\306\247"}, + {0x1AD, "\306\254"}, + {0x1B0, "\306\257"}, + {0x1B4, "\306\263"}, + {0x1B6, "\306\265"}, + {0x1B9, "\306\270"}, + {0x1BD, "\306\274"}, + {0x1BF, "\307\267"}, + {0x1C5, "\307\204"}, + {0x1C6, "\307\204"}, + {0x1C8, "\307\207"}, + {0x1C9, "\307\207"}, + {0x1CB, "\307\212"}, + {0x1CC, "\307\212"}, + {0x1CE, "\307\215"}, + {0x1D0, "\307\217"}, + {0x1D2, "\307\221"}, + {0x1D4, "\307\223"}, + {0x1D6, "\307\225"}, + {0x1D8, "\307\227"}, + {0x1DA, "\307\231"}, + {0x1DC, "\307\233"}, + {0x1DD, "\306\216"}, + {0x1DF, "\307\236"}, + {0x1E1, "\307\240"}, + {0x1E3, "\307\242"}, + {0x1E5, "\307\244"}, + {0x1E7, "\307\246"}, + {0x1E9, "\307\250"}, + {0x1EB, "\307\252"}, + {0x1ED, "\307\254"}, + {0x1EF, "\307\256"}, + {0x1F0, "J\314\214"}, + {0x1F2, "\307\261"}, + {0x1F3, "\307\261"}, + {0x1F5, "\307\264"}, + {0x1F9, "\307\270"}, + {0x1FB, "\307\272"}, + {0x1FD, "\307\274"}, + {0x1FF, "\307\276"}, + {0x201, "\310\200"}, + {0x203, "\310\202"}, + {0x205, "\310\204"}, + {0x207, "\310\206"}, + {0x209, "\310\210"}, + {0x20B, "\310\212"}, + {0x20D, "\310\214"}, + {0x20F, "\310\216"}, + {0x211, "\310\220"}, + {0x213, "\310\222"}, + {0x215, "\310\224"}, + {0x217, "\310\226"}, + {0x219, "\310\230"}, + {0x21B, "\310\232"}, + {0x21D, "\310\234"}, + {0x21F, "\310\236"}, + {0x223, "\310\242"}, + {0x225, "\310\244"}, + {0x227, "\310\246"}, + {0x229, "\310\250"}, + {0x22B, "\310\252"}, + {0x22D, "\310\254"}, + {0x22F, "\310\256"}, + {0x231, "\310\260"}, + {0x233, "\310\262"}, + {0x23C, "\310\273"}, + {0x23F, "\342\261\276"}, + {0x240, "\342\261\277"}, + {0x242, "\311\201"}, + {0x247, "\311\206"}, + {0x249, "\311\210"}, + {0x24B, "\311\212"}, + {0x24D, "\311\214"}, + {0x24F, "\311\216"}, + {0x250, "\342\261\257"}, + {0x251, "\342\261\255"}, + {0x252, "\342\261\260"}, + {0x253, "\306\201"}, + {0x254, "\306\206"}, + {0x256, "\306\211"}, + {0x257, "\306\212"}, + {0x259, "\306\217"}, + {0x25B, "\306\220"}, + {0x25C, "\352\236\253"}, + {0x260, "\306\223"}, + {0x261, "\352\236\254"}, + {0x263, "\306\224"}, + {0x264, "\352\237\213"}, + {0x265, "\352\236\215"}, + {0x266, "\352\236\252"}, + {0x268, "\306\227"}, + {0x269, "\306\226"}, + {0x26A, "\352\236\256"}, + {0x26B, "\342\261\242"}, + {0x26C, "\352\236\255"}, + {0x26F, "\306\234"}, + {0x271, "\342\261\256"}, + {0x272, "\306\235"}, + {0x275, "\306\237"}, + {0x27D, "\342\261\244"}, + {0x280, "\306\246"}, + {0x282, "\352\237\205"}, + {0x283, "\306\251"}, + {0x287, "\352\236\261"}, + {0x288, "\306\256"}, + {0x289, "\311\204"}, + {0x28A, "\306\261"}, + {0x28B, "\306\262"}, + {0x28C, "\311\205"}, + {0x292, "\306\267"}, + {0x29D, "\352\236\262"}, + {0x29E, "\352\236\260"}, + {0x345, "\316\231"}, + {0x371, "\315\260"}, + {0x373, "\315\262"}, + {0x377, "\315\266"}, + {0x37B, "\317\275"}, + {0x37C, "\317\276"}, + {0x37D, "\317\277"}, + {0x390, "\316\231\314\210\314\201"}, + {0x3AC, "\316\206"}, + {0x3AD, "\316\210"}, + {0x3AE, "\316\211"}, + {0x3AF, "\316\212"}, + {0x3B0, "\316\245\314\210\314\201"}, + {0x3B1, "\316\221"}, + {0x3B2, "\316\222"}, + {0x3B3, "\316\223"}, + {0x3B4, "\316\224"}, + {0x3B5, "\316\225"}, + {0x3B6, "\316\226"}, + {0x3B7, "\316\227"}, + {0x3B8, "\316\230"}, + {0x3B9, "\316\231"}, + {0x3BA, "\316\232"}, + {0x3BB, "\316\233"}, + {0x3BC, "\316\234"}, + {0x3BD, "\316\235"}, + {0x3BE, "\316\236"}, + {0x3BF, "\316\237"}, + {0x3C0, "\316\240"}, + {0x3C1, "\316\241"}, + {0x3C2, "\316\243"}, + {0x3C3, "\316\243"}, + {0x3C4, "\316\244"}, + {0x3C5, "\316\245"}, + {0x3C6, "\316\246"}, + {0x3C7, "\316\247"}, + {0x3C8, "\316\250"}, + {0x3C9, "\316\251"}, + {0x3CA, "\316\252"}, + {0x3CB, "\316\253"}, + {0x3CC, "\316\214"}, + {0x3CD, "\316\216"}, + {0x3CE, "\316\217"}, + {0x3D0, "\316\222"}, + {0x3D1, "\316\230"}, + {0x3D5, "\316\246"}, + {0x3D6, "\316\240"}, + {0x3D7, "\317\217"}, + {0x3D9, "\317\230"}, + {0x3DB, "\317\232"}, + {0x3DD, "\317\234"}, + {0x3DF, "\317\236"}, + {0x3E1, "\317\240"}, + {0x3E3, "\317\242"}, + {0x3E5, "\317\244"}, + {0x3E7, "\317\246"}, + {0x3E9, "\317\250"}, + {0x3EB, "\317\252"}, + {0x3ED, "\317\254"}, + {0x3EF, "\317\256"}, + {0x3F0, "\316\232"}, + {0x3F1, "\316\241"}, + {0x3F2, "\317\271"}, + {0x3F3, "\315\277"}, + {0x3F5, "\316\225"}, + {0x3F8, "\317\267"}, + {0x3FB, "\317\272"}, + {0x430, "\320\220"}, + {0x431, "\320\221"}, + {0x432, "\320\222"}, + {0x433, "\320\223"}, + {0x434, "\320\224"}, + {0x435, "\320\225"}, + {0x436, "\320\226"}, + {0x437, "\320\227"}, + {0x438, "\320\230"}, + {0x439, "\320\231"}, + {0x43A, "\320\232"}, + {0x43B, "\320\233"}, + {0x43C, "\320\234"}, + {0x43D, "\320\235"}, + {0x43E, "\320\236"}, + {0x43F, "\320\237"}, + {0x440, "\320\240"}, + {0x441, "\320\241"}, + {0x442, "\320\242"}, + {0x443, "\320\243"}, + {0x444, "\320\244"}, + {0x445, "\320\245"}, + {0x446, "\320\246"}, + {0x447, "\320\247"}, + {0x448, "\320\250"}, + {0x449, "\320\251"}, + {0x44A, "\320\252"}, + {0x44B, "\320\253"}, + {0x44C, "\320\254"}, + {0x44D, "\320\255"}, + {0x44E, "\320\256"}, + {0x44F, "\320\257"}, + {0x450, "\320\200"}, + {0x451, "\320\201"}, + {0x452, "\320\202"}, + {0x453, "\320\203"}, + {0x454, "\320\204"}, + {0x455, "\320\205"}, + {0x456, "\320\206"}, + {0x457, "\320\207"}, + {0x458, "\320\210"}, + {0x459, "\320\211"}, + {0x45A, "\320\212"}, + {0x45B, "\320\213"}, + {0x45C, "\320\214"}, + {0x45D, "\320\215"}, + {0x45E, "\320\216"}, + {0x45F, "\320\217"}, + {0x461, "\321\240"}, + {0x463, "\321\242"}, + {0x465, "\321\244"}, + {0x467, "\321\246"}, + {0x469, "\321\250"}, + {0x46B, "\321\252"}, + {0x46D, "\321\254"}, + {0x46F, "\321\256"}, + {0x471, "\321\260"}, + {0x473, "\321\262"}, + {0x475, "\321\264"}, + {0x477, "\321\266"}, + {0x479, "\321\270"}, + {0x47B, "\321\272"}, + {0x47D, "\321\274"}, + {0x47F, "\321\276"}, + {0x481, "\322\200"}, + {0x48B, "\322\212"}, + {0x48D, "\322\214"}, + {0x48F, "\322\216"}, + {0x491, "\322\220"}, + {0x493, "\322\222"}, + {0x495, "\322\224"}, + {0x497, "\322\226"}, + {0x499, "\322\230"}, + {0x49B, "\322\232"}, + {0x49D, "\322\234"}, + {0x49F, "\322\236"}, + {0x4A1, "\322\240"}, + {0x4A3, "\322\242"}, + {0x4A5, "\322\244"}, + {0x4A7, "\322\246"}, + {0x4A9, "\322\250"}, + {0x4AB, "\322\252"}, + {0x4AD, "\322\254"}, + {0x4AF, "\322\256"}, + {0x4B1, "\322\260"}, + {0x4B3, "\322\262"}, + {0x4B5, "\322\264"}, + {0x4B7, "\322\266"}, + {0x4B9, "\322\270"}, + {0x4BB, "\322\272"}, + {0x4BD, "\322\274"}, + {0x4BF, "\322\276"}, + {0x4C2, "\323\201"}, + {0x4C4, "\323\203"}, + {0x4C6, "\323\205"}, + {0x4C8, "\323\207"}, + {0x4CA, "\323\211"}, + {0x4CC, "\323\213"}, + {0x4CE, "\323\215"}, + {0x4CF, "\323\200"}, + {0x4D1, "\323\220"}, + {0x4D3, "\323\222"}, + {0x4D5, "\323\224"}, + {0x4D7, "\323\226"}, + {0x4D9, "\323\230"}, + {0x4DB, "\323\232"}, + {0x4DD, "\323\234"}, + {0x4DF, "\323\236"}, + {0x4E1, "\323\240"}, + {0x4E3, "\323\242"}, + {0x4E5, "\323\244"}, + {0x4E7, "\323\246"}, + {0x4E9, "\323\250"}, + {0x4EB, "\323\252"}, + {0x4ED, "\323\254"}, + {0x4EF, "\323\256"}, + {0x4F1, "\323\260"}, + {0x4F3, "\323\262"}, + {0x4F5, "\323\264"}, + {0x4F7, "\323\266"}, + {0x4F9, "\323\270"}, + {0x4FB, "\323\272"}, + {0x4FD, "\323\274"}, + {0x4FF, "\323\276"}, + {0x501, "\324\200"}, + {0x503, "\324\202"}, + {0x505, "\324\204"}, + {0x507, "\324\206"}, + {0x509, "\324\210"}, + {0x50B, "\324\212"}, + {0x50D, "\324\214"}, + {0x50F, "\324\216"}, + {0x511, "\324\220"}, + {0x513, "\324\222"}, + {0x515, "\324\224"}, + {0x517, "\324\226"}, + {0x519, "\324\230"}, + {0x51B, "\324\232"}, + {0x51D, "\324\234"}, + {0x51F, "\324\236"}, + {0x521, "\324\240"}, + {0x523, "\324\242"}, + {0x525, "\324\244"}, + {0x527, "\324\246"}, + {0x529, "\324\250"}, + {0x52B, "\324\252"}, + {0x52D, "\324\254"}, + {0x52F, "\324\256"}, + {0x561, "\324\261"}, + {0x562, "\324\262"}, + {0x563, "\324\263"}, + {0x564, "\324\264"}, + {0x565, "\324\265"}, + {0x566, "\324\266"}, + {0x567, "\324\267"}, + {0x568, "\324\270"}, + {0x569, "\324\271"}, + {0x56A, "\324\272"}, + {0x56B, "\324\273"}, + {0x56C, "\324\274"}, + {0x56D, "\324\275"}, + {0x56E, "\324\276"}, + {0x56F, "\324\277"}, + {0x570, "\325\200"}, + {0x571, "\325\201"}, + {0x572, "\325\202"}, + {0x573, "\325\203"}, + {0x574, "\325\204"}, + {0x575, "\325\205"}, + {0x576, "\325\206"}, + {0x577, "\325\207"}, + {0x578, "\325\210"}, + {0x579, "\325\211"}, + {0x57A, "\325\212"}, + {0x57B, "\325\213"}, + {0x57C, "\325\214"}, + {0x57D, "\325\215"}, + {0x57E, "\325\216"}, + {0x57F, "\325\217"}, + {0x580, "\325\220"}, + {0x581, "\325\221"}, + {0x582, "\325\222"}, + {0x583, "\325\223"}, + {0x584, "\325\224"}, + {0x585, "\325\225"}, + {0x586, "\325\226"}, + {0x587, "\324\265\325\222"}, + {0x10D0, "\341\262\220"}, + {0x10D1, "\341\262\221"}, + {0x10D2, "\341\262\222"}, + {0x10D3, "\341\262\223"}, + {0x10D4, "\341\262\224"}, + {0x10D5, "\341\262\225"}, + {0x10D6, "\341\262\226"}, + {0x10D7, "\341\262\227"}, + {0x10D8, "\341\262\230"}, + {0x10D9, "\341\262\231"}, + {0x10DA, "\341\262\232"}, + {0x10DB, "\341\262\233"}, + {0x10DC, "\341\262\234"}, + {0x10DD, "\341\262\235"}, + {0x10DE, "\341\262\236"}, + {0x10DF, "\341\262\237"}, + {0x10E0, "\341\262\240"}, + {0x10E1, "\341\262\241"}, + {0x10E2, "\341\262\242"}, + {0x10E3, "\341\262\243"}, + {0x10E4, "\341\262\244"}, + {0x10E5, "\341\262\245"}, + {0x10E6, "\341\262\246"}, + {0x10E7, "\341\262\247"}, + {0x10E8, "\341\262\250"}, + {0x10E9, "\341\262\251"}, + {0x10EA, "\341\262\252"}, + {0x10EB, "\341\262\253"}, + {0x10EC, "\341\262\254"}, + {0x10ED, "\341\262\255"}, + {0x10EE, "\341\262\256"}, + {0x10EF, "\341\262\257"}, + {0x10F0, "\341\262\260"}, + {0x10F1, "\341\262\261"}, + {0x10F2, "\341\262\262"}, + {0x10F3, "\341\262\263"}, + {0x10F4, "\341\262\264"}, + {0x10F5, "\341\262\265"}, + {0x10F6, "\341\262\266"}, + {0x10F7, "\341\262\267"}, + {0x10F8, "\341\262\270"}, + {0x10F9, "\341\262\271"}, + {0x10FA, "\341\262\272"}, + {0x10FD, "\341\262\275"}, + {0x10FE, "\341\262\276"}, + {0x10FF, "\341\262\277"}, + {0x13F8, "\341\217\260"}, + {0x13F9, "\341\217\261"}, + {0x13FA, "\341\217\262"}, + {0x13FB, "\341\217\263"}, + {0x13FC, "\341\217\264"}, + {0x13FD, "\341\217\265"}, + {0x1C80, "\320\222"}, + {0x1C81, "\320\224"}, + {0x1C82, "\320\236"}, + {0x1C83, "\320\241"}, + {0x1C84, "\320\242"}, + {0x1C85, "\320\242"}, + {0x1C86, "\320\252"}, + {0x1C87, "\321\242"}, + {0x1C88, "\352\231\212"}, + {0x1C8A, "\341\262\211"}, + {0x1D79, "\352\235\275"}, + {0x1D7D, "\342\261\243"}, + {0x1D8E, "\352\237\206"}, + {0x1E01, "\341\270\200"}, + {0x1E03, "\341\270\202"}, + {0x1E05, "\341\270\204"}, + {0x1E07, "\341\270\206"}, + {0x1E09, "\341\270\210"}, + {0x1E0B, "\341\270\212"}, + {0x1E0D, "\341\270\214"}, + {0x1E0F, "\341\270\216"}, + {0x1E11, "\341\270\220"}, + {0x1E13, "\341\270\222"}, + {0x1E15, "\341\270\224"}, + {0x1E17, "\341\270\226"}, + {0x1E19, "\341\270\230"}, + {0x1E1B, "\341\270\232"}, + {0x1E1D, "\341\270\234"}, + {0x1E1F, "\341\270\236"}, + {0x1E21, "\341\270\240"}, + {0x1E23, "\341\270\242"}, + {0x1E25, "\341\270\244"}, + {0x1E27, "\341\270\246"}, + {0x1E29, "\341\270\250"}, + {0x1E2B, "\341\270\252"}, + {0x1E2D, "\341\270\254"}, + {0x1E2F, "\341\270\256"}, + {0x1E31, "\341\270\260"}, + {0x1E33, "\341\270\262"}, + {0x1E35, "\341\270\264"}, + {0x1E37, "\341\270\266"}, + {0x1E39, "\341\270\270"}, + {0x1E3B, "\341\270\272"}, + {0x1E3D, "\341\270\274"}, + {0x1E3F, "\341\270\276"}, + {0x1E41, "\341\271\200"}, + {0x1E43, "\341\271\202"}, + {0x1E45, "\341\271\204"}, + {0x1E47, "\341\271\206"}, + {0x1E49, "\341\271\210"}, + {0x1E4B, "\341\271\212"}, + {0x1E4D, "\341\271\214"}, + {0x1E4F, "\341\271\216"}, + {0x1E51, "\341\271\220"}, + {0x1E53, "\341\271\222"}, + {0x1E55, "\341\271\224"}, + {0x1E57, "\341\271\226"}, + {0x1E59, "\341\271\230"}, + {0x1E5B, "\341\271\232"}, + {0x1E5D, "\341\271\234"}, + {0x1E5F, "\341\271\236"}, + {0x1E61, "\341\271\240"}, + {0x1E63, "\341\271\242"}, + {0x1E65, "\341\271\244"}, + {0x1E67, "\341\271\246"}, + {0x1E69, "\341\271\250"}, + {0x1E6B, "\341\271\252"}, + {0x1E6D, "\341\271\254"}, + {0x1E6F, "\341\271\256"}, + {0x1E71, "\341\271\260"}, + {0x1E73, "\341\271\262"}, + {0x1E75, "\341\271\264"}, + {0x1E77, "\341\271\266"}, + {0x1E79, "\341\271\270"}, + {0x1E7B, "\341\271\272"}, + {0x1E7D, "\341\271\274"}, + {0x1E7F, "\341\271\276"}, + {0x1E81, "\341\272\200"}, + {0x1E83, "\341\272\202"}, + {0x1E85, "\341\272\204"}, + {0x1E87, "\341\272\206"}, + {0x1E89, "\341\272\210"}, + {0x1E8B, "\341\272\212"}, + {0x1E8D, "\341\272\214"}, + {0x1E8F, "\341\272\216"}, + {0x1E91, "\341\272\220"}, + {0x1E93, "\341\272\222"}, + {0x1E95, "\341\272\224"}, + {0x1E96, "H\314\261"}, + {0x1E97, "T\314\210"}, + {0x1E98, "W\314\212"}, + {0x1E99, "Y\314\212"}, + {0x1E9A, "A\312\276"}, + {0x1E9B, "\341\271\240"}, + {0x1EA1, "\341\272\240"}, + {0x1EA3, "\341\272\242"}, + {0x1EA5, "\341\272\244"}, + {0x1EA7, "\341\272\246"}, + {0x1EA9, "\341\272\250"}, + {0x1EAB, "\341\272\252"}, + {0x1EAD, "\341\272\254"}, + {0x1EAF, "\341\272\256"}, + {0x1EB1, "\341\272\260"}, + {0x1EB3, "\341\272\262"}, + {0x1EB5, "\341\272\264"}, + {0x1EB7, "\341\272\266"}, + {0x1EB9, "\341\272\270"}, + {0x1EBB, "\341\272\272"}, + {0x1EBD, "\341\272\274"}, + {0x1EBF, "\341\272\276"}, + {0x1EC1, "\341\273\200"}, + {0x1EC3, "\341\273\202"}, + {0x1EC5, "\341\273\204"}, + {0x1EC7, "\341\273\206"}, + {0x1EC9, "\341\273\210"}, + {0x1ECB, "\341\273\212"}, + {0x1ECD, "\341\273\214"}, + {0x1ECF, "\341\273\216"}, + {0x1ED1, "\341\273\220"}, + {0x1ED3, "\341\273\222"}, + {0x1ED5, "\341\273\224"}, + {0x1ED7, "\341\273\226"}, + {0x1ED9, "\341\273\230"}, + {0x1EDB, "\341\273\232"}, + {0x1EDD, "\341\273\234"}, + {0x1EDF, "\341\273\236"}, + {0x1EE1, "\341\273\240"}, + {0x1EE3, "\341\273\242"}, + {0x1EE5, "\341\273\244"}, + {0x1EE7, "\341\273\246"}, + {0x1EE9, "\341\273\250"}, + {0x1EEB, "\341\273\252"}, + {0x1EED, "\341\273\254"}, + {0x1EEF, "\341\273\256"}, + {0x1EF1, "\341\273\260"}, + {0x1EF3, "\341\273\262"}, + {0x1EF5, "\341\273\264"}, + {0x1EF7, "\341\273\266"}, + {0x1EF9, "\341\273\270"}, + {0x1EFB, "\341\273\272"}, + {0x1EFD, "\341\273\274"}, + {0x1EFF, "\341\273\276"}, + {0x1F00, "\341\274\210"}, + {0x1F01, "\341\274\211"}, + {0x1F02, "\341\274\212"}, + {0x1F03, "\341\274\213"}, + {0x1F04, "\341\274\214"}, + {0x1F05, "\341\274\215"}, + {0x1F06, "\341\274\216"}, + {0x1F07, "\341\274\217"}, + {0x1F10, "\341\274\230"}, + {0x1F11, "\341\274\231"}, + {0x1F12, "\341\274\232"}, + {0x1F13, "\341\274\233"}, + {0x1F14, "\341\274\234"}, + {0x1F15, "\341\274\235"}, + {0x1F20, "\341\274\250"}, + {0x1F21, "\341\274\251"}, + {0x1F22, "\341\274\252"}, + {0x1F23, "\341\274\253"}, + {0x1F24, "\341\274\254"}, + {0x1F25, "\341\274\255"}, + {0x1F26, "\341\274\256"}, + {0x1F27, "\341\274\257"}, + {0x1F30, "\341\274\270"}, + {0x1F31, "\341\274\271"}, + {0x1F32, "\341\274\272"}, + {0x1F33, "\341\274\273"}, + {0x1F34, "\341\274\274"}, + {0x1F35, "\341\274\275"}, + {0x1F36, "\341\274\276"}, + {0x1F37, "\341\274\277"}, + {0x1F40, "\341\275\210"}, + {0x1F41, "\341\275\211"}, + {0x1F42, "\341\275\212"}, + {0x1F43, "\341\275\213"}, + {0x1F44, "\341\275\214"}, + {0x1F45, "\341\275\215"}, + {0x1F50, "\316\245\314\223"}, + {0x1F51, "\341\275\231"}, + {0x1F52, "\316\245\314\223\314\200"}, + {0x1F53, "\341\275\233"}, + {0x1F54, "\316\245\314\223\314\201"}, + {0x1F55, "\341\275\235"}, + {0x1F56, "\316\245\314\223\315\202"}, + {0x1F57, "\341\275\237"}, + {0x1F60, "\341\275\250"}, + {0x1F61, "\341\275\251"}, + {0x1F62, "\341\275\252"}, + {0x1F63, "\341\275\253"}, + {0x1F64, "\341\275\254"}, + {0x1F65, "\341\275\255"}, + {0x1F66, "\341\275\256"}, + {0x1F67, "\341\275\257"}, + {0x1F70, "\341\276\272"}, + {0x1F71, "\341\276\273"}, + {0x1F72, "\341\277\210"}, + {0x1F73, "\341\277\211"}, + {0x1F74, "\341\277\212"}, + {0x1F75, "\341\277\213"}, + {0x1F76, "\341\277\232"}, + {0x1F77, "\341\277\233"}, + {0x1F78, "\341\277\270"}, + {0x1F79, "\341\277\271"}, + {0x1F7A, "\341\277\252"}, + {0x1F7B, "\341\277\253"}, + {0x1F7C, "\341\277\272"}, + {0x1F7D, "\341\277\273"}, + {0x1F80, "\341\274\210\316\231"}, + {0x1F81, "\341\274\211\316\231"}, + {0x1F82, "\341\274\212\316\231"}, + {0x1F83, "\341\274\213\316\231"}, + {0x1F84, "\341\274\214\316\231"}, + {0x1F85, "\341\274\215\316\231"}, + {0x1F86, "\341\274\216\316\231"}, + {0x1F87, "\341\274\217\316\231"}, + {0x1F88, "\341\274\210\316\231"}, + {0x1F89, "\341\274\211\316\231"}, + {0x1F8A, "\341\274\212\316\231"}, + {0x1F8B, "\341\274\213\316\231"}, + {0x1F8C, "\341\274\214\316\231"}, + {0x1F8D, "\341\274\215\316\231"}, + {0x1F8E, "\341\274\216\316\231"}, + {0x1F8F, "\341\274\217\316\231"}, + {0x1F90, "\341\274\250\316\231"}, + {0x1F91, "\341\274\251\316\231"}, + {0x1F92, "\341\274\252\316\231"}, + {0x1F93, "\341\274\253\316\231"}, + {0x1F94, "\341\274\254\316\231"}, + {0x1F95, "\341\274\255\316\231"}, + {0x1F96, "\341\274\256\316\231"}, + {0x1F97, "\341\274\257\316\231"}, + {0x1F98, "\341\274\250\316\231"}, + {0x1F99, "\341\274\251\316\231"}, + {0x1F9A, "\341\274\252\316\231"}, + {0x1F9B, "\341\274\253\316\231"}, + {0x1F9C, "\341\274\254\316\231"}, + {0x1F9D, "\341\274\255\316\231"}, + {0x1F9E, "\341\274\256\316\231"}, + {0x1F9F, "\341\274\257\316\231"}, + {0x1FA0, "\341\275\250\316\231"}, + {0x1FA1, "\341\275\251\316\231"}, + {0x1FA2, "\341\275\252\316\231"}, + {0x1FA3, "\341\275\253\316\231"}, + {0x1FA4, "\341\275\254\316\231"}, + {0x1FA5, "\341\275\255\316\231"}, + {0x1FA6, "\341\275\256\316\231"}, + {0x1FA7, "\341\275\257\316\231"}, + {0x1FA8, "\341\275\250\316\231"}, + {0x1FA9, "\341\275\251\316\231"}, + {0x1FAA, "\341\275\252\316\231"}, + {0x1FAB, "\341\275\253\316\231"}, + {0x1FAC, "\341\275\254\316\231"}, + {0x1FAD, "\341\275\255\316\231"}, + {0x1FAE, "\341\275\256\316\231"}, + {0x1FAF, "\341\275\257\316\231"}, + {0x1FB0, "\341\276\270"}, + {0x1FB1, "\341\276\271"}, + {0x1FB2, "\341\276\272\316\231"}, + {0x1FB3, "\316\221\316\231"}, + {0x1FB4, "\316\206\316\231"}, + {0x1FB6, "\316\221\315\202"}, + {0x1FB7, "\316\221\315\202\316\231"}, + {0x1FBC, "\316\221\316\231"}, + {0x1FBE, "\316\231"}, + {0x1FC2, "\341\277\212\316\231"}, + {0x1FC3, "\316\227\316\231"}, + {0x1FC4, "\316\211\316\231"}, + {0x1FC6, "\316\227\315\202"}, + {0x1FC7, "\316\227\315\202\316\231"}, + {0x1FCC, "\316\227\316\231"}, + {0x1FD0, "\341\277\230"}, + {0x1FD1, "\341\277\231"}, + {0x1FD2, "\316\231\314\210\314\200"}, + {0x1FD3, "\316\231\314\210\314\201"}, + {0x1FD6, "\316\231\315\202"}, + {0x1FD7, "\316\231\314\210\315\202"}, + {0x1FE0, "\341\277\250"}, + {0x1FE1, "\341\277\251"}, + {0x1FE2, "\316\245\314\210\314\200"}, + {0x1FE3, "\316\245\314\210\314\201"}, + {0x1FE4, "\316\241\314\223"}, + {0x1FE5, "\341\277\254"}, + {0x1FE6, "\316\245\315\202"}, + {0x1FE7, "\316\245\314\210\315\202"}, + {0x1FF2, "\341\277\272\316\231"}, + {0x1FF3, "\316\251\316\231"}, + {0x1FF4, "\316\217\316\231"}, + {0x1FF6, "\316\251\315\202"}, + {0x1FF7, "\316\251\315\202\316\231"}, + {0x1FFC, "\316\251\316\231"}, + {0x214E, "\342\204\262"}, + {0x2170, "\342\205\240"}, + {0x2171, "\342\205\241"}, + {0x2172, "\342\205\242"}, + {0x2173, "\342\205\243"}, + {0x2174, "\342\205\244"}, + {0x2175, "\342\205\245"}, + {0x2176, "\342\205\246"}, + {0x2177, "\342\205\247"}, + {0x2178, "\342\205\250"}, + {0x2179, "\342\205\251"}, + {0x217A, "\342\205\252"}, + {0x217B, "\342\205\253"}, + {0x217C, "\342\205\254"}, + {0x217D, "\342\205\255"}, + {0x217E, "\342\205\256"}, + {0x217F, "\342\205\257"}, + {0x2184, "\342\206\203"}, + {0x24D0, "\342\222\266"}, + {0x24D1, "\342\222\267"}, + {0x24D2, "\342\222\270"}, + {0x24D3, "\342\222\271"}, + {0x24D4, "\342\222\272"}, + {0x24D5, "\342\222\273"}, + {0x24D6, "\342\222\274"}, + {0x24D7, "\342\222\275"}, + {0x24D8, "\342\222\276"}, + {0x24D9, "\342\222\277"}, + {0x24DA, "\342\223\200"}, + {0x24DB, "\342\223\201"}, + {0x24DC, "\342\223\202"}, + {0x24DD, "\342\223\203"}, + {0x24DE, "\342\223\204"}, + {0x24DF, "\342\223\205"}, + {0x24E0, "\342\223\206"}, + {0x24E1, "\342\223\207"}, + {0x24E2, "\342\223\210"}, + {0x24E3, "\342\223\211"}, + {0x24E4, "\342\223\212"}, + {0x24E5, "\342\223\213"}, + {0x24E6, "\342\223\214"}, + {0x24E7, "\342\223\215"}, + {0x24E8, "\342\223\216"}, + {0x24E9, "\342\223\217"}, + {0x2C30, "\342\260\200"}, + {0x2C31, "\342\260\201"}, + {0x2C32, "\342\260\202"}, + {0x2C33, "\342\260\203"}, + {0x2C34, "\342\260\204"}, + {0x2C35, "\342\260\205"}, + {0x2C36, "\342\260\206"}, + {0x2C37, "\342\260\207"}, + {0x2C38, "\342\260\210"}, + {0x2C39, "\342\260\211"}, + {0x2C3A, "\342\260\212"}, + {0x2C3B, "\342\260\213"}, + {0x2C3C, "\342\260\214"}, + {0x2C3D, "\342\260\215"}, + {0x2C3E, "\342\260\216"}, + {0x2C3F, "\342\260\217"}, + {0x2C40, "\342\260\220"}, + {0x2C41, "\342\260\221"}, + {0x2C42, "\342\260\222"}, + {0x2C43, "\342\260\223"}, + {0x2C44, "\342\260\224"}, + {0x2C45, "\342\260\225"}, + {0x2C46, "\342\260\226"}, + {0x2C47, "\342\260\227"}, + {0x2C48, "\342\260\230"}, + {0x2C49, "\342\260\231"}, + {0x2C4A, "\342\260\232"}, + {0x2C4B, "\342\260\233"}, + {0x2C4C, "\342\260\234"}, + {0x2C4D, "\342\260\235"}, + {0x2C4E, "\342\260\236"}, + {0x2C4F, "\342\260\237"}, + {0x2C50, "\342\260\240"}, + {0x2C51, "\342\260\241"}, + {0x2C52, "\342\260\242"}, + {0x2C53, "\342\260\243"}, + {0x2C54, "\342\260\244"}, + {0x2C55, "\342\260\245"}, + {0x2C56, "\342\260\246"}, + {0x2C57, "\342\260\247"}, + {0x2C58, "\342\260\250"}, + {0x2C59, "\342\260\251"}, + {0x2C5A, "\342\260\252"}, + {0x2C5B, "\342\260\253"}, + {0x2C5C, "\342\260\254"}, + {0x2C5D, "\342\260\255"}, + {0x2C5E, "\342\260\256"}, + {0x2C5F, "\342\260\257"}, + {0x2C61, "\342\261\240"}, + {0x2C65, "\310\272"}, + {0x2C66, "\310\276"}, + {0x2C68, "\342\261\247"}, + {0x2C6A, "\342\261\251"}, + {0x2C6C, "\342\261\253"}, + {0x2C73, "\342\261\262"}, + {0x2C76, "\342\261\265"}, + {0x2C81, "\342\262\200"}, + {0x2C83, "\342\262\202"}, + {0x2C85, "\342\262\204"}, + {0x2C87, "\342\262\206"}, + {0x2C89, "\342\262\210"}, + {0x2C8B, "\342\262\212"}, + {0x2C8D, "\342\262\214"}, + {0x2C8F, "\342\262\216"}, + {0x2C91, "\342\262\220"}, + {0x2C93, "\342\262\222"}, + {0x2C95, "\342\262\224"}, + {0x2C97, "\342\262\226"}, + {0x2C99, "\342\262\230"}, + {0x2C9B, "\342\262\232"}, + {0x2C9D, "\342\262\234"}, + {0x2C9F, "\342\262\236"}, + {0x2CA1, "\342\262\240"}, + {0x2CA3, "\342\262\242"}, + {0x2CA5, "\342\262\244"}, + {0x2CA7, "\342\262\246"}, + {0x2CA9, "\342\262\250"}, + {0x2CAB, "\342\262\252"}, + {0x2CAD, "\342\262\254"}, + {0x2CAF, "\342\262\256"}, + {0x2CB1, "\342\262\260"}, + {0x2CB3, "\342\262\262"}, + {0x2CB5, "\342\262\264"}, + {0x2CB7, "\342\262\266"}, + {0x2CB9, "\342\262\270"}, + {0x2CBB, "\342\262\272"}, + {0x2CBD, "\342\262\274"}, + {0x2CBF, "\342\262\276"}, + {0x2CC1, "\342\263\200"}, + {0x2CC3, "\342\263\202"}, + {0x2CC5, "\342\263\204"}, + {0x2CC7, "\342\263\206"}, + {0x2CC9, "\342\263\210"}, + {0x2CCB, "\342\263\212"}, + {0x2CCD, "\342\263\214"}, + {0x2CCF, "\342\263\216"}, + {0x2CD1, "\342\263\220"}, + {0x2CD3, "\342\263\222"}, + {0x2CD5, "\342\263\224"}, + {0x2CD7, "\342\263\226"}, + {0x2CD9, "\342\263\230"}, + {0x2CDB, "\342\263\232"}, + {0x2CDD, "\342\263\234"}, + {0x2CDF, "\342\263\236"}, + {0x2CE1, "\342\263\240"}, + {0x2CE3, "\342\263\242"}, + {0x2CEC, "\342\263\253"}, + {0x2CEE, "\342\263\255"}, + {0x2CF3, "\342\263\262"}, + {0x2D00, "\341\202\240"}, + {0x2D01, "\341\202\241"}, + {0x2D02, "\341\202\242"}, + {0x2D03, "\341\202\243"}, + {0x2D04, "\341\202\244"}, + {0x2D05, "\341\202\245"}, + {0x2D06, "\341\202\246"}, + {0x2D07, "\341\202\247"}, + {0x2D08, "\341\202\250"}, + {0x2D09, "\341\202\251"}, + {0x2D0A, "\341\202\252"}, + {0x2D0B, "\341\202\253"}, + {0x2D0C, "\341\202\254"}, + {0x2D0D, "\341\202\255"}, + {0x2D0E, "\341\202\256"}, + {0x2D0F, "\341\202\257"}, + {0x2D10, "\341\202\260"}, + {0x2D11, "\341\202\261"}, + {0x2D12, "\341\202\262"}, + {0x2D13, "\341\202\263"}, + {0x2D14, "\341\202\264"}, + {0x2D15, "\341\202\265"}, + {0x2D16, "\341\202\266"}, + {0x2D17, "\341\202\267"}, + {0x2D18, "\341\202\270"}, + {0x2D19, "\341\202\271"}, + {0x2D1A, "\341\202\272"}, + {0x2D1B, "\341\202\273"}, + {0x2D1C, "\341\202\274"}, + {0x2D1D, "\341\202\275"}, + {0x2D1E, "\341\202\276"}, + {0x2D1F, "\341\202\277"}, + {0x2D20, "\341\203\200"}, + {0x2D21, "\341\203\201"}, + {0x2D22, "\341\203\202"}, + {0x2D23, "\341\203\203"}, + {0x2D24, "\341\203\204"}, + {0x2D25, "\341\203\205"}, + {0x2D27, "\341\203\207"}, + {0x2D2D, "\341\203\215"}, + {0xA641, "\352\231\200"}, + {0xA643, "\352\231\202"}, + {0xA645, "\352\231\204"}, + {0xA647, "\352\231\206"}, + {0xA649, "\352\231\210"}, + {0xA64B, "\352\231\212"}, + {0xA64D, "\352\231\214"}, + {0xA64F, "\352\231\216"}, + {0xA651, "\352\231\220"}, + {0xA653, "\352\231\222"}, + {0xA655, "\352\231\224"}, + {0xA657, "\352\231\226"}, + {0xA659, "\352\231\230"}, + {0xA65B, "\352\231\232"}, + {0xA65D, "\352\231\234"}, + {0xA65F, "\352\231\236"}, + {0xA661, "\352\231\240"}, + {0xA663, "\352\231\242"}, + {0xA665, "\352\231\244"}, + {0xA667, "\352\231\246"}, + {0xA669, "\352\231\250"}, + {0xA66B, "\352\231\252"}, + {0xA66D, "\352\231\254"}, + {0xA681, "\352\232\200"}, + {0xA683, "\352\232\202"}, + {0xA685, "\352\232\204"}, + {0xA687, "\352\232\206"}, + {0xA689, "\352\232\210"}, + {0xA68B, "\352\232\212"}, + {0xA68D, "\352\232\214"}, + {0xA68F, "\352\232\216"}, + {0xA691, "\352\232\220"}, + {0xA693, "\352\232\222"}, + {0xA695, "\352\232\224"}, + {0xA697, "\352\232\226"}, + {0xA699, "\352\232\230"}, + {0xA69B, "\352\232\232"}, + {0xA723, "\352\234\242"}, + {0xA725, "\352\234\244"}, + {0xA727, "\352\234\246"}, + {0xA729, "\352\234\250"}, + {0xA72B, "\352\234\252"}, + {0xA72D, "\352\234\254"}, + {0xA72F, "\352\234\256"}, + {0xA733, "\352\234\262"}, + {0xA735, "\352\234\264"}, + {0xA737, "\352\234\266"}, + {0xA739, "\352\234\270"}, + {0xA73B, "\352\234\272"}, + {0xA73D, "\352\234\274"}, + {0xA73F, "\352\234\276"}, + {0xA741, "\352\235\200"}, + {0xA743, "\352\235\202"}, + {0xA745, "\352\235\204"}, + {0xA747, "\352\235\206"}, + {0xA749, "\352\235\210"}, + {0xA74B, "\352\235\212"}, + {0xA74D, "\352\235\214"}, + {0xA74F, "\352\235\216"}, + {0xA751, "\352\235\220"}, + {0xA753, "\352\235\222"}, + {0xA755, "\352\235\224"}, + {0xA757, "\352\235\226"}, + {0xA759, "\352\235\230"}, + {0xA75B, "\352\235\232"}, + {0xA75D, "\352\235\234"}, + {0xA75F, "\352\235\236"}, + {0xA761, "\352\235\240"}, + {0xA763, "\352\235\242"}, + {0xA765, "\352\235\244"}, + {0xA767, "\352\235\246"}, + {0xA769, "\352\235\250"}, + {0xA76B, "\352\235\252"}, + {0xA76D, "\352\235\254"}, + {0xA76F, "\352\235\256"}, + {0xA77A, "\352\235\271"}, + {0xA77C, "\352\235\273"}, + {0xA77F, "\352\235\276"}, + {0xA781, "\352\236\200"}, + {0xA783, "\352\236\202"}, + {0xA785, "\352\236\204"}, + {0xA787, "\352\236\206"}, + {0xA78C, "\352\236\213"}, + {0xA791, "\352\236\220"}, + {0xA793, "\352\236\222"}, + {0xA794, "\352\237\204"}, + {0xA797, "\352\236\226"}, + {0xA799, "\352\236\230"}, + {0xA79B, "\352\236\232"}, + {0xA79D, "\352\236\234"}, + {0xA79F, "\352\236\236"}, + {0xA7A1, "\352\236\240"}, + {0xA7A3, "\352\236\242"}, + {0xA7A5, "\352\236\244"}, + {0xA7A7, "\352\236\246"}, + {0xA7A9, "\352\236\250"}, + {0xA7B5, "\352\236\264"}, + {0xA7B7, "\352\236\266"}, + {0xA7B9, "\352\236\270"}, + {0xA7BB, "\352\236\272"}, + {0xA7BD, "\352\236\274"}, + {0xA7BF, "\352\236\276"}, + {0xA7C1, "\352\237\200"}, + {0xA7C3, "\352\237\202"}, + {0xA7C8, "\352\237\207"}, + {0xA7CA, "\352\237\211"}, + {0xA7CD, "\352\237\214"}, + {0xA7D1, "\352\237\220"}, + {0xA7D7, "\352\237\226"}, + {0xA7D9, "\352\237\230"}, + {0xA7DB, "\352\237\232"}, + {0xA7F6, "\352\237\265"}, + {0xAB53, "\352\236\263"}, + {0xAB70, "\341\216\240"}, + {0xAB71, "\341\216\241"}, + {0xAB72, "\341\216\242"}, + {0xAB73, "\341\216\243"}, + {0xAB74, "\341\216\244"}, + {0xAB75, "\341\216\245"}, + {0xAB76, "\341\216\246"}, + {0xAB77, "\341\216\247"}, + {0xAB78, "\341\216\250"}, + {0xAB79, "\341\216\251"}, + {0xAB7A, "\341\216\252"}, + {0xAB7B, "\341\216\253"}, + {0xAB7C, "\341\216\254"}, + {0xAB7D, "\341\216\255"}, + {0xAB7E, "\341\216\256"}, + {0xAB7F, "\341\216\257"}, + {0xAB80, "\341\216\260"}, + {0xAB81, "\341\216\261"}, + {0xAB82, "\341\216\262"}, + {0xAB83, "\341\216\263"}, + {0xAB84, "\341\216\264"}, + {0xAB85, "\341\216\265"}, + {0xAB86, "\341\216\266"}, + {0xAB87, "\341\216\267"}, + {0xAB88, "\341\216\270"}, + {0xAB89, "\341\216\271"}, + {0xAB8A, "\341\216\272"}, + {0xAB8B, "\341\216\273"}, + {0xAB8C, "\341\216\274"}, + {0xAB8D, "\341\216\275"}, + {0xAB8E, "\341\216\276"}, + {0xAB8F, "\341\216\277"}, + {0xAB90, "\341\217\200"}, + {0xAB91, "\341\217\201"}, + {0xAB92, "\341\217\202"}, + {0xAB93, "\341\217\203"}, + {0xAB94, "\341\217\204"}, + {0xAB95, "\341\217\205"}, + {0xAB96, "\341\217\206"}, + {0xAB97, "\341\217\207"}, + {0xAB98, "\341\217\210"}, + {0xAB99, "\341\217\211"}, + {0xAB9A, "\341\217\212"}, + {0xAB9B, "\341\217\213"}, + {0xAB9C, "\341\217\214"}, + {0xAB9D, "\341\217\215"}, + {0xAB9E, "\341\217\216"}, + {0xAB9F, "\341\217\217"}, + {0xABA0, "\341\217\220"}, + {0xABA1, "\341\217\221"}, + {0xABA2, "\341\217\222"}, + {0xABA3, "\341\217\223"}, + {0xABA4, "\341\217\224"}, + {0xABA5, "\341\217\225"}, + {0xABA6, "\341\217\226"}, + {0xABA7, "\341\217\227"}, + {0xABA8, "\341\217\230"}, + {0xABA9, "\341\217\231"}, + {0xABAA, "\341\217\232"}, + {0xABAB, "\341\217\233"}, + {0xABAC, "\341\217\234"}, + {0xABAD, "\341\217\235"}, + {0xABAE, "\341\217\236"}, + {0xABAF, "\341\217\237"}, + {0xABB0, "\341\217\240"}, + {0xABB1, "\341\217\241"}, + {0xABB2, "\341\217\242"}, + {0xABB3, "\341\217\243"}, + {0xABB4, "\341\217\244"}, + {0xABB5, "\341\217\245"}, + {0xABB6, "\341\217\246"}, + {0xABB7, "\341\217\247"}, + {0xABB8, "\341\217\250"}, + {0xABB9, "\341\217\251"}, + {0xABBA, "\341\217\252"}, + {0xABBB, "\341\217\253"}, + {0xABBC, "\341\217\254"}, + {0xABBD, "\341\217\255"}, + {0xABBE, "\341\217\256"}, + {0xABBF, "\341\217\257"}, + {0xFB00, "FF"}, + {0xFB01, "FI"}, + {0xFB02, "FL"}, + {0xFB03, "FFI"}, + {0xFB04, "FFL"}, + {0xFB05, "ST"}, + {0xFB06, "ST"}, + {0xFB13, "\325\204\325\206"}, + {0xFB14, "\325\204\324\265"}, + {0xFB15, "\325\204\324\273"}, + {0xFB16, "\325\216\325\206"}, + {0xFB17, "\325\204\324\275"}, + {0xFF41, "\357\274\241"}, + {0xFF42, "\357\274\242"}, + {0xFF43, "\357\274\243"}, + {0xFF44, "\357\274\244"}, + {0xFF45, "\357\274\245"}, + {0xFF46, "\357\274\246"}, + {0xFF47, "\357\274\247"}, + {0xFF48, "\357\274\250"}, + {0xFF49, "\357\274\251"}, + {0xFF4A, "\357\274\252"}, + {0xFF4B, "\357\274\253"}, + {0xFF4C, "\357\274\254"}, + {0xFF4D, "\357\274\255"}, + {0xFF4E, "\357\274\256"}, + {0xFF4F, "\357\274\257"}, + {0xFF50, "\357\274\260"}, + {0xFF51, "\357\274\261"}, + {0xFF52, "\357\274\262"}, + {0xFF53, "\357\274\263"}, + {0xFF54, "\357\274\264"}, + {0xFF55, "\357\274\265"}, + {0xFF56, "\357\274\266"}, + {0xFF57, "\357\274\267"}, + {0xFF58, "\357\274\270"}, + {0xFF59, "\357\274\271"}, + {0xFF5A, "\357\274\272"}, + {0x10428, "\360\220\220\200"}, + {0x10429, "\360\220\220\201"}, + {0x1042A, "\360\220\220\202"}, + {0x1042B, "\360\220\220\203"}, + {0x1042C, "\360\220\220\204"}, + {0x1042D, "\360\220\220\205"}, + {0x1042E, "\360\220\220\206"}, + {0x1042F, "\360\220\220\207"}, + {0x10430, "\360\220\220\210"}, + {0x10431, "\360\220\220\211"}, + {0x10432, "\360\220\220\212"}, + {0x10433, "\360\220\220\213"}, + {0x10434, "\360\220\220\214"}, + {0x10435, "\360\220\220\215"}, + {0x10436, "\360\220\220\216"}, + {0x10437, "\360\220\220\217"}, + {0x10438, "\360\220\220\220"}, + {0x10439, "\360\220\220\221"}, + {0x1043A, "\360\220\220\222"}, + {0x1043B, "\360\220\220\223"}, + {0x1043C, "\360\220\220\224"}, + {0x1043D, "\360\220\220\225"}, + {0x1043E, "\360\220\220\226"}, + {0x1043F, "\360\220\220\227"}, + {0x10440, "\360\220\220\230"}, + {0x10441, "\360\220\220\231"}, + {0x10442, "\360\220\220\232"}, + {0x10443, "\360\220\220\233"}, + {0x10444, "\360\220\220\234"}, + {0x10445, "\360\220\220\235"}, + {0x10446, "\360\220\220\236"}, + {0x10447, "\360\220\220\237"}, + {0x10448, "\360\220\220\240"}, + {0x10449, "\360\220\220\241"}, + {0x1044A, "\360\220\220\242"}, + {0x1044B, "\360\220\220\243"}, + {0x1044C, "\360\220\220\244"}, + {0x1044D, "\360\220\220\245"}, + {0x1044E, "\360\220\220\246"}, + {0x1044F, "\360\220\220\247"}, + {0x104D8, "\360\220\222\260"}, + {0x104D9, "\360\220\222\261"}, + {0x104DA, "\360\220\222\262"}, + {0x104DB, "\360\220\222\263"}, + {0x104DC, "\360\220\222\264"}, + {0x104DD, "\360\220\222\265"}, + {0x104DE, "\360\220\222\266"}, + {0x104DF, "\360\220\222\267"}, + {0x104E0, "\360\220\222\270"}, + {0x104E1, "\360\220\222\271"}, + {0x104E2, "\360\220\222\272"}, + {0x104E3, "\360\220\222\273"}, + {0x104E4, "\360\220\222\274"}, + {0x104E5, "\360\220\222\275"}, + {0x104E6, "\360\220\222\276"}, + {0x104E7, "\360\220\222\277"}, + {0x104E8, "\360\220\223\200"}, + {0x104E9, "\360\220\223\201"}, + {0x104EA, "\360\220\223\202"}, + {0x104EB, "\360\220\223\203"}, + {0x104EC, "\360\220\223\204"}, + {0x104ED, "\360\220\223\205"}, + {0x104EE, "\360\220\223\206"}, + {0x104EF, "\360\220\223\207"}, + {0x104F0, "\360\220\223\210"}, + {0x104F1, "\360\220\223\211"}, + {0x104F2, "\360\220\223\212"}, + {0x104F3, "\360\220\223\213"}, + {0x104F4, "\360\220\223\214"}, + {0x104F5, "\360\220\223\215"}, + {0x104F6, "\360\220\223\216"}, + {0x104F7, "\360\220\223\217"}, + {0x104F8, "\360\220\223\220"}, + {0x104F9, "\360\220\223\221"}, + {0x104FA, "\360\220\223\222"}, + {0x104FB, "\360\220\223\223"}, + {0x10597, "\360\220\225\260"}, + {0x10598, "\360\220\225\261"}, + {0x10599, "\360\220\225\262"}, + {0x1059A, "\360\220\225\263"}, + {0x1059B, "\360\220\225\264"}, + {0x1059C, "\360\220\225\265"}, + {0x1059D, "\360\220\225\266"}, + {0x1059E, "\360\220\225\267"}, + {0x1059F, "\360\220\225\270"}, + {0x105A0, "\360\220\225\271"}, + {0x105A1, "\360\220\225\272"}, + {0x105A3, "\360\220\225\274"}, + {0x105A4, "\360\220\225\275"}, + {0x105A5, "\360\220\225\276"}, + {0x105A6, "\360\220\225\277"}, + {0x105A7, "\360\220\226\200"}, + {0x105A8, "\360\220\226\201"}, + {0x105A9, "\360\220\226\202"}, + {0x105AA, "\360\220\226\203"}, + {0x105AB, "\360\220\226\204"}, + {0x105AC, "\360\220\226\205"}, + {0x105AD, "\360\220\226\206"}, + {0x105AE, "\360\220\226\207"}, + {0x105AF, "\360\220\226\210"}, + {0x105B0, "\360\220\226\211"}, + {0x105B1, "\360\220\226\212"}, + {0x105B3, "\360\220\226\214"}, + {0x105B4, "\360\220\226\215"}, + {0x105B5, "\360\220\226\216"}, + {0x105B6, "\360\220\226\217"}, + {0x105B7, "\360\220\226\220"}, + {0x105B8, "\360\220\226\221"}, + {0x105B9, "\360\220\226\222"}, + {0x105BB, "\360\220\226\224"}, + {0x105BC, "\360\220\226\225"}, + {0x10CC0, "\360\220\262\200"}, + {0x10CC1, "\360\220\262\201"}, + {0x10CC2, "\360\220\262\202"}, + {0x10CC3, "\360\220\262\203"}, + {0x10CC4, "\360\220\262\204"}, + {0x10CC5, "\360\220\262\205"}, + {0x10CC6, "\360\220\262\206"}, + {0x10CC7, "\360\220\262\207"}, + {0x10CC8, "\360\220\262\210"}, + {0x10CC9, "\360\220\262\211"}, + {0x10CCA, "\360\220\262\212"}, + {0x10CCB, "\360\220\262\213"}, + {0x10CCC, "\360\220\262\214"}, + {0x10CCD, "\360\220\262\215"}, + {0x10CCE, "\360\220\262\216"}, + {0x10CCF, "\360\220\262\217"}, + {0x10CD0, "\360\220\262\220"}, + {0x10CD1, "\360\220\262\221"}, + {0x10CD2, "\360\220\262\222"}, + {0x10CD3, "\360\220\262\223"}, + {0x10CD4, "\360\220\262\224"}, + {0x10CD5, "\360\220\262\225"}, + {0x10CD6, "\360\220\262\226"}, + {0x10CD7, "\360\220\262\227"}, + {0x10CD8, "\360\220\262\230"}, + {0x10CD9, "\360\220\262\231"}, + {0x10CDA, "\360\220\262\232"}, + {0x10CDB, "\360\220\262\233"}, + {0x10CDC, "\360\220\262\234"}, + {0x10CDD, "\360\220\262\235"}, + {0x10CDE, "\360\220\262\236"}, + {0x10CDF, "\360\220\262\237"}, + {0x10CE0, "\360\220\262\240"}, + {0x10CE1, "\360\220\262\241"}, + {0x10CE2, "\360\220\262\242"}, + {0x10CE3, "\360\220\262\243"}, + {0x10CE4, "\360\220\262\244"}, + {0x10CE5, "\360\220\262\245"}, + {0x10CE6, "\360\220\262\246"}, + {0x10CE7, "\360\220\262\247"}, + {0x10CE8, "\360\220\262\250"}, + {0x10CE9, "\360\220\262\251"}, + {0x10CEA, "\360\220\262\252"}, + {0x10CEB, "\360\220\262\253"}, + {0x10CEC, "\360\220\262\254"}, + {0x10CED, "\360\220\262\255"}, + {0x10CEE, "\360\220\262\256"}, + {0x10CEF, "\360\220\262\257"}, + {0x10CF0, "\360\220\262\260"}, + {0x10CF1, "\360\220\262\261"}, + {0x10CF2, "\360\220\262\262"}, + {0x10D70, "\360\220\265\220"}, + {0x10D71, "\360\220\265\221"}, + {0x10D72, "\360\220\265\222"}, + {0x10D73, "\360\220\265\223"}, + {0x10D74, "\360\220\265\224"}, + {0x10D75, "\360\220\265\225"}, + {0x10D76, "\360\220\265\226"}, + {0x10D77, "\360\220\265\227"}, + {0x10D78, "\360\220\265\230"}, + {0x10D79, "\360\220\265\231"}, + {0x10D7A, "\360\220\265\232"}, + {0x10D7B, "\360\220\265\233"}, + {0x10D7C, "\360\220\265\234"}, + {0x10D7D, "\360\220\265\235"}, + {0x10D7E, "\360\220\265\236"}, + {0x10D7F, "\360\220\265\237"}, + {0x10D80, "\360\220\265\240"}, + {0x10D81, "\360\220\265\241"}, + {0x10D82, "\360\220\265\242"}, + {0x10D83, "\360\220\265\243"}, + {0x10D84, "\360\220\265\244"}, + {0x10D85, "\360\220\265\245"}, + {0x118C0, "\360\221\242\240"}, + {0x118C1, "\360\221\242\241"}, + {0x118C2, "\360\221\242\242"}, + {0x118C3, "\360\221\242\243"}, + {0x118C4, "\360\221\242\244"}, + {0x118C5, "\360\221\242\245"}, + {0x118C6, "\360\221\242\246"}, + {0x118C7, "\360\221\242\247"}, + {0x118C8, "\360\221\242\250"}, + {0x118C9, "\360\221\242\251"}, + {0x118CA, "\360\221\242\252"}, + {0x118CB, "\360\221\242\253"}, + {0x118CC, "\360\221\242\254"}, + {0x118CD, "\360\221\242\255"}, + {0x118CE, "\360\221\242\256"}, + {0x118CF, "\360\221\242\257"}, + {0x118D0, "\360\221\242\260"}, + {0x118D1, "\360\221\242\261"}, + {0x118D2, "\360\221\242\262"}, + {0x118D3, "\360\221\242\263"}, + {0x118D4, "\360\221\242\264"}, + {0x118D5, "\360\221\242\265"}, + {0x118D6, "\360\221\242\266"}, + {0x118D7, "\360\221\242\267"}, + {0x118D8, "\360\221\242\270"}, + {0x118D9, "\360\221\242\271"}, + {0x118DA, "\360\221\242\272"}, + {0x118DB, "\360\221\242\273"}, + {0x118DC, "\360\221\242\274"}, + {0x118DD, "\360\221\242\275"}, + {0x118DE, "\360\221\242\276"}, + {0x118DF, "\360\221\242\277"}, + {0x16E60, "\360\226\271\200"}, + {0x16E61, "\360\226\271\201"}, + {0x16E62, "\360\226\271\202"}, + {0x16E63, "\360\226\271\203"}, + {0x16E64, "\360\226\271\204"}, + {0x16E65, "\360\226\271\205"}, + {0x16E66, "\360\226\271\206"}, + {0x16E67, "\360\226\271\207"}, + {0x16E68, "\360\226\271\210"}, + {0x16E69, "\360\226\271\211"}, + {0x16E6A, "\360\226\271\212"}, + {0x16E6B, "\360\226\271\213"}, + {0x16E6C, "\360\226\271\214"}, + {0x16E6D, "\360\226\271\215"}, + {0x16E6E, "\360\226\271\216"}, + {0x16E6F, "\360\226\271\217"}, + {0x16E70, "\360\226\271\220"}, + {0x16E71, "\360\226\271\221"}, + {0x16E72, "\360\226\271\222"}, + {0x16E73, "\360\226\271\223"}, + {0x16E74, "\360\226\271\224"}, + {0x16E75, "\360\226\271\225"}, + {0x16E76, "\360\226\271\226"}, + {0x16E77, "\360\226\271\227"}, + {0x16E78, "\360\226\271\230"}, + {0x16E79, "\360\226\271\231"}, + {0x16E7A, "\360\226\271\232"}, + {0x16E7B, "\360\226\271\233"}, + {0x16E7C, "\360\226\271\234"}, + {0x16E7D, "\360\226\271\235"}, + {0x16E7E, "\360\226\271\236"}, + {0x16E7F, "\360\226\271\237"}, + {0x1E922, "\360\236\244\200"}, + {0x1E923, "\360\236\244\201"}, + {0x1E924, "\360\236\244\202"}, + {0x1E925, "\360\236\244\203"}, + {0x1E926, "\360\236\244\204"}, + {0x1E927, "\360\236\244\205"}, + {0x1E928, "\360\236\244\206"}, + {0x1E929, "\360\236\244\207"}, + {0x1E92A, "\360\236\244\210"}, + {0x1E92B, "\360\236\244\211"}, + {0x1E92C, "\360\236\244\212"}, + {0x1E92D, "\360\236\244\213"}, + {0x1E92E, "\360\236\244\214"}, + {0x1E92F, "\360\236\244\215"}, + {0x1E930, "\360\236\244\216"}, + {0x1E931, "\360\236\244\217"}, + {0x1E932, "\360\236\244\220"}, + {0x1E933, "\360\236\244\221"}, + {0x1E934, "\360\236\244\222"}, + {0x1E935, "\360\236\244\223"}, + {0x1E936, "\360\236\244\224"}, + {0x1E937, "\360\236\244\225"}, + {0x1E938, "\360\236\244\226"}, + {0x1E939, "\360\236\244\227"}, + {0x1E93A, "\360\236\244\230"}, + {0x1E93B, "\360\236\244\231"}, + {0x1E93C, "\360\236\244\232"}, + {0x1E93D, "\360\236\244\233"}, + {0x1E93E, "\360\236\244\234"}, + {0x1E93F, "\360\236\244\235"}, + {0x1E940, "\360\236\244\236"}, + {0x1E941, "\360\236\244\237"}, + {0x1E942, "\360\236\244\240"}, + {0x1E943, "\360\236\244\241"}, + }; + return v; +} +const std::vector& SpaceRanges() { + static const std::vector v = { + {0x9, 0xD}, + {0x1C, 0x20}, + {0x85, 0x85}, + {0xA0, 0xA0}, + {0x1680, 0x1680}, + {0x2000, 0x200A}, + {0x2028, 0x2029}, + {0x202F, 0x202F}, + {0x205F, 0x205F}, + {0x3000, 0x3000}, + }; + return v; +} +const std::vector& SigmaIgnorableRanges() { + static const std::vector v = { + {0x27, 0x27}, + {0x2E, 0x2E}, + {0x3A, 0x3A}, + {0x5E, 0x5E}, + {0x60, 0x60}, + {0xA8, 0xA8}, + {0xAD, 0xAD}, + {0xAF, 0xAF}, + {0xB4, 0xB4}, + {0xB7, 0xB8}, + {0x2B0, 0x36F}, + {0x374, 0x375}, + {0x37A, 0x37A}, + {0x384, 0x385}, + {0x387, 0x387}, + {0x483, 0x489}, + {0x559, 0x559}, + {0x55F, 0x55F}, + {0x591, 0x5BD}, + {0x5BF, 0x5BF}, + {0x5C1, 0x5C2}, + {0x5C4, 0x5C5}, + {0x5C7, 0x5C7}, + {0x5F4, 0x5F4}, + {0x600, 0x605}, + {0x610, 0x61A}, + {0x61C, 0x61C}, + {0x640, 0x640}, + {0x64B, 0x65F}, + {0x670, 0x670}, + {0x6D6, 0x6DD}, + {0x6DF, 0x6E8}, + {0x6EA, 0x6ED}, + {0x70F, 0x70F}, + {0x711, 0x711}, + {0x730, 0x74A}, + {0x7A6, 0x7B0}, + {0x7EB, 0x7F5}, + {0x7FA, 0x7FA}, + {0x7FD, 0x7FD}, + {0x816, 0x82D}, + {0x859, 0x85B}, + {0x888, 0x888}, + {0x890, 0x891}, + {0x897, 0x89F}, + {0x8C9, 0x902}, + {0x93A, 0x93A}, + {0x93C, 0x93C}, + {0x941, 0x948}, + {0x94D, 0x94D}, + {0x951, 0x957}, + {0x962, 0x963}, + {0x971, 0x971}, + {0x981, 0x981}, + {0x9BC, 0x9BC}, + {0x9C1, 0x9C4}, + {0x9CD, 0x9CD}, + {0x9E2, 0x9E3}, + {0x9FE, 0x9FE}, + {0xA01, 0xA02}, + {0xA3C, 0xA3C}, + {0xA41, 0xA42}, + {0xA47, 0xA48}, + {0xA4B, 0xA4D}, + {0xA51, 0xA51}, + {0xA70, 0xA71}, + {0xA75, 0xA75}, + {0xA81, 0xA82}, + {0xABC, 0xABC}, + {0xAC1, 0xAC5}, + {0xAC7, 0xAC8}, + {0xACD, 0xACD}, + {0xAE2, 0xAE3}, + {0xAFA, 0xAFF}, + {0xB01, 0xB01}, + {0xB3C, 0xB3C}, + {0xB3F, 0xB3F}, + {0xB41, 0xB44}, + {0xB4D, 0xB4D}, + {0xB55, 0xB56}, + {0xB62, 0xB63}, + {0xB82, 0xB82}, + {0xBC0, 0xBC0}, + {0xBCD, 0xBCD}, + {0xC00, 0xC00}, + {0xC04, 0xC04}, + {0xC3C, 0xC3C}, + {0xC3E, 0xC40}, + {0xC46, 0xC48}, + {0xC4A, 0xC4D}, + {0xC55, 0xC56}, + {0xC62, 0xC63}, + {0xC81, 0xC81}, + {0xCBC, 0xCBC}, + {0xCBF, 0xCBF}, + {0xCC6, 0xCC6}, + {0xCCC, 0xCCD}, + {0xCE2, 0xCE3}, + {0xD00, 0xD01}, + {0xD3B, 0xD3C}, + {0xD41, 0xD44}, + {0xD4D, 0xD4D}, + {0xD62, 0xD63}, + {0xD81, 0xD81}, + {0xDCA, 0xDCA}, + {0xDD2, 0xDD4}, + {0xDD6, 0xDD6}, + {0xE31, 0xE31}, + {0xE34, 0xE3A}, + {0xE46, 0xE4E}, + {0xEB1, 0xEB1}, + {0xEB4, 0xEBC}, + {0xEC6, 0xEC6}, + {0xEC8, 0xECE}, + {0xF18, 0xF19}, + {0xF35, 0xF35}, + {0xF37, 0xF37}, + {0xF39, 0xF39}, + {0xF71, 0xF7E}, + {0xF80, 0xF84}, + {0xF86, 0xF87}, + {0xF8D, 0xF97}, + {0xF99, 0xFBC}, + {0xFC6, 0xFC6}, + {0x102D, 0x1030}, + {0x1032, 0x1037}, + {0x1039, 0x103A}, + {0x103D, 0x103E}, + {0x1058, 0x1059}, + {0x105E, 0x1060}, + {0x1071, 0x1074}, + {0x1082, 0x1082}, + {0x1085, 0x1086}, + {0x108D, 0x108D}, + {0x109D, 0x109D}, + {0x10FC, 0x10FC}, + {0x135D, 0x135F}, + {0x1712, 0x1714}, + {0x1732, 0x1733}, + {0x1752, 0x1753}, + {0x1772, 0x1773}, + {0x17B4, 0x17B5}, + {0x17B7, 0x17BD}, + {0x17C6, 0x17C6}, + {0x17C9, 0x17D3}, + {0x17D7, 0x17D7}, + {0x17DD, 0x17DD}, + {0x180B, 0x180F}, + {0x1843, 0x1843}, + {0x1885, 0x1886}, + {0x18A9, 0x18A9}, + {0x1920, 0x1922}, + {0x1927, 0x1928}, + {0x1932, 0x1932}, + {0x1939, 0x193B}, + {0x1A17, 0x1A18}, + {0x1A1B, 0x1A1B}, + {0x1A56, 0x1A56}, + {0x1A58, 0x1A5E}, + {0x1A60, 0x1A60}, + {0x1A62, 0x1A62}, + {0x1A65, 0x1A6C}, + {0x1A73, 0x1A7C}, + {0x1A7F, 0x1A7F}, + {0x1AA7, 0x1AA7}, + {0x1AB0, 0x1ACE}, + {0x1B00, 0x1B03}, + {0x1B34, 0x1B34}, + {0x1B36, 0x1B3A}, + {0x1B3C, 0x1B3C}, + {0x1B42, 0x1B42}, + {0x1B6B, 0x1B73}, + {0x1B80, 0x1B81}, + {0x1BA2, 0x1BA5}, + {0x1BA8, 0x1BA9}, + {0x1BAB, 0x1BAD}, + {0x1BE6, 0x1BE6}, + {0x1BE8, 0x1BE9}, + {0x1BED, 0x1BED}, + {0x1BEF, 0x1BF1}, + {0x1C2C, 0x1C33}, + {0x1C36, 0x1C37}, + {0x1C78, 0x1C7D}, + {0x1CD0, 0x1CD2}, + {0x1CD4, 0x1CE0}, + {0x1CE2, 0x1CE8}, + {0x1CED, 0x1CED}, + {0x1CF4, 0x1CF4}, + {0x1CF8, 0x1CF9}, + {0x1D2C, 0x1D6A}, + {0x1D78, 0x1D78}, + {0x1D9B, 0x1DFF}, + {0x1FBD, 0x1FBD}, + {0x1FBF, 0x1FC1}, + {0x1FCD, 0x1FCF}, + {0x1FDD, 0x1FDF}, + {0x1FED, 0x1FEF}, + {0x1FFD, 0x1FFE}, + {0x200B, 0x200F}, + {0x2018, 0x2019}, + {0x2024, 0x2024}, + {0x2027, 0x2027}, + {0x202A, 0x202E}, + {0x2060, 0x2064}, + {0x2066, 0x206F}, + {0x2071, 0x2071}, + {0x207F, 0x207F}, + {0x2090, 0x209C}, + {0x20D0, 0x20F0}, + {0x2C7C, 0x2C7D}, + {0x2CEF, 0x2CF1}, + {0x2D6F, 0x2D6F}, + {0x2D7F, 0x2D7F}, + {0x2DE0, 0x2DFF}, + {0x2E2F, 0x2E2F}, + {0x3005, 0x3005}, + {0x302A, 0x302D}, + {0x3031, 0x3035}, + {0x303B, 0x303B}, + {0x3099, 0x309E}, + {0x30FC, 0x30FE}, + {0xA015, 0xA015}, + {0xA4F8, 0xA4FD}, + {0xA60C, 0xA60C}, + {0xA66F, 0xA672}, + {0xA674, 0xA67D}, + {0xA67F, 0xA67F}, + {0xA69C, 0xA69F}, + {0xA6F0, 0xA6F1}, + {0xA700, 0xA721}, + {0xA770, 0xA770}, + {0xA788, 0xA78A}, + {0xA7F2, 0xA7F4}, + {0xA7F8, 0xA7F9}, + {0xA802, 0xA802}, + {0xA806, 0xA806}, + {0xA80B, 0xA80B}, + {0xA825, 0xA826}, + {0xA82C, 0xA82C}, + {0xA8C4, 0xA8C5}, + {0xA8E0, 0xA8F1}, + {0xA8FF, 0xA8FF}, + {0xA926, 0xA92D}, + {0xA947, 0xA951}, + {0xA980, 0xA982}, + {0xA9B3, 0xA9B3}, + {0xA9B6, 0xA9B9}, + {0xA9BC, 0xA9BD}, + {0xA9CF, 0xA9CF}, + {0xA9E5, 0xA9E6}, + {0xAA29, 0xAA2E}, + {0xAA31, 0xAA32}, + {0xAA35, 0xAA36}, + {0xAA43, 0xAA43}, + {0xAA4C, 0xAA4C}, + {0xAA70, 0xAA70}, + {0xAA7C, 0xAA7C}, + {0xAAB0, 0xAAB0}, + {0xAAB2, 0xAAB4}, + {0xAAB7, 0xAAB8}, + {0xAABE, 0xAABF}, + {0xAAC1, 0xAAC1}, + {0xAADD, 0xAADD}, + {0xAAEC, 0xAAED}, + {0xAAF3, 0xAAF4}, + {0xAAF6, 0xAAF6}, + {0xAB5B, 0xAB5F}, + {0xAB69, 0xAB6B}, + {0xABE5, 0xABE5}, + {0xABE8, 0xABE8}, + {0xABED, 0xABED}, + {0xFB1E, 0xFB1E}, + {0xFBB2, 0xFBC2}, + {0xFE00, 0xFE0F}, + {0xFE13, 0xFE13}, + {0xFE20, 0xFE2F}, + {0xFE52, 0xFE52}, + {0xFE55, 0xFE55}, + {0xFEFF, 0xFEFF}, + {0xFF07, 0xFF07}, + {0xFF0E, 0xFF0E}, + {0xFF1A, 0xFF1A}, + {0xFF3E, 0xFF3E}, + {0xFF40, 0xFF40}, + {0xFF70, 0xFF70}, + {0xFF9E, 0xFF9F}, + {0xFFE3, 0xFFE3}, + {0xFFF9, 0xFFFB}, + {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, + {0x10376, 0x1037A}, + {0x10780, 0x10785}, + {0x10787, 0x107B0}, + {0x107B2, 0x107BA}, + {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, + {0x10A0C, 0x10A0F}, + {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, + {0x10AE5, 0x10AE6}, + {0x10D24, 0x10D27}, + {0x10D4E, 0x10D4E}, + {0x10D69, 0x10D6D}, + {0x10D6F, 0x10D6F}, + {0x10EAB, 0x10EAC}, + {0x10EFC, 0x10EFF}, + {0x10F46, 0x10F50}, + {0x10F82, 0x10F85}, + {0x11001, 0x11001}, + {0x11038, 0x11046}, + {0x11070, 0x11070}, + {0x11073, 0x11074}, + {0x1107F, 0x11081}, + {0x110B3, 0x110B6}, + {0x110B9, 0x110BA}, + {0x110BD, 0x110BD}, + {0x110C2, 0x110C2}, + {0x110CD, 0x110CD}, + {0x11100, 0x11102}, + {0x11127, 0x1112B}, + {0x1112D, 0x11134}, + {0x11173, 0x11173}, + {0x11180, 0x11181}, + {0x111B6, 0x111BE}, + {0x111C9, 0x111CC}, + {0x111CF, 0x111CF}, + {0x1122F, 0x11231}, + {0x11234, 0x11234}, + {0x11236, 0x11237}, + {0x1123E, 0x1123E}, + {0x11241, 0x11241}, + {0x112DF, 0x112DF}, + {0x112E3, 0x112EA}, + {0x11300, 0x11301}, + {0x1133B, 0x1133C}, + {0x11340, 0x11340}, + {0x11366, 0x1136C}, + {0x11370, 0x11374}, + {0x113BB, 0x113C0}, + {0x113CE, 0x113CE}, + {0x113D0, 0x113D0}, + {0x113D2, 0x113D2}, + {0x113E1, 0x113E2}, + {0x11438, 0x1143F}, + {0x11442, 0x11444}, + {0x11446, 0x11446}, + {0x1145E, 0x1145E}, + {0x114B3, 0x114B8}, + {0x114BA, 0x114BA}, + {0x114BF, 0x114C0}, + {0x114C2, 0x114C3}, + {0x115B2, 0x115B5}, + {0x115BC, 0x115BD}, + {0x115BF, 0x115C0}, + {0x115DC, 0x115DD}, + {0x11633, 0x1163A}, + {0x1163D, 0x1163D}, + {0x1163F, 0x11640}, + {0x116AB, 0x116AB}, + {0x116AD, 0x116AD}, + {0x116B0, 0x116B5}, + {0x116B7, 0x116B7}, + {0x1171D, 0x1171D}, + {0x1171F, 0x1171F}, + {0x11722, 0x11725}, + {0x11727, 0x1172B}, + {0x1182F, 0x11837}, + {0x11839, 0x1183A}, + {0x1193B, 0x1193C}, + {0x1193E, 0x1193E}, + {0x11943, 0x11943}, + {0x119D4, 0x119D7}, + {0x119DA, 0x119DB}, + {0x119E0, 0x119E0}, + {0x11A01, 0x11A0A}, + {0x11A33, 0x11A38}, + {0x11A3B, 0x11A3E}, + {0x11A47, 0x11A47}, + {0x11A51, 0x11A56}, + {0x11A59, 0x11A5B}, + {0x11A8A, 0x11A96}, + {0x11A98, 0x11A99}, + {0x11C30, 0x11C36}, + {0x11C38, 0x11C3D}, + {0x11C3F, 0x11C3F}, + {0x11C92, 0x11CA7}, + {0x11CAA, 0x11CB0}, + {0x11CB2, 0x11CB3}, + {0x11CB5, 0x11CB6}, + {0x11D31, 0x11D36}, + {0x11D3A, 0x11D3A}, + {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D45}, + {0x11D47, 0x11D47}, + {0x11D90, 0x11D91}, + {0x11D95, 0x11D95}, + {0x11D97, 0x11D97}, + {0x11EF3, 0x11EF4}, + {0x11F00, 0x11F01}, + {0x11F36, 0x11F3A}, + {0x11F40, 0x11F40}, + {0x11F42, 0x11F42}, + {0x11F5A, 0x11F5A}, + {0x13430, 0x13440}, + {0x13447, 0x13455}, + {0x1611E, 0x16129}, + {0x1612D, 0x1612F}, + {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, + {0x16B40, 0x16B43}, + {0x16D40, 0x16D42}, + {0x16D6B, 0x16D6C}, + {0x16F4F, 0x16F4F}, + {0x16F8F, 0x16F9F}, + {0x16FE0, 0x16FE1}, + {0x16FE3, 0x16FE4}, + {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, + {0x1BC9D, 0x1BC9E}, + {0x1BCA0, 0x1BCA3}, + {0x1CF00, 0x1CF2D}, + {0x1CF30, 0x1CF46}, + {0x1D167, 0x1D169}, + {0x1D173, 0x1D182}, + {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, + {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, + {0x1DAA1, 0x1DAAF}, + {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, + {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, + {0x1E030, 0x1E06D}, + {0x1E08F, 0x1E08F}, + {0x1E130, 0x1E13D}, + {0x1E2AE, 0x1E2AE}, + {0x1E2EC, 0x1E2EF}, + {0x1E4EB, 0x1E4EF}, + {0x1E5EE, 0x1E5EF}, + {0x1E8D0, 0x1E8D6}, + {0x1E944, 0x1E94B}, + {0x1F3FB, 0x1F3FF}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F}, + {0xE0100, 0xE01EF}, + }; + return v; +} +const std::vector& SigmaCasedRanges() { + static const std::vector v = { + {0x41, 0x5A}, + {0x61, 0x7A}, + {0xAA, 0xAA}, + {0xB5, 0xB5}, + {0xBA, 0xBA}, + {0xC0, 0xD6}, + {0xD8, 0xF6}, + {0xF8, 0x1BA}, + {0x1BC, 0x1BF}, + {0x1C4, 0x293}, + {0x295, 0x2AF}, + {0x370, 0x373}, + {0x376, 0x377}, + {0x37B, 0x37D}, + {0x37F, 0x37F}, + {0x386, 0x386}, + {0x388, 0x38A}, + {0x38C, 0x38C}, + {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, + {0x3F7, 0x481}, + {0x48A, 0x52F}, + {0x531, 0x556}, + {0x560, 0x588}, + {0x10A0, 0x10C5}, + {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, + {0x10D0, 0x10FA}, + {0x10FD, 0x10FF}, + {0x13A0, 0x13F5}, + {0x13F8, 0x13FD}, + {0x1C80, 0x1C8A}, + {0x1C90, 0x1CBA}, + {0x1CBD, 0x1CBF}, + {0x1D00, 0x1D2B}, + {0x1D6B, 0x1D77}, + {0x1D79, 0x1D9A}, + {0x1E00, 0x1F15}, + {0x1F18, 0x1F1D}, + {0x1F20, 0x1F45}, + {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, + {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, + {0x1F5F, 0x1F7D}, + {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, + {0x1FBE, 0x1FBE}, + {0x1FC2, 0x1FC4}, + {0x1FC6, 0x1FCC}, + {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, + {0x1FE0, 0x1FEC}, + {0x1FF2, 0x1FF4}, + {0x1FF6, 0x1FFC}, + {0x2102, 0x2102}, + {0x2107, 0x2107}, + {0x210A, 0x2113}, + {0x2115, 0x2115}, + {0x2119, 0x211D}, + {0x2124, 0x2124}, + {0x2126, 0x2126}, + {0x2128, 0x2128}, + {0x212A, 0x212D}, + {0x212F, 0x2134}, + {0x2139, 0x2139}, + {0x213C, 0x213F}, + {0x2145, 0x2149}, + {0x214E, 0x214E}, + {0x2160, 0x217F}, + {0x2183, 0x2184}, + {0x24B6, 0x24E9}, + {0x2C00, 0x2C7B}, + {0x2C7E, 0x2CE4}, + {0x2CEB, 0x2CEE}, + {0x2CF2, 0x2CF3}, + {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, + {0xA640, 0xA66D}, + {0xA680, 0xA69B}, + {0xA722, 0xA76F}, + {0xA771, 0xA787}, + {0xA78B, 0xA78E}, + {0xA790, 0xA7CD}, + {0xA7D0, 0xA7D1}, + {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7DC}, + {0xA7F5, 0xA7F6}, + {0xA7FA, 0xA7FA}, + {0xAB30, 0xAB5A}, + {0xAB60, 0xAB68}, + {0xAB70, 0xABBF}, + {0xFB00, 0xFB06}, + {0xFB13, 0xFB17}, + {0xFF21, 0xFF3A}, + {0xFF41, 0xFF5A}, + {0x10400, 0x1044F}, + {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, + {0x10570, 0x1057A}, + {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, + {0x10594, 0x10595}, + {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, + {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, + {0x10C80, 0x10CB2}, + {0x10CC0, 0x10CF2}, + {0x10D50, 0x10D65}, + {0x10D70, 0x10D85}, + {0x118A0, 0x118DF}, + {0x16E40, 0x16E7F}, + {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, + {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, + {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, + {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, + {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, + {0x1D53B, 0x1D53E}, + {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, + {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, + {0x1D6A8, 0x1D6C0}, + {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, + {0x1D6FC, 0x1D714}, + {0x1D716, 0x1D734}, + {0x1D736, 0x1D74E}, + {0x1D750, 0x1D76E}, + {0x1D770, 0x1D788}, + {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7CB}, + {0x1DF00, 0x1DF09}, + {0x1DF0B, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, + {0x1E900, 0x1E943}, + {0x1F130, 0x1F149}, + {0x1F150, 0x1F169}, + {0x1F170, 0x1F189}, + }; + return v; +} +const std::vector& DecimalRanges() { + static const std::vector v = { + {0x30, 0x39}, + {0x660, 0x669}, + {0x6F0, 0x6F9}, + {0x7C0, 0x7C9}, + {0x966, 0x96F}, + {0x9E6, 0x9EF}, + {0xA66, 0xA6F}, + {0xAE6, 0xAEF}, + {0xB66, 0xB6F}, + {0xBE6, 0xBEF}, + {0xC66, 0xC6F}, + {0xCE6, 0xCEF}, + {0xD66, 0xD6F}, + {0xDE6, 0xDEF}, + {0xE50, 0xE59}, + {0xED0, 0xED9}, + {0xF20, 0xF29}, + {0x1040, 0x1049}, + {0x1090, 0x1099}, + {0x17E0, 0x17E9}, + {0x1810, 0x1819}, + {0x1946, 0x194F}, + {0x19D0, 0x19D9}, + {0x1A80, 0x1A89}, + {0x1A90, 0x1A99}, + {0x1B50, 0x1B59}, + {0x1BB0, 0x1BB9}, + {0x1C40, 0x1C49}, + {0x1C50, 0x1C59}, + {0xA620, 0xA629}, + {0xA8D0, 0xA8D9}, + {0xA900, 0xA909}, + {0xA9D0, 0xA9D9}, + {0xA9F0, 0xA9F9}, + {0xAA50, 0xAA59}, + {0xABF0, 0xABF9}, + {0xFF10, 0xFF19}, + {0x104A0, 0x104A9}, + {0x10D30, 0x10D39}, + {0x10D40, 0x10D49}, + {0x11066, 0x1106F}, + {0x110F0, 0x110F9}, + {0x11136, 0x1113F}, + {0x111D0, 0x111D9}, + {0x112F0, 0x112F9}, + {0x11450, 0x11459}, + {0x114D0, 0x114D9}, + {0x11650, 0x11659}, + {0x116C0, 0x116C9}, + {0x116D0, 0x116E3}, + {0x11730, 0x11739}, + {0x118E0, 0x118E9}, + {0x11950, 0x11959}, + {0x11BF0, 0x11BF9}, + {0x11C50, 0x11C59}, + {0x11D50, 0x11D59}, + {0x11DA0, 0x11DA9}, + {0x11F50, 0x11F59}, + {0x16130, 0x16139}, + {0x16A60, 0x16A69}, + {0x16AC0, 0x16AC9}, + {0x16B50, 0x16B59}, + {0x16D70, 0x16D79}, + {0x1CCF0, 0x1CCF9}, + {0x1D7CE, 0x1D7FF}, + {0x1E140, 0x1E149}, + {0x1E2F0, 0x1E2F9}, + {0x1E4F0, 0x1E4F9}, + {0x1E5F1, 0x1E5FA}, + {0x1E950, 0x1E959}, + {0x1FBF0, 0x1FBF9}, + }; + return v; +} +} // namespace detz::py::ucd diff --git a/taicpp/src/llm/unicode_data.h b/taicpp/src/llm/unicode_data.h new file mode 100644 index 0000000..1f1894a --- /dev/null +++ b/taicpp/src/llm/unicode_data.h @@ -0,0 +1,16 @@ +#pragma once +// Unicode tables generated from Python (src/agent/gen/gen_data.py unicode): +// full case mappings, whitespace, decimal digits and the final-sigma context sets. +#include +#include +#include +namespace detz::py::ucd { +struct CaseEntry { uint32_t cp; const char* text; }; +struct CpRange { uint32_t first, last; }; +const std::vector& LowerMap(); // str.lower() full mapping (cp -> UTF-8) +const std::vector& UpperMap(); // str.upper() full mapping +const std::vector& SpaceRanges(); // str.isspace() +const std::vector& SigmaIgnorableRanges(); // case-ignorable (skipped by the final-sigma rule) +const std::vector& SigmaCasedRanges(); // cased, for the final-sigma rule +const std::vector& DecimalRanges(); // str.isdecimal(), ranges of cp +} // namespace detz::py::ucd diff --git a/taicpp/tests/agent_diff.cpp b/taicpp/tests/agent_diff.cpp new file mode 100644 index 0000000..49c5db9 --- /dev/null +++ b/taicpp/tests/agent_diff.cpp @@ -0,0 +1,273 @@ +// Differential driver for the agent core: reads one JSON case per line from +// stdin ({"fn": name, "args": [...]}) and prints one JSON result per line. +// tests/agent_diff.py computes the same cases with tai.py and compares. +#include "../src/agent/agent.h" +#include "../src/agent/text.h" +#include "../src/agent/data.h" +#include "../src/llm/cli_backends.h" +#include "../src/llm/helpers.h" +#include "../src/llm/model_health.h" +#include "../src/llm/muse.h" +#include "../src/llm/py.h" +#include +#include +#include +using namespace detz; +using json::Value; + +namespace { +Value S(const std::string& s) { return Value(s); } +Value strs(const std::vector& v) { + Value a = Value::array(); + for (const auto& s : v) { a.push(s); } + return a; +} +Value verdict_value(const agent::GoalVerdict& v, const std::string& goal) { + Value o = Value::object(); + o.set("ok", v.ok); + Value fs = Value::array(); + for (const auto& f : v.failures) { fs.push(f.to_dict()); } + o.set("failures", fs); + o.set("commentary", v.commentary); + o.set("raw", v.raw); + o.set("report", v.report(goal)); + o.set("followup", v.followup()); + return o; +} +Value try_value(const std::function& f) { + try { + return f(); + } catch (const llm::BackendError& e) { + Value o = Value::object(); + o.set("backend_error", std::string(e.what())); + o.set("status", e.status()); + return o; + } catch (const std::exception& e) { + Value o = Value::object(); + o.set("exception", std::string(e.what())); + return o; + } +} + +using Fn = std::function; +std::map registry() { + std::map r; + r["json_roundtrip"] = [](const Value& a) { + return try_value([&]() { return S(Value::loads(a.at(0).str()).dumps()); }); + }; + r["json_compact_sorted"] = [](const Value& a) { + return try_value([&]() { + json::DumpOptions o; + o.sort_keys = true; o.item_sep = ","; o.key_sep = ":"; + return S(Value::loads(a.at(0).str()).dumps(o)); + }); + }; + r["schemas"] = [](const Value&) { + Value arr = Value::array(); + for (const auto& s : agent::tool_schemas()) { arr.push(s); } + return S(arr.dumps()); + }; + r["select_tools"] = [](const Value& a) { + Value picked = agent::select_tools(a.at(0).str()); + Value names = Value::array(); + for (const auto& s : picked.items()) { names.push(s.get("function").get("name")); } + return names; + }; + r["select_tools_extra"] = [](const Value& a) { + Value picked = agent::select_tools(a.at(0).str(), a.at(1)); + Value names = Value::array(); + for (const auto& s : picked.items()) { names.push(s.get("function").get("name")); } + return names; + }; + r["tool_catalog"] = [](const Value&) { return S(agent::tool_catalog()); }; + r["truncate"] = [](const Value& a) { + return S(llm::truncate(a.at(0).str(), static_cast(a.at(1).as_int()), static_cast(a.at(2).as_int()))); + }; + r["estimate_tokens"] = [](const Value& a) { return Value(llm::estimate_tokens(a.at(0))); }; + r["normalize_tag"] = [](const Value& a) { return S(agent::normalize_tag(a.at(0).str())); }; + r["normalize_tags"] = [](const Value& a) { + std::vector in; + for (const auto& x : a.at(0).items()) { in.push_back(x.str()); } + return strs(agent::normalize_tags(in)); + }; + r["singular_noun"] = [](const Value& a) { return S(agent::singular_noun(a.at(0).str())); }; + r["tag_variants"] = [](const Value& a) { return strs(agent::tag_variants(a.at(0).str())); }; + r["parse_node_id"] = [](const Value& a) { + auto p = agent::parse_node_id(a.at(0).str()); + if (!p.has_value()) { return Value(); } + Value o = Value::array(); + o.push(p->first); o.push(p->second); + return o; + }; + r["secret_env_name"] = [](const Value& a) { return S(agent::secret_env_name(a.at(0).str())); }; + r["format_delay"] = [](const Value& a) { return S(agent::format_delay(a.at(0).as_int())); }; + r["parse_duration"] = [](const Value& a) { + auto d = agent::parse_duration(a.at(0).str()); + return d.has_value() ? Value(*d) : Value(); + }; + r["parse_schedule_at"] = [](const Value& a) { + return try_value([&]() { return S(agent::parse_schedule_at(a.at(0).str())); }); + }; + r["local_display"] = [](const Value& a) { return S(agent::local_display(a.at(0).str())); }; + r["html_unescape"] = [](const Value& a) { return S(agent::html_unescape(a.at(0).str())); }; + r["url_quote_plus"] = [](const Value& a) { return S(agent::url_quote_plus(a.at(0).str())); }; + r["chunk_lines"] = [](const Value& a) { return strs(agent::chunk_lines(a.at(0).str())); }; + r["collapse_repeated_lines"] = [](const Value& a) { return S(agent::collapse_repeated_lines(a.at(0).str())); }; + r["tool_name_clean"] = [](const Value& a) { return S(llm::tool_name_clean(a.at(0))); }; + r["tool_args_valid"] = [](const Value& a) { return S(llm::tool_args_valid(a.at(0))); }; + r["tool_args_text"] = [](const Value& a) { return S(llm::tool_args_text(a.at(0))); }; + r["action_signature"] = [](const Value& a) { return S(llm::action_signature(a.at(0).str(), a.at(1))); }; + r["result_digest"] = [](const Value& a) { return S(llm::result_digest(a.at(0).str())); }; + r["clean_history"] = [](const Value& a) { return llm::clean_history(a.at(0)); }; + r["conversation_tokens"] = [](const Value& a) { return Value(llm::conversation_tokens(a.at(0), a.size() > 1 ? a.at(1) : Value())); }; + r["conversation_text"] = [](const Value& a) { return S(agent::conversation_text(a.at(0))); }; + r["approval_parse"] = [](const Value& a) { return S(agent::Approval::parse_allow(a.at(0).str())); }; + r["approval_affirm"] = [](const Value& a) { return Value(agent::Approval::is_affirmative(a.at(0).str())); }; + r["goal_verdict"] = [](const Value& a) { + return verdict_value(agent::GoalVerdict::from_text(a.at(0).str()), a.at(1).str()); + }; + r["extract_json"] = [](const Value& a) { + auto [data, err] = agent::GoalVerifier::extract_json(a.at(0).str()); + Value o = Value::array(); + o.push(data); + o.push(err.has_value() ? Value(*err) : Value()); + return o; + }; + r["goal_transcript"] = [](const Value& a) { return S(agent::GoalVerifier::transcript(a.at(0))); }; + r["goal_tried"] = [](const Value& a) { return strs(agent::GoalVerifier::tried_actions(a.at(0))); }; + r["goal_task"] = [](const Value& a) { + std::vector tried; + for (const auto& x : a.at(3).items()) { tried.push_back(x.str()); } + return S(agent::GoalVerifier::build_task(a.at(0).str(), a.at(1).str(), a.at(2).str(), tried)); + }; + r["muse_prompt"] = [](const Value& a) { return S(llm::muse_prompt(a.at(0), a.at(1))); }; + r["opencode_prompt"] = [](const Value& a) { return S(llm::opencode_prompt(a.at(0), a.at(1))); }; + r["opencode_parse"] = [](const Value& a) { return llm::opencode_parse(a.at(0).str()); }; + r["opencode_find_answer"] = [](const Value& a) { return llm::opencode_find_answer(a.at(0).str()); }; + r["opencode_parse_events"] = [](const Value& a) { + return try_value([&]() { return S(llm::opencode_parse_events(a.at(0).str(), static_cast(a.at(1).as_int()), a.at(2).str())); }); + }; + r["grok_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::grok_parse_output(a.at(0).str()); }); }; + r["claude_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::claude_parse_output(a.at(0).str()); }); }; + r["gemini_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::gemini_parse_output(a.at(0).str()); }); }; + r["codex_parse_output"] = [](const Value& a) { + return try_value([&]() { return llm::codex_parse_output(a.at(0).str(), a.at(1).str(), a.at(2).as_bool()); }); + }; + r["muse_valid_answer"] = [](const Value& a) { return llm::muse_valid_answer(a.at(0).str()); }; + r["muse_message_hash"] = [](const Value& a) { return S(llm::muse_message_hash(a.at(0))); }; + r["model_weight"] = [](const Value& a) { return Value(llm::model_weight(a.at(0))); }; + r["model_outcome_reward"] = [](const Value& a) { + auto opt = [&](size_t i) -> std::optional { return a.at(i).is_null() ? std::nullopt : std::optional(a.at(i).as_double()); }; + return Value(llm::model_outcome_reward(opt(0), opt(1))); + }; + r["summarize_errors"] = [](const Value& a) { + std::vector errs; + for (const auto& x : a.at(0).items()) { errs.push_back(x.str()); } + return S(llm::ChatClient::summarize_errors(errs, static_cast(a.at(1).as_int()), static_cast(a.at(2).as_int()))); + }; + r["split_models"] = [](const Value& a) { return strs(llm::split_models(a.at(0).str())); }; + r["parse_backends"] = [](const Value& a) { + Value arr = Value::array(); + for (const auto& b : llm::parse_backends(a.at(0).str())) { + Value o = Value::object(); + o.set("label", b.label); o.set("base", b.base); o.set("key", b.key); o.set("model", b.model); o.set("profile", b.profile); + arr.push(o); + } + return arr; + }; + r["is_free_model"] = [](const Value& a) { return Value(llm::is_free_model(a.at(0).str())); }; + r["embedded_failure_message"] = [](const Value& a) { return S(llm::embedded_failure_message(a.at(0))); }; + r["stream_status"] = [](const Value& a) { + return S(agent::stream_status(static_cast(a.at(0).as_int()), a.at(1).str(), a.at(2).as_double(), a.at(3).as_bool())); + }; + r["curl_hint"] = [](const Value& a) { return S(agent::curl_hint(a.at(0).str())); }; + r["format_schedule_lines"] = [](const Value& a) { return S(agent::format_schedule_lines(a.at(0), a.at(1).as_double())); }; + r["py_repr"] = [](const Value& a) { return S(py::repr(a.at(0))); }; + r["py_str"] = [](const Value& a) { return S(py::str(a.at(0))); }; + r["py_strip"] = [](const Value& a) { return S(py::strip(a.at(0).str())); }; + r["py_lower"] = [](const Value& a) { return S(py::lower(a.at(0).str())); }; + r["py_splitlines"] = [](const Value& a) { return strs(py::splitlines(a.at(0).str())); }; + r["py_split_ws"] = [](const Value& a) { return strs(py::split_ws(a.at(0).str())); }; + r["py_int"] = [](const Value& a) { + auto v = py::to_int(a.at(0)); + return v.has_value() ? Value(*v) : Value(); + }; + r["py_float"] = [](const Value& a) { + auto v = py::to_float(a.at(0)); + if (!v.has_value()) { return Value(); } + return Value(*v); + }; + r["py_slice"] = [](const Value& a) { + std::optional s, e; + if (!a.at(1).is_null()) { s = static_cast(a.at(1).as_int()); } + if (!a.at(2).is_null()) { e = static_cast(a.at(2).as_int()); } + return S(py::slice(a.at(0).str(), s, e)); + }; + r["py_len"] = [](const Value& a) { return Value(static_cast(py::len(a.at(0).str()))); }; + r["float_repr"] = [](const Value& a) { return S(json::float_repr(a.at(0).as_double())); }; + r["sha1"] = [](const Value& a) { return S(py::sha1_hex(a.at(0).str())); }; + r["sha256"] = [](const Value& a) { return S(py::sha256_hex(a.at(0).str())); }; + r["utf8_replace"] = [](const Value& a) { + // bytes arrive as a latin-1 style string of code points 0-255 + std::string raw; + for (uint32_t cp : py::decode(a.at(0).str())) { raw += static_cast(cp); } + return S(py::utf8_replace(raw)); + }; + r["orientation_block"] = [](const Value& a) { return S(agent::orientation_block(a.at(0).str(), a.at(1).as_bool())); }; + r["capability_block"] = [](const Value& a) { return S(agent::capability_block(a.at(0).as_bool(), a.at(1).as_bool())); }; + r["split_leading_tags"] = [](const Value& a) { + auto [tags, rest] = agent::split_leading_tags(a.at(0).str()); + Value o = Value::array(); + o.push(strs(tags)); + o.push(rest); + return o; + }; + r["expand_file_mentions"] = [](const Value& a) { return S(agent::expand_file_mentions(a.at(0).str(), a.at(1).str())); }; + r["profile_ok"] = [](const Value& a) { return Value(agent::profile_ok(a.at(0).str())); }; + r["bot_ok"] = [](const Value& a) { return Value(agent::bot_ok(a.at(0).str())); }; + r["secret_ok"] = [](const Value& a) { return Value(agent::secret_ok(a.at(0).str())); }; + r["skill_name_ok"] = [](const Value& a) { return Value(agent::skill_name_ok(a.at(0).str())); }; + r["mcp_server_ok"] = [](const Value& a) { return Value(agent::mcp_server_ok(a.at(0).str())); }; + r["create_skill_prompt"] = [](const Value& a) { + return S(agent::create_skill_prompt(a.at(0).str(), a.at(1).str(), a.at(2).str())); + }; + r["data_texts"] = [](const Value&) { + Value o = Value::object(); + o.set("default_system", agent::data::DefaultSystem()); + o.set("merge_system", agent::data::MergeSystem()); + o.set("compact_system", agent::data::CompactSystem()); + o.set("autonomous_note", agent::data::AutonomousNote()); + o.set("legacy_password_note", agent::data::LegacyPasswordNote()); + o.set("vault_memory_note", agent::data::VaultMemoryNote()); + o.set("mcp_resolver_brief", agent::data::McpResolverBrief()); + o.set("goal_task", agent::data::GoalVerifierTask()); + o.set("curl_hint", agent::data::CurlHint()); + return o; + }; + return r; +} +} // namespace + +int main() { + auto reg = registry(); + std::string line; + while (std::getline(std::cin, line)) { + if (line.empty()) { continue; } + Value out; + try { + Value c = Value::loads(line); + auto it = reg.find(c.get("fn").str()); + if (it == reg.end()) { + out = Value::object(); + out.set("missing_fn", c.get("fn")); + } else { + out = it->second(c.get("args")); + } + } catch (const std::exception& e) { + out = Value::object(); + out.set("harness_error", std::string(e.what())); + } + std::cout << out.dumps() << "\n"; + } + return 0; +} diff --git a/taicpp/tests/agent_diff.py b/taicpp/tests/agent_diff.py new file mode 100644 index 0000000..4f2d88b --- /dev/null +++ b/taicpp/tests/agent_diff.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""Differential test of the C++ agent core against tai.py. + +Builds cases (curated plus seeded fuzz), computes the expected result with +tai.py, feeds the same cases to build/agent_diff (tests/agent_diff.cpp) and +compares the JSON results. Exit 0 only when every case matches. + + python3 tests/agent_diff.py [--seed N] [--fuzz N] [--bin build/agent_diff] [-v] +""" +import argparse +import hashlib +import html +import json +import os +import random +import subprocess +import sys +import tempfile +import urllib.parse +from datetime import datetime, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) +# a hermetic environment: no tmux, no plexer, no tui socket, fixed zone +HOME = tempfile.mkdtemp(prefix="agent-diff-home-") +os.environ.pop("TMUX", None) +os.environ.pop("TMUX_PANE", None) +for key in list(os.environ): + if key.startswith("TAI_"): + del os.environ[key] +os.environ["TAI_HOME"] = HOME +os.environ["TZ"] = "Europe/Amsterdam" +sys.path.insert(0, ROOT) +sys.dont_write_bytecode = True +import tai # noqa: E402 + + +def dumps_sorted(value): + return json.dumps(value, sort_keys=True, ensure_ascii=True) + + +def guard(func): + def run(*args): + try: + return func(*args) + except tai.BackendError as exc: + return {"backend_error": str(exc), "status": exc.status} + except Exception as exc: # noqa: BLE001 + return {"exception": str(exc)} + return run + + +def verdict_value(text, goal): + v = tai.GoalVerdict.from_text(text) + return {"ok": v.ok, "failures": [f.to_dict() for f in v.failures], "commentary": v.commentary, + "raw": v.raw, "report": v.report(goal), "followup": v.followup()} + + +def names(picked): + return [s["function"]["name"] for s in picked] + + +def int_or_none(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + +def float_or_none(value): + try: + return float(value) + except (TypeError, ValueError): + return None + + +def schedule_now(epoch): + return datetime.fromtimestamp(epoch, timezone.utc) + + +EXPECT = { + "json_roundtrip": guard(lambda text: json.dumps(json.loads(text))), + "json_compact_sorted": guard(lambda text: json.dumps(json.loads(text), sort_keys=True, separators=(",", ":"))), + "schemas": lambda: json.dumps(tai.TOOL_SCHEMAS), + "select_tools": lambda text: names(tai.select_tools(text)), + "select_tools_extra": lambda text, extra: names(tai.select_tools(text, extra)), + "tool_catalog": lambda: tai.tool_catalog(), + "truncate": lambda text, head, tail: tai.truncate(text, head, tail), + "estimate_tokens": lambda value: tai.estimate_tokens(value), + "normalize_tag": lambda raw: tai.normalize_tag(raw), + "normalize_tags": lambda items: tai.normalize_tags(items), + "singular_noun": lambda word: tai.singular_noun(word), + "tag_variants": lambda tag: tai.tag_variants(tag), + "parse_node_id": lambda raw: (lambda r: list(r) if r else None)(tai.parse_node_id(raw)), + "secret_env_name": lambda name: tai.secret_env_name(name), + "format_delay": lambda s: tai.format_delay(s), + "parse_duration": lambda raw: tai.parse_duration(raw), + "parse_schedule_at": guard(lambda raw: tai.parse_schedule_at(raw)), + "local_display": lambda raw: tai.local_display(raw), + "html_unescape": lambda text: html.unescape(text), + "url_quote_plus": lambda text: urllib.parse.urlencode({"q": text})[2:], + "chunk_lines": lambda text: tai.chunk_lines(text), + "collapse_repeated_lines": lambda text: tai.collapse_repeated_lines(text), + "tool_name_clean": lambda raw: tai.tool_name_clean(raw), + "tool_args_valid": lambda raw: tai.tool_args_valid(raw), + "tool_args_text": lambda raw: tai.tool_args_text(raw), + "action_signature": lambda name, raw: tai.action_signature(name, raw), + "result_digest": lambda text: tai.result_digest(text), + "clean_history": lambda messages: tai.clean_history(messages), + "conversation_tokens": lambda messages, tools=None: tai.conversation_tokens(messages, tools), + "conversation_text": lambda messages: tai.conversation_text(messages), + "approval_parse": lambda answer: tai.Approval.parse_allow(answer), + "approval_affirm": lambda text: tai.Approval.is_affirmative(text), + "goal_verdict": lambda text, goal: verdict_value(text, goal), + "extract_json": lambda text: list(tai.GoalVerifier.extract_json(text)), + "goal_transcript": lambda messages: tai.GoalVerifier.transcript(messages), + "goal_tried": lambda messages: tai.GoalVerifier.tried_actions(messages), + "goal_task": lambda goal, final, transcript, tried: tai.GoalVerifier.build_task(goal, final, transcript, tried), + "muse_prompt": lambda messages, tools: tai.muse_prompt(messages, tools), + "opencode_prompt": lambda messages, tools: tai.opencode_prompt(messages, tools), + "opencode_parse": lambda text: tai.opencode_parse(text), + "opencode_find_answer": lambda text: tai.opencode_find_answer(text), + "opencode_parse_events": guard(lambda out, code, err: tai.opencode_parse_events(out, code, err)), + "grok_parse_output": guard(lambda text: tai.grok_parse_output(text)), + "claude_parse_output": guard(lambda text: tai.claude_parse_output(text)), + "gemini_parse_output": guard(lambda text: tai.gemini_parse_output(text)), + "codex_parse_output": guard(lambda out, outfile, used: tai.codex_parse_output(out, outfile, used)), + "muse_valid_answer": lambda text: tai.muse_valid_answer(text), + "muse_message_hash": lambda message: tai.muse_message_hash(message), + "model_weight": lambda entry: tai.model_weight(entry), + "model_outcome_reward": lambda lat, tps: tai.model_outcome_reward(lat, tps), + "summarize_errors": lambda errors, shown, width: tai.ChatClient.summarize_errors(errors, shown, width), + "split_models": lambda spec: tai.split_models(spec), + "parse_backends": lambda spec: tai.parse_backends(spec), + "is_free_model": lambda model: tai.is_free_model(model), + "embedded_failure_message": lambda data: tai.embedded_failure_message(data), + "stream_status": lambda code, output, elapsed, timed_out: tai.stream_status(code, output, elapsed, timed_out), + "curl_hint": lambda command: tai.curl_hint(command), + "format_schedule_lines": lambda items, now: tai.format_schedule_lines(items, schedule_now(now)), + "py_repr": lambda value: repr(value), + "py_str": lambda value: str(value), + "py_strip": lambda text: text.strip(), + "py_lower": lambda text: text.lower(), + "py_splitlines": lambda text: text.splitlines(), + "py_split_ws": lambda text: text.split(), + "py_int": lambda value: int_or_none(value), + "py_float": lambda value: float_or_none(value), + "py_slice": lambda text, start, end: text[start:end], + "py_len": lambda text: len(text), + "float_repr": lambda value: repr(float(value)), + "sha1": lambda text: hashlib.sha1(text.encode("utf-8", "surrogatepass")).hexdigest(), + "sha256": lambda text: hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest(), + "utf8_replace": lambda text: text.encode("latin-1").decode("utf-8", "replace"), + "orientation_block": lambda directory, sandbox: tai.orientation_block(directory or None, sandbox), + "capability_block": lambda yolo, auto: tai.capability_block(yolo, auto), + "split_leading_tags": lambda text: list(tai.split_leading_tags(text)), + "expand_file_mentions": lambda text, directory: tai.expand_file_mentions(text, directory), + "profile_ok": lambda s: bool(tai.PROFILE_RE.match(s)), + "bot_ok": lambda s: bool(tai.BOT_RE.match(s)), + "secret_ok": lambda s: bool(tai.SECRET_RE.match(s)), + "skill_name_ok": lambda s: bool(tai.SKILL_NAME_RE.match(s)), + "mcp_server_ok": lambda s: bool(tai.MCP_SERVER_RE.match(s)), + "create_skill_prompt": lambda name, brief, skill_dir: tai.create_skill_prompt(name, brief, skill_dir), + "data_texts": lambda: {"default_system": tai.DEFAULT_SYSTEM, "merge_system": tai.MERGE_SYSTEM, + "compact_system": tai.COMPACT_SYSTEM, "autonomous_note": tai.AUTONOMOUS_NOTE, + "legacy_password_note": tai.LEGACY_PASSWORD_NOTE, "vault_memory_note": tai.VAULT_MEMORY_NOTE, + "mcp_resolver_brief": tai.MCP_RESOLVER_BRIEF, "goal_task": tai.GoalVerifier.TASK, + "curl_hint": tai.CURL_HINT}, +} + +POOL = ["a", "b", "c", "x", "Y", "Z", "0", "1", "9", " ", " ", "\t", "\n", "\r\n", "-", "_", ".", ",", ";", ":", "!", "?", + "(", ")", "[", "]", "{", "}", "\"", "'", "\\", "/", "@", "#", "&", "<", ">", "=", "+", "*", "%", "é", "é", + "ß", "İ", "K", " ", "
", "　", "中", "文", "\U0001f600", "€", "\x7f", "\x1c", + "TODO", "token", "secret", "readme", "README", "http://x", "curl", "wget", "&", "<", "A", "A", " ", + "©", "&unknown;", "ies", "ses", "news", "ss", "tags", "record"] + + +def fuzz_text(rng, max_parts=14): + return "".join(rng.choice(POOL) for _ in range(rng.randint(0, max_parts))) + + +def fuzz_json(rng, depth=0): + kind = rng.randint(0, 9 if depth < 3 else 5) + if kind == 0: + return None + if kind == 1: + return rng.choice([True, False]) + if kind == 2: + return rng.randint(-1000, 10 ** 12) + if kind == 3: + return rng.choice([0.0, 1.5, -2.25, 1e20, 1e-7, 123456.789, 0.1, 1 / 3]) + if kind in (4, 5): + return fuzz_text(rng, 6) + if kind in (6, 7): + return [fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))] + return {fuzz_text(rng, 3): fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))} + + +def fuzz_messages(rng): + messages = [] + for _ in range(rng.randint(0, 7)): + role = rng.choice(["system", "user", "assistant", "tool"]) + item = {"role": role, "content": rng.choice([fuzz_text(rng, 5), None, "", fuzz_text(rng, 30)])} + if role == "assistant" and rng.random() < 0.6: + calls = [] + for k in range(rng.randint(1, 3)): + name = rng.choice(["shell", "read_file", "bad name!", "x" * 70, "web_fetch", "", "a.b-c_d"]) + args = rng.choice(['{"a": 1}', "", "not json", "{}", None, {"a": 1}, "[1, 2]"]) + calls.append({"id": "c%d" % rng.randint(0, 3), "type": "function", "function": {"name": name, "arguments": args}}) + item["tool_calls"] = calls + if role == "tool": + item["tool_call_id"] = "c%d" % rng.randint(0, 3) + messages.append(item) + return messages + + +def build_cases(rng, fuzz): + cases = [] + + def add(fn, *args): + cases.append({"fn": fn, "args": list(args)}) + + add("schemas") + add("tool_catalog") + add("data_texts") + prompts = ["", "hello", "please run the tests and fix the failing build", "search the web for rust news", "read the file readme", + "schedule a reminder every hour", "remember my favorite color", "store a password in the vault", "delete the old file", + "describe this image screenshot", "fork a subagent to research in parallel", "install the skill from github", + "mcp server list", "audit changes since yesterday", "release a new version", "create a bot persona", "graph link nodes", + "tail terminals tmux panes", "what is the system info of this machine?", "translate this poem", + "SEARCH the WEB", "Kelvin K sign and İstanbul", "tag tags tagging", "ask the user a question and wait"] + for p in prompts: + add("select_tools", p) + for _ in range(fuzz): + add("select_tools", fuzz_text(rng, 25)) + extra = [{"type": "function", "function": {"name": "mcp_demo_weather", "description": "Fetch the weather forecast for a city", "parameters": {}}}, + {"type": "function", "function": {"name": "mcp_demo_list", "description": "short", "parameters": {}}}] + for p in ["weather forecast please", "list things", "forecast", ""]: + add("select_tools_extra", p, extra) + + for _ in range(fuzz): + t = fuzz_text(rng, 40) + add("truncate", t * rng.randint(1, 120), rng.choice([10, 100, 1500]), rng.choice([5, 50, 500])) + add("estimate_tokens", rng.choice([t, None, "", [1, 2, 3, 4, 5]])) + add("normalize_tag", t) + add("normalize_tags", [fuzz_text(rng, 4) for _ in range(rng.randint(0, 25))]) + add("singular_noun", fuzz_text(rng, 3)) + add("tag_variants", fuzz_text(rng, 3)) + add("secret_env_name", t) + add("html_unescape", t) + add("url_quote_plus", t) + add("chunk_lines", "\n".join(fuzz_text(rng, 30) for _ in range(rng.randint(0, 200)))) + add("collapse_repeated_lines", "\n".join(rng.choice(["a", "a", "a", "b", fuzz_text(rng, 2)]) for _ in range(rng.randint(0, 30)))) + add("tool_name_clean", rng.choice([t, None, "shell<|channel|>commentary", " x "])) + add("tool_args_valid", rng.choice([t, None, "{}", '{"a":1}', {"a": 1}, "", "[1]"])) + add("tool_args_text", rng.choice([t, None, "", {"a": [1, "é"]}, ["x"], 0, 5])) + add("action_signature", rng.choice(["shell", "read_file"]), rng.choice(['{"b":1,"a":2}', "nope", None, "", {"z": 1, "a": 2}])) + add("result_digest", t) + msgs = fuzz_messages(rng) + add("clean_history", msgs) + add("conversation_tokens", msgs) + add("conversation_text", msgs) + add("goal_transcript", msgs) + add("goal_tried", msgs) + add("approval_parse", t) + add("approval_affirm", rng.choice([t, "Yes!", " go ahead. ", "LOOKS good", "yes, but no", "ok.?"])) + add("py_repr", fuzz_json(rng)) + add("py_str", fuzz_json(rng)) + add("py_strip", t) + add("py_lower", t) + add("py_splitlines", t) + add("py_split_ws", t) + add("py_int", rng.choice([fuzz_json(rng), "12", " 7 ", "1_0", "0x10", "1e3", "-5", "+3", "", "٣"])) + add("py_float", rng.choice([fuzz_json(rng), "1.5", " 2 ", "1e3", "nan", "inf", "-Infinity", "1_0.5", "0x1p3", "."])) + add("py_slice", t, rng.choice([None, -3, 0, 2, 50]), rng.choice([None, -1, 0, 4, 99])) + add("py_len", t) + add("float_repr", rng.choice([0.1, 1e16, 1e-5, 123456789.123456789, 5e-324, 1.7976931348623157e308, 100.0, 1 / 3, 2.5e22])) + add("sha1", t) + add("sha256", t) + add("split_leading_tags", rng.choice(["@a b", "@a @b rest", "@@x", "plain", "@ x", "@x　y", "@a\n@b\tc " + t])) + add("profile_ok", rng.choice([t, "abc", "abc\n", "a" * 32, "a" * 33, "-x", "a_b-c"])) + add("bot_ok", rng.choice([t, "abc", "ABC", "a-b_c"])) + add("secret_ok", rng.choice([t, "a.b-c", "a" * 64, "a" * 65])) + add("skill_name_ok", rng.choice([t, "ab", "a-b", "a-", "-a", "a" * 64, "a" * 65, "x\n"])) + add("mcp_server_ok", rng.choice([t, "ok.name", "a" * 64])) + add("json_roundtrip", json.dumps(fuzz_json(rng), ensure_ascii=rng.random() < 0.5)) + add("json_compact_sorted", json.dumps(fuzz_json(rng))) + add("model_outcome_reward", rng.choice([None, 0, 120.5, 8000]), rng.choice([None, 0, 12.5, 100])) + add("model_weight", rng.choice([{}, {"s": 3, "f": 1, "r": 2.5}, {"s": 0, "f": 0}, {"r": 1.0, "s": None, "f": 2}])) + + for raw in ["", "{", "[1,]", '{"a":1,}', '{"a":1 "b":2}', '{"a" 1}', '"abc', '"a\\q"', '"\\u12"', "tru", "nan", "NaN", "-", "1e", "[1 2]", + "[1,2] x", '{"a":1}\n', "\t[ ]", '"a\tb"', '{"é": "ü\\ud83d\\ude00"}', '"\\ud800"', "1.5e300", "1" * 30, "-0", "-0.0", + "[" * 5 + "]" * 5, "{}", '{"a":{"b":[1,2,{"c":null}]}}', '{"a":1,"a":2}', "1.", ".5", "01", "+1"]: + add("json_roundtrip", raw) + add("json_compact_sorted", raw) + + for raw in ["", "2026-10-08T09:00", "2026-10-08 09:00:15.5", "2026-10-08T09:00Z", "2026-10-08T09:00+02:00", "2026-10-08T09:00-0530", + "2026-10-08", "20261008", "2026-13-01", "2026-02-30", "garbage", "2026-10-08T25:00", "2026-03-29T02:30", "2026-10-25T02:30", + "2026-10-08T09:00:00.123456789", " 2026-10-08T09:00 ", "2026-10-08t09:00", "2026-10-08T09", "2026-10-08T09:00+02"]: + add("parse_schedule_at", raw) + add("local_display", raw) + for s in [-5, 0, 59, 60, 3599, 3600, 86399, 86400, 10 ** 7]: + add("format_delay", s) + for raw in ["", "5", "5s", "10 m", "3h", "2d", "1w", "x", "5x", "1.5h", " 7D ", "99999999999999999999"]: + add("parse_duration", raw) + items = [{"id": 1, "status": "pending", "every": 0, "next_run": "2026-10-25T12:00:00+00:00", "tags": ["a", "b"], "name": "x", "prompt": "p" * 100, "last": "done", "result": "r" * 200}, + {"id": 2, "status": "pending", "every": 3600, "next_run": "bad", "tags": [], "name": "", "prompt": "q", "last": "", "result": ""}] + add("format_schedule_lines", items, 1792924800.0) + add("format_schedule_lines", [], 1792924800.0) + + for raw in ["&<>"'", "¬it;", "∉", "�", "€", "€", "�", "�", "�", "&", "&x", "a&b", + "&;", "&#;", "&#x;", "&#xzz;", "&", "≂̸", "♥&hearts", "€", "A", "A", " x&NBSP;", "&" * 40]: + add("html_unescape", raw) + for raw in ["hello world", "a+b=c&d", "é中\U0001f600", "~_.-", "100%"]: + add("url_quote_plus", raw) + + for raw in ["", "{}", '```json\n{"success": true, "failures": []}\n```', '{"success": true, "failures": [{"item": "a", "reason": "b", "proposal": "c"}]}', + '{"success": false, "failures": [{"item": "a"}]}', '{"success": false, "failures": []}', '{"success": "yes", "failures": []}', + '{"success": true, "failures": [], "commentary": " fine "}', "no json", '{"success": true,', "[1, 2]", + 'text {"success": false, "failures": [{"item": "i", "reason": "r", "proposal": "p"}], "commentary": "c"} tail', + '{"success": false, "failures": ' + json.dumps([{"item": str(i), "reason": "r", "proposal": "p"} for i in range(12)]) + "}", + '{"success": true, "failures": "none"}', '{"success": null}', "```\n{}\n```", "```JSON {\"success\": true, \"failures\": []}"]: + add("goal_verdict", raw, " fix the\tbug ") + add("extract_json", raw) + add("goal_task", "do x", "done", "[user] hi", ["a", "b"]) + add("goal_task", "", "", "", []) + + sample_msgs = [{"role": "system", "content": "s"}, {"role": "user", "content": "do thing\nplease"}, + {"role": "assistant", "content": "ok", "tool_calls": [{"id": "1", "type": "function", "function": {"name": "shell", "arguments": '{"command": "ls"}'}}]}, + {"role": "tool", "tool_call_id": "1", "content": "out"}, + {"role": "user", "content": [{"type": "text", "text": "part one"}, {"type": "text", "text": "part two"}, "x"]}] + tools = [{"type": "function", "function": {"name": "t", "description": "d", "parameters": {"type": "object", "properties": {}, "required": []}}}] + add("muse_prompt", sample_msgs, tools) + add("muse_prompt", sample_msgs, None) + add("muse_prompt", [], []) + add("opencode_prompt", sample_msgs, tools) + for raw in ["", "plain answer", '{"content": "hi", "tool_calls": []}', '```json\n{"content": "x", "tool_calls": [{"name": "a", "arguments": "{}"}]}\n```', + 'noise {"schema": 1} then {"content": "late", "tool_calls": []} tail', '{"tool_calls": []}', '{"nothing": 1}', "{{{", '```\n```', + "```json\n```", "``````", "```json{\"content\":\"z\"}```", '{"content": "a"} {"content": "b"}']: + add("opencode_parse", raw) + add("opencode_find_answer", raw) + add("grok_parse_output", raw) + add("claude_parse_output", raw) + add("gemini_parse_output", raw) + add("codex_parse_output", raw, "", False) + add("codex_parse_output", "", raw, True) + add("codex_parse_output", raw, raw, False) + add("muse_valid_answer", raw) + for obj in [{"is_error": True, "result": "bad"}, {"error": "x"}, {"error": {"a": 1}}, {"result": " answer "}, {"text": "t"}, {"structured_output": {"content": "c", "tool_calls": []}}, + {"response": '{"content": "r", "tool_calls": []}'}, {"content": "c", "tool_calls": []}, {"output": "o"}, [1], 5, {"is_error": False, "result": 5}]: + raw = json.dumps(obj) + add("grok_parse_output", raw) + add("claude_parse_output", raw) + add("gemini_parse_output", raw) + add("muse_valid_answer", raw) + add("codex_parse_output", '{"type": "item.completed", "item": {"type": "agent_message", "text": "{\\"content\\": \\"q\\", \\"tool_calls\\": []}"}}\nnoise', "", False) + add("codex_parse_output", '{"type": "turn.failed", "message": "boom"}', "", False) + add("codex_parse_output", '{"type": "error", "error": {"message": "deep"}}', "", False) + add("codex_parse_output", '{"type": "error", "error": {"nomsg": 1}}', "", False) + add("opencode_parse_events", '{"type": "text", "part": {"text": "hello "}}\n{"type": "text", "part": {"text": "world"}}\nnot json\n', 0, "") + add("opencode_parse_events", '{"type": "error", "error": {"data": {"message": "m", "statusCode": 429}}}', 1, "") + add("opencode_parse_events", '{"type": "error", "error": {"name": "X"}}', 1, "") + add("opencode_parse_events", "", 3, "stderr text " * 40) + add("opencode_parse_events", "out", 3, "") + add("opencode_parse_events", '[1]\n{"type": "text"}\n', 0, "e") + + for ident, errs in enumerate([[], ["a/b: x"], ["a: CLI not found", "b: err1", "b: err1", "c: err2", "d: err3"], ["muse: muse CLI not found"], ["x"] * 3 + ["y" * 500]]): + add("summarize_errors", errs, 2, 320) + add("summarize_errors", errs, 6, 900) + for spec in ["a,b\nc, ,d", "", " x "]: + add("split_models", spec) + add("parse_backends", "lab|https://x/v1/|key|m1,m2|opencode;bad|x\nl2|http://h|k2") + add("parse_backends", "a|b|$NOPE_NOT_SET") + add("parse_backends", "") + for model in ["a:free", "x-FREE", "Model-free", "paid", "", "free"]: + add("is_free_model", model) + for d in [{"error": {"message": " m "}}, {"error": {"type": "t"}}, {"error": " s "}, {"type": "error", "message": "mm"}, {"type": "error"}, {"ok": 1}, [1], {"error": {}}]: + add("embedded_failure_message", d) + for args in [(0, "a\nb\n", 1.234, False), (1, "", 0.05, False), (0, "x" * 2000, 12.55, True), (124, "ü\r\n", 3.25, False)]: + add("stream_status", *args) + for command in ["curl http://x", "wget https://y", "curl localhost", "xcurl http://x", "echo curl_http", "WGET HTTP://X", "curl-x http://z"]: + add("curl_hint", command) + add("create_skill_prompt", "n", "brief\nmore", "/tmp/skills/n") + add("capability_block", False, False) + add("capability_block", True, False) + add("capability_block", False, True) + add("orientation_block", "", True) + add("orientation_block", HOME, False) + add("orientation_block", "/nonexistent/dir/../x", False) + workdir = tempfile.mkdtemp(prefix="agent-diff-work-") + with open(os.path.join(workdir, "a.txt"), "w") as handle: + handle.write("x") + for text in ["see @a.txt, and @missing.txt!", "@a.txt)", "mail me@a.txt", "@", "@@a.txt", "x @%s/a.txt y" % workdir, "@~", "no mentions", "@a.txt　@a.txt"]: + add("expand_file_mentions", text, workdir) + for raw in ["\xc3\xa9", "\xff\xfe", "\xe2\x82", "\xe2\x82\xac", "\xed\xa0\x80", "\xf0\x9f\x98", "\xf0\x9f\x98\x80", "\xc0\x80", "\xf4\x90\x80\x80", "ok", "\xe0\x80\x80\xe0\xa0\x80", "\x80\x80"]: + add("utf8_replace", raw) + for value in [None, True, 5, 2.5, "it's", 'say "hi"', "both ' and \"", "tab\there", "\x00\x1f\x7f", "é ​\U0001f600", [], {}, [1, "a", None, {"k": [True]}], {"a": {"b": 1.0}}, "\ud800"]: + add("py_repr", value) + add("py_str", value) + return cases + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--fuzz", type=int, default=150) + parser.add_argument("--bin", default=os.path.join(HERE, "..", "build", "agent_diff")) + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + rng = random.Random(args.seed) + cases = build_cases(rng, args.fuzz) + payload = "\n".join(json.dumps(case) for case in cases) + "\n" + env = dict(os.environ) + done = subprocess.run([args.bin], input=payload.encode("utf-8", "surrogatepass"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + if done.returncode != 0: + sys.stderr.write("binary failed (%s): %s\n" % (done.returncode, done.stderr.decode("utf-8", "replace")[-2000:])) + return 2 + outputs = done.stdout.decode("utf-8").splitlines() + if len(outputs) != len(cases): + sys.stderr.write("expected %d result lines, got %d\n" % (len(cases), len(outputs))) + return 2 + failures = 0 + per_fn = {} + for case, line in zip(cases, outputs): + expected_fn = EXPECT[case["fn"]] + try: + expected = expected_fn(*case["args"]) + except Exception as exc: # noqa: BLE001 + expected = {"exception": str(exc)} + actual = json.loads(line) + # documented limit: int() results beyond int64 clamp in C++ (Python ints are unbounded) + if case["fn"] in ("py_int", "parse_duration") and isinstance(expected, int) and not -2 ** 63 <= expected < 2 ** 63: + expected = 2 ** 63 - 1 if expected > 0 else -2 ** 63 + # normalize through JSON so tuples/ints compare like the C++ side + expected_norm = json.loads(json.dumps(expected)) + stats = per_fn.setdefault(case["fn"], [0, 0]) + stats[0] += 1 + if dumps_sorted(expected_norm) != dumps_sorted(actual): + failures += 1 + stats[1] += 1 + if failures <= 40 or args.verbose: + sys.stdout.write("MISMATCH %s %s\n expected: %s\n actual: %s\n" % ( + case["fn"], json.dumps(case["args"], ensure_ascii=True)[:300], dumps_sorted(expected_norm)[:400], dumps_sorted(actual)[:400])) + sys.stdout.write("%d cases, %d mismatches\n" % (len(cases), failures)) + for fn in sorted(per_fn): + total, bad = per_fn[fn] + if bad: + sys.stdout.write(" %-26s %4d cases %4d mismatches\n" % (fn, total, bad)) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())