diff --git a/taicpp/docs/foundation.md b/taicpp/docs/foundation.md new file mode 100644 index 0000000..5f7ad48 --- /dev/null +++ b/taicpp/docs/foundation.md @@ -0,0 +1,123 @@ +# foundation: store, config, session, repl, plexer + +Port of the vault, settings, debug log, conversation persistence, chat REPL and +plexer controllers of `tai.py` (spec files: `settings.md`, `logging.md`, +`uiflow.md`). Behaviour is the contract: every operation below is replayed +through both implementations by `tests/foundation_parity.py` and diffed. + +## Layout + +| Directory | Classes (one header + source each) | +|---|---| +| `src/store` | `Store` (store.cpp, store_vault.cpp, store_files.cpp, store_settings.cpp, store_mcp.cpp), `SqlDb`, `Schema`, `DebugLog`, `Spill`, `Json`, `Fts`, `TagRules`, `Patterns`, `VaultFiles`, `Clock`, `Utf8`, `Sha256`, `SealPort`/`NullSeal`, errors | +| `src/config` | `Config`, `SettingsCatalog`, `SettingsRules`, `BootSettings`, `Prompts`, constants | +| `src/session` | `ShellContext` (tai.py `ShellSession`), `Conversation` (persistence half of `Agent`), `DirDefaults`, `Term`, `Md5` | +| `src/repl` | `Repl`, `ReplHost`, `ReplStyle`, `LineInput`, `Interrupt`, `Boot` | +| `src/plexer` | `PlexerController`, `NativePlexerController`, `TmuxPlexerController`, `PlexerControllerFactory`, `FlowValidator`, `PlexerFlow`, `PlexerError`, `Plexer` (constants), `ProcRunner` | + +## Where history lives and how it is keyed (the pane question) + +tai.py does not read `TMUX_PANE` or `TAI_PLEXER_PANE` to key a conversation. +The key is the **shell context id** (`ShellSession.current`), derived from the +nearest shell ancestor of the process: + + YYYYMMDD-HHMMSS-p- start time of that shell, its pid, 8 hex of the kernel boot id + +A terminal pane is exactly one long-lived shell, so quitting and restarting tai +in the same pane (tmux, the native plexer, any terminal tab) resolves to the +same id and resumes the same messages; another pane has another shell and a +fresh history. `boot()` exports the id as `TAI_SESSION`; a plain word in +`TAI_SESSION` pins one shared context for every shell. + +Files, all under `/profiles`: + +| File | Content | +|---|---| +| `.session..json` | user/assistant/tool messages of the main bot, system messages never stored | +| `.bot..session..json` | same for another bot | +| `.turn..json` | `{"open", "goal", "steps", "tools", "updated"}` of the last turn (`--continue`) | +| `.sys.md`, `.bots.json`, `.bot..sys.md` | system messages and the bot registry | + +At most `FILES_CAP` (20) scoped files per profile and bot are kept; the oldest +are pruned on save. Load: `Conversation::switch_profile` (also run by the +constructor) calls `Store::load_session(profile, "main", ctx_id)`. Save: +`Conversation::close_turn`/`checkpoint` and the REPL after every turn and on +exit. A session file that does not parse is renamed `*.corrupt-`. + +The `shell_sessions` table records every shell id with its parent chain +(`ShellContext::register_in`, `/ctx`). + +## Symbol map + +| tai.py | C++ | +|---|---| +| `SCHEMA`, `SCHEMA_INDEXES`, `ensure_schema`, `migrate_profiles`, `rebuild_with_profile`, `table_pk_columns`, `ensure_column`, `setup_fts` | `Schema` | +| `Store.*` (history, events, search, secrets, tags, graph, records, chunks, audit, schedules, files, bots, settings, mcp, dir defaults) | `Store` same method names | +| `fts_escape`, `fts_match*`, `fts_rank` | `Fts` | +| `normalize_tag(s)`, `singular_noun`, `tag_variants`, `parse_node_id` | `TagRules` | +| `PROFILE_RE`, `BOT_RE`, `SECRET_RE`, `MCP_SERVER_RE` | `Patterns` | +| `atomic_write`, `sealed_write`, `quarantine` | `VaultFiles` | +| `now_iso`, `uuid4().hex` | `Clock` | +| `spill_output`, `spilled_view`, `spill_or_truncate`, `truncate`, `chunk_lines` | `Spill` | +| `DebugLog`, `LOG`, `--debug`, `--debug-format` | `DebugLog`, `DebugLog::Global()`, `Boot::configure_debug` | +| `SETTINGS_CATALOG`, `CONFIG_SETTING_FIELDS`, `SETTING_EXPORT_KEYS` | `SettingsCatalog()`, `ConfigSettingFields()`, `SettingExportKeys()` | +| `normalize_setting_value`, `parse_setting_env`, `parse_setting_stored`, `validate_setting_scope` | `SettingsRules` | +| `get_or_create_settings`, `get_all_settings`, `upsert_settings` | `Store` | +| `export_settings_to_env`, `boot_settings`, `Config.apply_store_settings` | `BootSettings::ExportToEnv`, `BootSettings::Boot`, `Config::ApplyStoreSettings` | +| `Config`, `parse_backends`, `split_models`, `known_model_ids` | `Config::Build`/`FromEnv`, `ParseBackends`, `SplitModels`, `KnownModelIds` | +| `DEFAULT_SYSTEM` and the other prompt texts, `orientation_block`, `capability_block` | `Prompts` (generated byte for byte from tai.py) | +| `ShellSession` | `session::ShellContext` | +| `Agent.switch_profile`, `switch_bot`, `apply_system`, `reset_history`, `update_system`, `checkpoint`, `close_turn`, `mark_turn_open` | `session::Conversation` | +| `apply_dir_defaults` | `session::DirDefaults::apply` | +| `boot` (config, seal, store, settings, shell context, agent) | `repl::Boot::open` | +| `banner`, `resume_turn`, `repl`, one-shot tail of `main`, `handle_command` for `/quit /exit /help /profiles /bots /bot /profile /ctx /clear` | `repl::Repl` | +| `Glow.statusbar`, `Glow.prompt_text`, banner glow strings, `strip_ansi`, `truncate_ansi` | `repl::ReplStyle` | +| `PlexerError`, `plexer_dir`, `PLEXER_*` constants | `plexer::PlexerError`, `plexer::Plexer` | +| `PlexerController`, `NativePlexerController`, `TmuxPlexerController`, `PlexerControllerFactory.detect` | same names in `detz::plexer` | +| `validate_flow_*`, `PlexerFlow` | `plexer::FlowValidator`, `plexer::PlexerFlow` | + +The python `Seal` is not part of this area: `Store` takes a `store::SealPort` +(`enabled`, `lock`, `unlock`, `safe_unlock`, `kPrefix = "tai1$"`). `NullSeal` +is the python "seal disabled" path. + +## Deliberate differences + +- Python dicts returned by `Store` are `store::Json` objects (insertion ordered); settings rows are `SettingRow`. +- `ShellSession` is named `ShellContext` because `detz::shell::ShellSession` is the command runner. +- A bare `/` at the prompt prints "unknown command, try /help" (tai.py raises IndexError). +- Without readline there is no line editing, arrow-key history or tab completion; `LineInput` is a plain line reader. +- `PlexerController::ask` takes the timeout as `Json` so an int stays an int on the wire like in python; a `double` overload exists. +- Unbounded python bigints in JSON become doubles. + +## Gaps (stated, not hidden) + +| Gap | Effect | +|---|---| +| No `Seal` implementation here (`SealPort` only) | `Boot::open` uses `NullSeal` when `TAI_PASSPHRASE` is unset (vault stored in plain text, which python seals on its next start); an explicit `TAI_PASSPHRASE` without a `seal_factory` is refused with a `SealError`; a vault sealed by python is refused ("memory is sealed"). `rotate_seal` and the default-key upgrade are not ported | +| readline (history recall, `complete_input`) | typed input is still recorded in `input_history` | +| `start_scheduler`, `running_agents`, `probe_backend` | host hooks (`ReplHost::on_start`, `running_agents`, `backend_status`) | +| `ensure_self_backup`, `snapshot_self`, `do_release` | script-specific, belong to release tooling | +| tmux `ask` with a client attached (menu/popup answered) | error and dismissal paths are compared with python; the answered path is unverified | +| `DebugLog` uncaught-exception hook | `std::set_terminate` analog of `sys.excepthook`, not exercised by a crash test | +| SIGINT in the middle of a turn | `Interrupt` flag and `TurnInterrupted` exist; not driven by a live signal test | +| Glow banner on a real tty | pieces are compared with python functions, the composed screen is not | + +## Caps (all static) + +`FILES_CAP` 20 scoped files, `ROWS_CAP` 1000 `shell_sessions`, `CHAIN_CAP` 64 process levels, +`INPUT_HISTORY_CAP` 1000 rows of 4000 chars, `LOG_DB_KEEP` 20000 log rows, `LOG_META_MAX` 16384 bytes, +`MEM_EVENTS_CAP` 10000 events in the in-memory index, `MCP_MAX_SERVERS` 32, `DIR_DEFAULTS_CAP` 512, +`SPILL_LIMIT` 6000 chars, `AUDIT_MAX_CHARS` 20000, `FILE_RECORD_MAX` 50000, flow limits in `Plexer`. + +## Tests + +Run from `taicpp/`: `make -f tests/foundation.mk test` (needs only g++, sqlite3, python3, tmux for the tmux cases). + +| Test | What it proves | +|---|---| +| `tests/foundation_parity.py` + `foundation_probe` | one scenario replayed through tai.py and the C++ store: results, every table, the FTS index, the schema DDL and every profile file identical (timestamps and random ids masked). Pure functions (JSON, tags, settings rules, prompts, chunking, shell context against a fake /proc, flow validator and driver, Glow pieces), DebugLog sinks, REPL slash commands, native plexer against a scripted backend, tmux against a private server, and vault portability (each implementation reads the vault the other wrote) | +| `tests/repl_persist.py` + `repl_probe` | the history bug: the REPL started repeatedly under a long-lived shell per pane resumes the same session file, another pane stays separate, a pinned `TAI_SESSION` is shared, an interrupted turn is resumed by `--continue`; every assertion reads the vault | +| `tests/foundation_test.cpp` | hash vectors, UTF-8 helpers, seal-enabled store paths with a stand-in seal, boot settings, dir defaults, turn files, plexer detection, line input | + +`repl_probe` runs the real `Repl`, `Conversation` and `Store` with a scripted +host in place of the model loop; no model is called. diff --git a/taicpp/docs/store.md b/taicpp/docs/store.md index 30567b8..83b335d 100644 --- a/taicpp/docs/store.md +++ b/taicpp/docs/store.md @@ -1,12 +1,17 @@ -# store parity (tai.py -> taicpp M3) +# store parity (tai.py -> taicpp) -| tai.py | taicpp | notes | +Superseded detail lives in [foundation.md](foundation.md). Summary of the +current state, replacing the M3 stub notes: + +| tai.py | taicpp | status | |---|---|---| -| Store (sqlite+FTS5, WAL) | detz::store::Store | sqlite3, WAL+NORMAL+timeout | -| ensure_schema/migrate_profiles | Store::migrate + schema_version | v1: records/chunks/logs | -| setup_fts + triggers | Store::setup_fts | fts_docs + ai/au/ad triggers | -| add/get/update/delete record | add_record/get_record/update_record/delete_record | parity | -| search (fts_docs MATCH) | Store::search | snippet col 4 | -| add_chunks/chunk_lines | Store::add_chunks | single-chunk insert | -| spill_output/spilled_view/spill_or_truncate | same names | LIM 4000 | -| DebugLog | detz::store::DebugLog | memory lines | +| `Store` (sqlite + FTS5, WAL, busy timeout 30 s, per-profile slices) | `detz::store::Store(const Config&, const SealPort&)` | all methods ported, replayed against tai.py | +| 17 tables, 19 indexes, FTS triggers, `migrate_profiles` | `Schema` | schema DDL compared byte for byte | +| `records.id` is `mem:<16 hex>` text | `add_record` returns `std::string` | the pre-foundation stub used integers and a different schema; it is gone | +| `DebugLog`, `LOG` | `DebugLog`, `DebugLog::Global()` | sentence and json sinks and the `logs` table compared | +| `spill_output`, `spilled_view`, `spill_or_truncate` | `Spill` | python semantics (6000 char limit, head 1500, tail 500) | +| session, turn, bots, profile files | `Store::load_session` and friends | compared file by file | + +Callers of the old stub (`Store st(path)`, `add_record` returning an integer, +`search`) must switch to `Store::Open(path)` and the python method names; see +`docs/wiring/foundation.md`. diff --git a/taicpp/docs/wiring/foundation.md b/taicpp/docs/wiring/foundation.md new file mode 100644 index 0000000..080caa3 --- /dev/null +++ b/taicpp/docs/wiring/foundation.md @@ -0,0 +1,78 @@ +# Wiring: foundation (for the integrator) + +Everything below is applied outside the foundation directories. The foundation +area edited none of `main.cpp`, `src/cli`, `Makefile`, `PARITY.md`. + +## 1. Makefile + +`ALL_SRCS` already picks up every `src/**/*.cpp` (store, session, config, plexer, repl). Needed changes: + +- Link line (`$(BIN)`, `detz_test`, `static`): `-lsqlite3` is enough; nothing else new. `-pthread` is not required by the foundation code. +- Replace the stale rule + + $(BUILD)/store_test: tests/store_test.cpp src/store/store.cpp + ... tests/store_test.cpp src/store/store.cpp -lsqlite3 -lcurl -o $@ + + by `$(BUILD)/store_test: tests/store_test.cpp $(ALL_SRCS)` compiled like `detz_test` (the new `Store` needs the other store, session and config sources). +- Test target: add `include tests/foundation.mk` (or call `$(MAKE) -f tests/foundation.mk test` from `make test`). It builds `build/foundation_probe`, `build/repl_probe`, `build/foundation_test` and runs the three suites. +- `plexer.mk` still says `SRCS += src/plexer/plexer.cpp`; the other `*.mk` files are not used by the Makefile. + +## 2. main.cpp (REPL, one-shot, debug, profile, --continue) + +```cpp +#include "repl/boot.h" +#include "repl/repl.h" +... +auto& o = cmd.opts; +detz::repl::Boot::configure_debug(o.debug, o.debug_format); // before anything else +if (!detz::repl::Boot::valid_profile(o.profile.value_or(o.telegram ? "telegram" : "default"))) return 1; +detz::repl::BootOptions boot; +boot.profile = o.profile.value_or(o.telegram ? "telegram" : "default"); +boot.yes = o.yes; boot.yolo = o.yolo; boot.autonomous = o.auto_; boot.no_opencode = o.no_opencode; +boot.version = detz::kVersion; +boot.mode = !o.prompt.empty() ? "one-shot" : "repl"; +boot.seal_factory = ...; // adapter from the Seal owner, see section 4 +boot.catalogs = {skill_catalog, tool_catalog, scope_changed, profile_reset}; // agent module hooks +std::unique_ptr app; +try { app = detz::repl::Boot::open(boot); } +catch (const detz::store::SealError& e) { std::fprintf(stderr, "seal error: %s\n", e.what()); return 2; } +MyAgentHost host(*app->conversation, ...); // implements detz::repl::ReplHost, appends messages to app->conversation +detz::repl::Repl::Options ro; ro.version = detz::kVersion; ro.resume = o.resume; ro.telegram = o.telegram; ro.scheduler = o.scheduler; +detz::repl::Repl repl(*app->conversation, host, ro); +if (!o.prompt.empty()) return repl.oneshot(join(o.prompt, " ")); // exit codes 0, 1, 130 as tai.py +return repl.run(); +``` + +Per-step order is tai.py `main`: debug sink, `--version` etc., then boot, then `--env`, `--eval-retrieval`, `--stats-rag`, `--scheduler`, `--telegram`, `--tui-local`, prompt, REPL. `Boot::open` prints `vault schema upgraded: ...` and `settings from vault: ...` like python. `Repl::run` and `Repl::oneshot` close the store. + +`--plexer-wrap` and `--mplex` are not foundation code; `plexer::PlexerControllerFactory::detect()` is what the plexer tools call. + +## 3. Agent module hooks + +- Implement `detz::repl::ReplHost` (`src/repl/repl_host.h`): `run_turn` must call `conversation.begin_turn(goal)`, append `user`/`assistant`/`tool` messages with `conversation.add_message` (or push Json into `conversation.messages()`), call `note_step()`/`note_tool()` while working, and `close_turn()` at the end (the equivalent of `Agent.run_turn`). Throw `repl::TurnInterrupted` (or poll `repl::Interrupt::requested()`) on Ctrl-C. +- `Conversation::switch_profile/switch_bot` call `Catalogs::profile_reset` (clear read files and secret grants) and `scope_changed` (`chat.scope = (profile, bot)`). +- `session::DirDefaults::apply(conversation, hooks)` replaces `apply_dir_defaults`; call it after `Boot::open` and print the returned line dimmed when non-empty. +- Record tools in `src/agent/exec.cpp` use the removed stub API. Replace (python names, ids are `mem:<16 hex>` strings): + + auto st = detz::store::Store::Open(workdir + "/detz.db"); + record_save : std::string id = st->add_record(kind, title, content); return ok(id); + record_read : auto r = st->get_record(id); if (!r) return err("not found"); return ok(r->get_string("content")); + record_search : for (auto& r : st->search_records(query)) ... (fields id, kind, title, size, reads, updated, tags) + search : for (auto& h : st->fts_search(query)) ... (fields item, kind, sub, title, snippet, rank) + record_delete : return ok(st->delete_record(id) ? "deleted" : "not found"); + + In the real app the tools should share the booted `Store` (`*app->store`) and the current profile instead of opening `detz.db`. +- Large tool output: `detz::store::Spill::spill_or_truncate(&store, kind, title, text)`. +- Debug events: `detz::store::DebugLog::Global().event(level, component, event, message, profile, bot, chat, meta)`; no-op while disabled. + +## 4. Seal + +`src/seal` is outside this area. Provide a class deriving `detz::store::SealPort` (`enabled`, `default_key`, `lock`, `unlock`; prefix `tai1$`) and return it from `BootOptions::seal_factory(home, passphrase, default_key)`. The factory also owns the python behaviours not ported here: `.seal` file handling, `rotate_seal` and the default-key to `TAI_PASSPHRASE` upgrade. Until then the vault is plain text when `TAI_PASSPHRASE` is unset and `Boot::open` refuses an explicit passphrase. + +## 5. CLI facts the foundation relies on + +`src/cli/cli.h` `Options` already has `profile`, `debug`, `debug_format`, `resume`, `yes`, `yolo`, `auto_`, `no_opencode`, `telegram`, `scheduler`, `prompt`. No change needed. + +## 6. PARITY.md / CHANGELOG + +Symbols covered are listed in `docs/foundation.md` ("Symbol map"); the foundation area did not touch `PARITY.md` or `CHANGELOG.md`. diff --git a/taicpp/src/config/boot_settings.cpp b/taicpp/src/config/boot_settings.cpp new file mode 100644 index 0000000..3583c26 --- /dev/null +++ b/taicpp/src/config/boot_settings.cpp @@ -0,0 +1,37 @@ +#include "boot_settings.h" +#include +#include +#include +#include +#include "settings_rules.h" +#include "store/store.h" +namespace detz { + +std::vector BootSettings::ExportToEnv(store::Store& st, const std::vector& keys) { + std::map rows; + for (auto& r : st.get_or_create_settings(std::nullopt, SettingsRules::kScopeGlobal)) rows[r.key] = r; + std::vector exported; + for (auto& key : keys.empty() ? SettingExportKeys() : keys) { + const SettingSpec& spec = SettingsCatalog().at(key); + const char* cur = std::getenv(spec.env.c_str()); + if (cur && *cur) continue; + const auto& stored = rows.at(key).stored; + if (!stored) continue; + ::setenv(spec.env.c_str(), stored->c_str(), 1); + exported.push_back(key); + } + return exported; +} + +std::vector BootSettings::Boot(Config& config, store::Store& st) { + std::vector applied = config.ApplyStoreSettings(st); + std::vector keys = SettingExportKeys(); + for (auto& k : applied) if (std::find(keys.begin(), keys.end(), k) == keys.end()) keys.push_back(k); + if (config.noOpencodeCli) keys.erase(std::remove(keys.begin(), keys.end(), "opencode_enabled"), keys.end()); + std::vector exported = ExportToEnv(st, keys); + std::set all(applied.begin(), applied.end()); + all.insert(exported.begin(), exported.end()); + return std::vector(all.begin(), all.end()); +} + +} // namespace detz diff --git a/taicpp/src/config/boot_settings.h b/taicpp/src/config/boot_settings.h new file mode 100644 index 0000000..51c4e66 --- /dev/null +++ b/taicpp/src/config/boot_settings.h @@ -0,0 +1,19 @@ +#pragma once +#include +#include +#include "config.h" +namespace detz { + +// export_settings_to_env + boot_settings of tai.py (settings.md "Precedence"): +// vault settings fill the Config fields the environment left unset, then the +// peripheral keys are exported into the process environment so later readers +// see them. A non-empty environment variable always wins. +class BootSettings { + public: + // Fill unset env vars from vault settings; keys empty = SETTING_EXPORT_KEYS. Returns the exported key names. + static std::vector ExportToEnv(store::Store& store, const std::vector& keys = {}); + // Config fields first, then the env export. Returns the sorted names that took effect from the vault. + static std::vector Boot(Config& config, store::Store& store); +}; + +} // namespace detz diff --git a/taicpp/src/config/config.cpp b/taicpp/src/config/config.cpp index d70b2df..d474bf4 100644 --- a/taicpp/src/config/config.cpp +++ b/taicpp/src/config/config.cpp @@ -1,44 +1,208 @@ #include "config.h" #include "consts.h" +#include #include #include +#include +#include +#include "settings_rules.h" +#include "store/store.h" +#include "store/utf8.h" namespace detz { -static std::string Env(const char* k){ const char* v=std::getenv(k); return v?v:""; } +namespace { + +std::string Env(const char* k){ const char* v=std::getenv(k); return v?v:""; } +std::string Lower(std::string s){ for(auto&c:s) if(c>='A'&&c<='Z') c=static_cast(c+32); return s; } +std::string RStripSlash(std::string s){ while(!s.empty()&&s.back()=='/') s.pop_back(); return s; } +std::string HomeDir(){ + std::string h=Env("HOME"); + if(!h.empty()) return h; + if(struct passwd* pw=getpwuid(getuid())) return pw->pw_dir; + return "~"; +} +bool EnvFlag(const char* name, bool def_on){ + std::string raw=Env(name); + std::string v=Lower(SettingsRules::PyStrip(raw.empty()?(def_on?"1":""):raw)); + if(def_on) return !(v=="0"||v=="false"||v=="no"||v=="off"); + return v=="1"||v=="true"||v=="yes"||v=="on"; +} +SettingSpec Spec(const std::string& key, const std::string& env, const std::string& type, SettingValue def, const std::string& help, + bool required=false, bool nullable=false, bool encrypted=false) { + SettingSpec s; s.key=key; s.env=env; s.type=type; s.def=std::move(def); s.help=help; + s.required=required; s.nullable=nullable; s.encrypted=encrypted; return s; +} +} // namespace + std::vector SplitModels(const std::string& raw) { + std::string t=raw; + for(auto&c:t) if(c=='\n') c=','; std::vector out; std::string cur; - for(char c: raw+ ","){ if(c==','||c==';'){ size_t a=cur.find_first_not_of(" \t"); size_t b=cur.find_last_not_of(" \t"); if(b!=std::string::npos) out.push_back(cur.substr(a,b-a+1)); cur.clear();} else cur+=c; } + auto flush=[&](){ std::string s=store::Utf8::strip(cur); if(!s.empty()) out.push_back(s); cur.clear(); }; + for(char c: t){ if(c==',') flush(); else cur+=c; } + flush(); return out; } -const std::map& SettingsCatalog() { - static const std::map c = { - {"model",{"model","TAI_MODEL","string","muse","Default model id.",false,false}}, - {"openrouter_key",{"openrouter_key","OPENROUTER_API_KEY","string","","OpenRouter API key. Encrypted at rest.",true,true}}, - {"openrouter_models",{"openrouter_models","TAI_OPENROUTER_MODELS","list","","Extra OpenRouter models.",true,false}}, - {"devplace_key",{"devplace_key","DEVPLACE_API_KEY","string","","Devplace key. Encrypted at rest.",true,true}}, - {"devplace_base",{"devplace_base","TAI_DEVPLACE_BASE","string","https://devplace.nl","Devplace base.",false,false}}, - {"devplace_models",{"devplace_models","TAI_DEVPLACE_MODELS","list","","Extra devplace models.",true,false}}, - {"pollinations_models",{"pollinations_models","TAI_POLLINATIONS_MODELS","list","","Extra pollinations models.",true,false}}, - {"zen_models",{"zen_models","TAI_ZEN_MODELS","list","","Extra zen models.",true,false}}, - {"opencode_models",{"opencode_models","TAI_OPENCODE_MODELS","list","","Extra opencode models.",true,false}}, - {"voice",{"voice","TAI_VOICE","string","en-US-EmmaMultilingualNeural","TTS voice.",false,false}}, - {"retry_seconds",{"retry_seconds","TAI_RETRY_SECONDS","float","0","Retry delay.",false,false}}, + +std::vector ParseBackends(const std::string& spec) { + std::string t=spec; + for(auto&c:t) if(c=='\n') c=';'; + std::vector found; + std::string entry; + auto handle=[&](const std::string& e){ + std::vector fields; std::string cur; + for(char c: e){ if(c=='|'){ fields.push_back(store::Utf8::strip(cur)); cur.clear(); } else cur+=c; } + fields.push_back(store::Utf8::strip(cur)); + if(fields.size()<3) return; + BackendSpec b; b.label=fields[0]; b.base=RStripSlash(fields[1]); b.key=fields[2]; + if(!b.key.empty()&&b.key[0]=='$') b.key=Env(b.key.substr(1).c_str()); + b.model=fields.size()>3?store::Utf8::strip(fields[3]):""; + b.profile=fields.size()>4?Lower(store::Utf8::strip(fields[4])):""; + if(b.profile!=""&&b.profile!="opencode") b.profile=""; + if(b.label.empty()||b.base.empty()) return; + found.push_back(b); }; + for(char c: t){ if(c==';'){ handle(entry); entry.clear(); } else entry+=c; } + handle(entry); + return found; +} + +const std::map& SettingsCatalog() { + using V=SettingValue; + static const std::map c = [](){ + std::map m; + auto add=[&](SettingSpec s){ m[s.key]=std::move(s); }; + add(Spec("model","TAI_MODEL","string",V::String(kDefaultModel),"Default model id. A stored pin counts as explicit: per-directory model defaults do not override it.",true)); + add(Spec("backends","TAI_BACKENDS","string",V::Null(),"Extra OpenAI-compatible backends. Semicolon-separated label|base|key[|model][|profile] entries; a key starting with $ reads another env var. Holds secrets: encrypted at rest.",false,true,true)); + add(Spec("openrouter_key","OPENROUTER_API_KEY","string",V::Null(),"OpenRouter API key. Most free models work without one; some require it. Encrypted at rest.",false,true,true)); + add(Spec("openrouter_models","TAI_OPENROUTER_MODELS","string",V::Null(),"Comma-separated OpenRouter model ids to offer. Null means live discovery.",false,true)); + add(Spec("openrouter_enabled","TAI_OPENROUTER","bool",V::Bool(true),"Offer the OpenRouter backend.")); + add(Spec("opencode_enabled","TAI_OPENCODE","bool",V::Bool(true),"Offer the opencode backend. CLI --no-opencode always wins over this stored flag.")); + add(Spec("opencode_models","TAI_OPENCODE_MODELS","string",V::Null(),"Comma-separated opencode model ids to offer. Null means live discovery.",false,true)); + add(Spec("devplace_key","DEVPLACE_API_KEY","string",V::Null(),"DevPlace API key. Null disables the DevPlace fallback backend. Encrypted at rest.",false,true,true)); + add(Spec("devplace_base","TAI_DEVPLACE_BASE","string",V::String(kDevplaceBase),"DevPlace API base URL. Trailing slashes are stripped.")); + add(Spec("devplace_models","TAI_DEVPLACE_MODELS","string",V::Null(),"Comma-separated DevPlace model ids to offer. Null means live discovery.",false,true)); + add(Spec("pollinations_enabled","TAI_POLLINATIONS","bool",V::Bool(true),"Offer the Pollinations backend.")); + add(Spec("pollinations_models","TAI_POLLINATIONS_MODELS","string",V::Null(),"Comma-separated Pollinations model ids to offer. Null means live discovery.",false,true)); + add(Spec("zen_enabled","TAI_ZEN","bool",V::Bool(false),"Opt in to the opencode.ai zen backend.")); + add(Spec("zen_models","TAI_ZEN_MODELS","string",V::Null(),"Comma-separated zen model ids to offer. Null means live discovery.",false,true)); + add(Spec("retry_seconds","TAI_RETRY_SECONDS","float",V::Float(0.0),"Seconds to keep retrying all backends on total failure. 0 means forever. Plain decimal, no units.")); + add(Spec("voice","TAI_VOICE","string",V::String(kDefaultVoice),"Edge TTS voice name for the speak tool.")); + { SettingSpec s=Spec("theme","TAI_THEME","string",V::String("dark"),"TUI theme. Strict: must be one of the listed options. CLI --theme wins over env and vault."); + s.options={"dark","light","nord","gruvbox","ansi"}; s.hasOptions=true; add(s); } + { SettingSpec s=Spec("gemini_plan","TAI_GEMINI_PLAN","bool",V::Bool(false),"Use the Gemini CLI plan approval variant first."); + s.trueWords={"1","true","yes","on","plan"}; add(s); } + { SettingSpec s=Spec("codex_sandbox","TAI_CODEX_SANDBOX","bool",V::Bool(false),"Re-enable Codex CLI sandbox flags."); + s.trueWords={"1","true","yes","on","read-only","readonly"}; add(s); } + { SettingSpec s=Spec("box_caps","TAI_BOX_CAPS","bool",V::Bool(true),"Apply resource caps to pre-existing sandbox boxes. Escape hatch: false skips the live update. Env honors only 0 as false."); + s.falseWords={"0"}; add(s); } + add(Spec("muse_bin","MUSE_BIN","string",V::Null(),"Absolute muse CLI path override. Null means PATH lookup.",false,true)); + add(Spec("opencode_bin","OPENCODE_BIN","string",V::Null(),"Absolute opencode CLI path override. Null means PATH lookup.",false,true)); + add(Spec("grok_bin","GROK_BIN","string",V::Null(),"Absolute grok CLI path override (falls back to grokcli). Null means PATH lookup.",false,true)); + add(Spec("claude_bin","CLAUDE_BIN","string",V::Null(),"Absolute claude CLI path override. Null means PATH lookup.",false,true)); + add(Spec("codex_bin","CODEX_BIN","string",V::Null(),"Absolute codex CLI path override. Null means PATH lookup.",false,true)); + add(Spec("gemini_bin","GEMINI_BIN","string",V::Null(),"Absolute gemini CLI path override. Null means PATH lookup.",false,true)); + add(Spec("opencode_client_name","OPENCODE_CLIENT_NAME","string",V::Null(),"x-opencode-client header override for the zen backend. Null means the built-in.",false,true)); + add(Spec("opencode_client_ua","OPENCODE_CLIENT_USER_AGENT","string",V::Null(),"User-Agent header override for the zen backend. Null means the built-in.",false,true)); + return m; + }(); return c; } + +const std::vector& ConfigSettingFields() { + static const std::vector v = { + {"model","model","TAI_MODEL","str"}, + {"backends","backends","TAI_BACKENDS","backends"}, + {"openrouter_key","openrouter_key","OPENROUTER_API_KEY","str"}, + {"openrouter_models","openrouter_models","TAI_OPENROUTER_MODELS","models"}, + {"retry_seconds","retry_seconds","TAI_RETRY_SECONDS","float"}, + {"opencode_enabled","opencode_enabled","TAI_OPENCODE","bool"}, + {"opencode_models","opencode_models","TAI_OPENCODE_MODELS","models"}, + {"devplace_key","devplace_key","DEVPLACE_API_KEY","str"}, + {"devplace_base","devplace_base","TAI_DEVPLACE_BASE","rstrip"}, + {"devplace_models","devplace_models","TAI_DEVPLACE_MODELS","models"}, + {"openrouter_enabled","openrouter_enabled","TAI_OPENROUTER","bool"}, + {"pollinations_enabled","pollinations_enabled","TAI_POLLINATIONS","bool"}, + {"pollinations_models","pollinations_models","TAI_POLLINATIONS_MODELS","models"}, + {"zen_enabled","zen_enabled","TAI_ZEN","bool"}, + {"zen_models","zen_models","TAI_ZEN_MODELS","models"}, + {"voice","voice","TAI_VOICE","str"}, + }; + return v; +} + +const std::vector& SettingExportKeys() { + static const std::vector v = {"theme","muse_bin","opencode_bin","grok_bin","claude_bin","codex_bin","gemini_bin","gemini_plan","codex_sandbox","box_caps","opencode_client_name","opencode_client_ua"}; + return v; +} + Config Config::FromEnv(bool noOpencode) { Config c; c.noOpencodeCli=noOpencode; std::string home=Env("TAI_HOME"); - if(home.empty()){ const char* h=std::getenv("HOME"); home=std::string(h?h:"/tmp")+"/.tai"; } + if(home.empty()) home=HomeDir()+"/.tai"; c.home=home; + c.profilesDir=home+"/profiles"; c.audioDir=home+"/audio"; c.dbPath=home+"/memory.db"; c.historyPath=home+"/history"; std::string m=Env("TAI_MODEL"); c.model=m.empty()?kDefaultModel:m; c.explicitModel=m; + c.backends=ParseBackends(Env("TAI_BACKENDS")); c.openrouterKey=Env("OPENROUTER_API_KEY"); c.openrouterModels=SplitModels(Env("TAI_OPENROUTER_MODELS")); - c.devplaceKey=Env("DEVPLACE_API_KEY"); std::string b=Env("TAI_DEVPLACE_BASE"); c.devplaceBase=b.empty()?"https://devplace.nl":b; + { // float(env or 0) if re.match(r"^\d+(\.\d+)?$", (env or "0").strip()) else 0.0 + std::string raw=Env("TAI_RETRY_SECONDS"); + std::string t=SettingsRules::PyStrip(raw.empty()?"0":raw); + size_t i=0; while(i='0'&&t[i]<='9') i++; + bool ok=i>0; + if(ok&&i='0'&&t[j]<='9') j++; ok=j>i+1&&j==t.size(); } } + c.retrySeconds=ok?SettingsRules::PyFloat(raw.empty()?"0":raw):0.0; + } + c.opencodeEnabled=EnvFlag("TAI_OPENCODE",true); + if(noOpencode) c.opencodeEnabled=false; + c.opencodeModels=SplitModels(Env("TAI_OPENCODE_MODELS")); + c.devplaceKey=Env("DEVPLACE_API_KEY"); + std::string b=Env("TAI_DEVPLACE_BASE"); c.devplaceBase=RStripSlash(b.empty()?kDevplaceBase:b); c.devplaceModels=SplitModels(Env("TAI_DEVPLACE_MODELS")); + c.openrouterEnabled=EnvFlag("TAI_OPENROUTER",true); + c.pollinationsEnabled=EnvFlag("TAI_POLLINATIONS",true); c.pollinationsModels=SplitModels(Env("TAI_POLLINATIONS_MODELS")); - c.zenModels=SplitModels(Env("TAI_ZEN_MODELS")); c.opencodeModels=SplitModels(Env("TAI_OPENCODE_MODELS")); + c.zenEnabled=EnvFlag("TAI_ZEN",false); + c.zenModels=SplitModels(Env("TAI_ZEN_MODELS")); std::string v=Env("TAI_VOICE"); if(!v.empty()) c.voice=v; return c; } + +Config Config::Build(const std::string& profile, bool yes, bool noOpencode) { + Config c=FromEnv(noOpencode); + c.autoApprove=yes; + c.profile=profile.empty()?"default":profile; + std::filesystem::create_directories(c.profilesDir); + std::filesystem::create_directories(c.audioDir); + return c; +} + +std::vector Config::ApplyStoreSettings(store::Store& st) { + std::map rows; + for(auto& r: st.get_or_create_settings(std::nullopt, SettingsRules::kScopeGlobal)) rows[r.key]=r; + std::vector applied; + for(auto& f: ConfigSettingFields()){ + if(!Env(f.env.c_str()).empty()) continue; + if(f.key=="opencode_enabled"&&noOpencodeCli) continue; + const auto& stored=rows.at(f.key).stored; + if(!stored) continue; + const std::string& s=*stored; + if(f.kind=="bool"){ bool on=(s=="1"); if(f.attr=="opencode_enabled") opencodeEnabled=on; else if(f.attr=="openrouter_enabled") openrouterEnabled=on; else if(f.attr=="pollinations_enabled") pollinationsEnabled=on; else if(f.attr=="zen_enabled") zenEnabled=on; } + else if(f.kind=="float"){ try{ retrySeconds=SettingsRules::PyFloat(s); }catch(const store::ValueError&){ continue; } } + else if(f.kind=="models"){ + auto models=SplitModels(s); + if(f.attr=="openrouter_models") openrouterModels=models; else if(f.attr=="opencode_models") opencodeModels=models; + else if(f.attr=="devplace_models") devplaceModels=models; else if(f.attr=="pollinations_models") pollinationsModels=models; + else if(f.attr=="zen_models") zenModels=models; + } + else if(f.kind=="backends") backends=ParseBackends(s); + else if(f.kind=="rstrip") devplaceBase=RStripSlash(s); + else { if(f.attr=="model") model=s; else if(f.attr=="openrouter_key") openrouterKey=s; else if(f.attr=="devplace_key") devplaceKey=s; else if(f.attr=="voice") voice=s; } + if(f.key=="model") explicitModel=s; + applied.push_back(f.key); + } + return applied; +} + std::string Config::Get(const std::string& key) const { if (key == "model") return model; if (key == "devplace_base") return devplaceBase; @@ -59,6 +223,7 @@ std::vector KnownModelIds(const Config& cfg, const std::vector #include #include +namespace detz::store { class Store; } namespace detz { -struct SettingSpec { std::string key; std::string env; std::string type; std::string def; std::string help; bool nullable = false; bool encrypted = false; }; + +// A python-typed setting value: None, bool, float or str. +struct SettingValue { + enum class Kind { Null, Bool, Float, String }; + Kind kind = Kind::Null; + bool b = false; + double d = 0.0; + std::string s; + static SettingValue Null() { return {}; } + static SettingValue Bool(bool v) { SettingValue x; x.kind = Kind::Bool; x.b = v; return x; } + static SettingValue Float(double v) { SettingValue x; x.kind = Kind::Float; x.d = v; return x; } + static SettingValue String(std::string v) { SettingValue x; x.kind = Kind::String; x.s = std::move(v); return x; } + bool IsNull() const { return kind == Kind::Null; } + bool operator==(const SettingValue& o) const { return kind == o.kind && b == o.b && d == o.d && s == o.s; } +}; + +// One SETTINGS_CATALOG entry (tai.py setting_spec). +struct SettingSpec { + std::string key, env, type; // type: string | bool | float + SettingValue def; // catalog default + std::string help; // desc + bool nullable = false; + bool encrypted = false; + bool required = false; + std::vector options; // strict list, empty = none + bool hasOptions = false; + std::vector trueWords; // env word sets overriding the defaults, empty = default + std::vector falseWords; +}; const std::map& SettingsCatalog(); + +// CONFIG_SETTING_FIELDS: (attr, key, env, kind) routed through Config at boot. +struct ConfigSettingField { std::string attr, key, env, kind; }; +const std::vector& ConfigSettingFields(); +// SETTING_EXPORT_KEYS: peripheral keys exported into the environment at boot. +const std::vector& SettingExportKeys(); + +// One entry of TAI_BACKENDS (tai.py parse_backends). +struct BackendSpec { std::string label, base, key, model, profile; }; +std::vector ParseBackends(const std::string& spec); + struct Config { std::string home; std::string model; std::string explicitModel; + std::string profilesDir, audioDir, dbPath, historyPath; + std::string profile = "default"; + std::vector backends; std::string openrouterKey; std::vector openrouterModels; std::string devplaceKey; std::string devplaceBase; std::vector devplaceModels; std::vector pollinationsModels; std::vector zenModels; std::vector opencodeModels; std::string voice = "en-US-EmmaMultilingualNeural"; + double retrySeconds = 0.0; + bool opencodeEnabled = true, openrouterEnabled = true, pollinationsEnabled = true, zenEnabled = false; bool noOpencodeCli = false; + bool autoApprove = false; + // Environment only (no directories created): the pre-existing entry point. static Config FromEnv(bool noOpencode = false); + // tai.py Config(args): environment plus profile/--yes/--no-opencode, creates profiles/ and audio/. + static Config Build(const std::string& profile, bool yes, bool noOpencode); std::string Get(const std::string& key) const; + // Second boot pass: fill fields the environment left unset from the vault settings. Returns applied keys. + std::vector ApplyStoreSettings(store::Store& store); }; std::vector SplitModels(const std::string& raw); std::vector KnownModelIds(const Config& cfg, const std::vector& extra = {}); diff --git a/taicpp/src/config/consts.h b/taicpp/src/config/consts.h index 53b3739..053facd 100644 --- a/taicpp/src/config/consts.h +++ b/taicpp/src/config/consts.h @@ -24,7 +24,7 @@ struct Consts { inline constexpr const char* kDefaultModel = "muse"; inline constexpr const char* kDefaultVoice = "en-US-EmmaMultilingualNeural"; inline constexpr const char* kOpenRouterModel = "deepseek/deepseek-v4.1-flash"; -inline constexpr const char* kDevplaceBase = "https://devplace.nl"; +inline constexpr const char* kDevplaceBase = "https://devplace.net/openai/v1"; inline const std::vector& ZenFreeModels() { static const std::vector v = {"mimo-v2.6-flash-free","nemotron-3.5-lightning-free","muse-spark-1.3-contributor-free","muse-spark-1.2-contributor-free","jev-1.13-free","step-5-preview-free","space-bunny-free","longcat-2.5-preview-free","ling-3.1-flash-free"}; diff --git a/taicpp/src/config/prompts.cpp b/taicpp/src/config/prompts.cpp new file mode 100644 index 0000000..22aa469 --- /dev/null +++ b/taicpp/src/config/prompts.cpp @@ -0,0 +1,100 @@ +// Generated verbatim from tai.py (DEFAULT_SYSTEM, MERGE_SYSTEM, COMPACT_SYSTEM, AUTONOMOUS_NOTE, +// HELP_TEXT, LEGACY_PASSWORD_NOTE, VAULT_MEMORY_NOTE). Parity is asserted by tests/foundation_test.cpp. +#include "prompts.h" +#include +#include "session/shell_context.h" +namespace detz { + +const std::string& Prompts::DefaultSystem() { + static const std::string v = R"TAI(You are tai, a professional autonomous assistant. You act with tools, verify results, and report concisely. Rules: prefer the smallest change that solves the task; never silently ignore errors; call ask_user when requirements are ambiguous (never end your turn with open questions in prose: one-shot runs exit and prose questions are never answered); keep answers short and factual. Tools: shell and file tools for the local machine, web_search and web_fetch for the internet, web_fetch handles every HTTP method with headers and bodies so never curl or wget through shell, speak for voice output, recall to search past session memory. Memory: call remember whenever you learn durable facts, preferences, or standing behavior changes. Store passwords, tokens, and secrets with store_secret and reference them by name; never write secret values into memory. Name and value suffice for secrets; never pester the user for more. Call remember with a forget instruction to drop outdated knowledge. Delegation: do quick, interactive, or memory-changing work yourself; fork background subagents for independent, long, or context-heavy work and collect them with poll; schedule future work instead of waiting on it. Records: save big or durable findings with record_save and tag generously; large tool outputs spill to mem: ids automatically, page them with record_read, connect related nodes with graph_link. Tags: strict rules, lowercase singular only, shortest common word for the subject, one canonical tag per subject, at most 5 per item, always reuse existing tags from the tags tool instead of inventing synonyms. Content words matching known tags attach automatically, and new records link to recent same-tag records. Lazy tools: only everyday tools load by default; name any other tool or topic in your reply text and it becomes callable on your next step. Bots: one identity holds many bots; a bot is only a system message plus its own history, all vault data is shared. @name routes one turn to another bot and files the exchange in both histories; /bot switches, create_bot makes bots. Terminal: the user's question is very likely about what is visible in their terminal, especially right after a terminal capture or when launched with a single question straight from a shell; resolve vague references (this, that, it, the error) against that content, capture it with get_current_terminal_content instead of asking, and answer from what you see. Retrieval: use search first for anything stored: one ranked full-text query across records, events, and audit plus graph expansion beats paging and guessing. Chain tool calls until the task is done. Never narrate a planned action in text without emitting its tool call: text without tool calls ends your turn and the action will not happen.)TAI"; + return v; +} + +const std::string& Prompts::MergeSystem() { + static const std::string v = R"TAI(You maintain an AI assistant system prompt. You receive the current system prompt and one memory instruction. Rewrite the system prompt to incorporate the instruction: add new facts, update changed behavior, or remove forgotten items. Never persist passwords, tokens, or other secret values; if the instruction holds one, note only that a secret exists and must be stored with store_secret. Preserve everything unrelated. Keep it organized with short sections. Output ONLY the rewritten system prompt, no explanation, no code fences.)TAI"; + return v; +} + +const std::string& Prompts::CompactSystem() { + static const std::string v = R"TAI(You compress assistant session history into a dense handoff summary. Cover: goal, completed work with outcomes, current state, key decisions, files touched, errors and fixes, pending next steps. Output only the summary, no preamble.)TAI"; + return v; +} + +const std::string& Prompts::AutonomousNote() { + static const std::string v = R"TAI(Autonomous mode: never ask the user anything this session (the ask_user tool refuses, do not call it). When requirements are unclear, blocked, or need input, research deeply with web_search, web_fetch, shell, and subagents, then decide and proceed. State assumptions briefly in your reply.)TAI"; + return v; +} + +const std::string& Prompts::HelpText() { + static const std::string v = R"TAI(commands: + /profile [name] show current profile or switch to it + /profiles list all profiles, current marked with * + /bots list bots, current marked with * + /bot switch bot, history resumes + /env [home|sandbox] show or switch execution environment + /ctx show shell session tree and recent sessions + /skills list loaded skills with provenance + /skills install ... install an existing skill (copy or symlink) + /secret manage sealed secrets (set|list|delete) + /sysinfo show host environment checks + /models show model roster with speed health + /schedule at|every schedule a prompt for later + /schedules list scheduled prompts + /unschedule delete a scheduled prompt + /records [query] search saved records + /record read one record page + /graph show node neighborhood + /tags [prefix] list tags with usage counts + /tools [name] show core and lazy tools + /install ... install status, install, upgrade, reinstall + /search ranked search over everything + /audit [path] show file audit trail + /restore restore a file from an audit row + /release part msg bump version, back up, log message + /fork spawn a background subagent, REPL stays free + /agents list background subagents + /agent show one result, or purge finished agents + /compact compress history into a summary + /clear drop history, keep system message + /demo fullscreen TUI framework showcase + /help this text + /quit exit + @bot @provider @model route this turn and save dir defaults (@auto clears pins, @path references a file) +anything else is sent to the agent.)TAI"; + return v; +} + +const std::string& Prompts::LegacyPasswordNote() { + static const std::string v = R"TAI(especially passwords (collect and keep every password the user shares), preferences, or standing behavior changes)TAI"; + return v; +} + +const std::string& Prompts::VaultMemoryNote() { + static const std::string v = R"TAI(preferences, or standing behavior changes. Store passwords, tokens, and secrets with store_secret and reference them by name; never write secret values into memory)TAI"; + return v; +} + +std::string Prompts::OrientationBlock(const std::string& directory, bool sandbox) { + if (sandbox) { + return "\n\n## Working directory\nSandboxed container: shell and file tools execute " + "inside the container, not on the host. Stay inside the mounted workdir " + "unless the user names another path."; + } + return "\n\n## Working directory\n" + session::ShellContext::dir_key(directory) + + "\nYour shell and file tools execute here. Stay inside " + "it unless the user names another path. If your harness reports a different " + "working directory (a scratch or temp dir), ignore it: it is not the workspace."; +} + +std::string Prompts::CapabilityBlock(bool yolo, bool autonomous) { + if (yolo || autonomous) { + return "\n\n## Session capabilities\nApprovals are off and every tool is callable, " + "including shell and file writes. Only a failed " + "tool call proves a limit, so act first and adjust on evidence."; + } + return "\n\n## Session capabilities\nEvery tool is callable, including shell and file writes; " + "some commands ask for approval first. Only a failed " + "tool call proves a limit, so act first and adjust on evidence."; +} + +} // namespace detz diff --git a/taicpp/src/config/prompts.h b/taicpp/src/config/prompts.h new file mode 100644 index 0000000..3c046ce --- /dev/null +++ b/taicpp/src/config/prompts.h @@ -0,0 +1,22 @@ +#pragma once +#include +namespace detz { + +// The fixed prompt and help texts of tai.py, byte for byte. +class Prompts { + public: + static const std::string& DefaultSystem(); // DEFAULT_SYSTEM + static const std::string& MergeSystem(); // MERGE_SYSTEM + static const std::string& CompactSystem(); // COMPACT_SYSTEM + static const std::string& AutonomousNote(); // AUTONOMOUS_NOTE + static const std::string& HelpText(); // HELP_TEXT + static const std::string& LegacyPasswordNote(); // LEGACY_PASSWORD_NOTE + static const std::string& VaultMemoryNote(); // VAULT_MEMORY_NOTE + + // orientation_block(directory, sandbox): working-directory block of the system prompt + static std::string OrientationBlock(const std::string& directory = "", bool sandbox = false); + // capability_block(yolo, auto): live approval state of the system prompt + static std::string CapabilityBlock(bool yolo = false, bool autonomous = false); +}; + +} // namespace detz diff --git a/taicpp/src/config/settings_rules.cpp b/taicpp/src/config/settings_rules.cpp new file mode 100644 index 0000000..65530a0 --- /dev/null +++ b/taicpp/src/config/settings_rules.cpp @@ -0,0 +1,148 @@ +#include "settings_rules.h" +#include +#include +#include +#include +#include "store/errors.h" +#include "store/patterns.h" +#include "store/utf8.h" +namespace detz { + +using store::ValueError; + +const std::vector& SettingsRules::TrueWords() { static const std::vector v = {"1", "true", "yes", "on"}; return v; } +const std::vector& SettingsRules::FalseWords() { static const std::vector v = {"0", "false", "no", "off"}; return v; } + +std::string SettingsRules::PyLower(const std::string& s) { + std::string o = s; + for (auto& c : o) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + return o; +} +std::string SettingsRules::PyStrip(const std::string& s) { return store::Utf8::strip(s); } + +const std::string& SettingsRules::ValidateScope(const std::string& scope) { + if (scope != kScopeGlobal && !store::Patterns::profile(scope)) throw ValueError("invalid settings scope '" + scope + "'"); + return scope; +} + +double SettingsRules::PyFloat(const std::string& text) { + std::string t = PyStrip(text); + auto fail = [&]() { throw ValueError("could not convert string to float: '" + text + "'"); }; + if (t.empty()) fail(); + size_t p = 0; + bool neg = false; + if (t[p] == '+' || t[p] == '-') { neg = t[p] == '-'; p++; } + std::string rest = PyLower(t.substr(p)); + if (rest == "inf" || rest == "infinity") return neg ? -HUGE_VAL : HUGE_VAL; + if (rest == "nan") return std::nan(""); + std::string clean; + auto digits = [&](size_t& i) { + bool any = false, last_us = false; + while (i < t.size()) { + char c = t[i]; + if (c >= '0' && c <= '9') { clean += c; any = true; last_us = false; i++; } + else if (c == '_' && any && !last_us && i + 1 < t.size() && t[i + 1] >= '0' && t[i + 1] <= '9') { last_us = true; i++; } + else break; + } + return any; + }; + size_t i = p; + bool intpart = digits(i); + bool frac = false; + if (i < t.size() && t[i] == '.') { + clean += '.'; + i++; + frac = digits(i); + } + if (!intpart && !frac) fail(); + if (i < t.size() && (t[i] == 'e' || t[i] == 'E')) { + clean += 'e'; + i++; + if (i < t.size() && (t[i] == '+' || t[i] == '-')) { clean += t[i]; i++; } + if (!digits(i)) fail(); + } + if (i != t.size()) fail(); + double v = std::strtod(clean.c_str(), nullptr); + return neg ? -v : v; +} + +std::optional SettingsRules::Normalize(const std::string& key, const SettingSpec& spec, const store::Json& value) { + using store::Json; + if (value.is_null()) { + if (spec.nullable) return std::nullopt; + throw ValueError("setting '" + key + "' is not nullable"); + } + const std::string& kind = spec.type; + if (kind == "bool") { + // value is True, or value == 1 for a non-str (1, 1.0) + if (value.is_bool() && value.as_bool()) return std::string("1"); + if (value.is_number() && value.as_double() == 1.0) return std::string("1"); + if (value.is_bool() && !value.as_bool()) return std::string("0"); + if (value.is_number() && value.as_double() == 0.0) return std::string("0"); + std::string text = value.is_string() ? PyLower(PyStrip(value.as_string())) : ""; + if (std::find(TrueWords().begin(), TrueWords().end(), text) != TrueWords().end()) return std::string("1"); + if (std::find(FalseWords().begin(), FalseWords().end(), text) != FalseWords().end()) return std::string("0"); + throw ValueError("setting '" + key + "' wants a bool (1/0, true/false, yes/no, on/off)"); + } + if (kind == "float") { + if (value.is_bool()) throw ValueError("setting '" + key + "' wants a float, not a bool"); + double number; + if (value.is_number()) number = value.as_double(); + else if (value.is_string()) { + try { number = PyFloat(value.as_string()); } catch (const ValueError&) { throw ValueError("setting '" + key + "' wants a float"); } + } else throw ValueError("setting '" + key + "' wants a float"); + if (!std::isfinite(number) || number < 0) throw ValueError("setting '" + key + "' wants a finite float >= 0"); + return Json::float_repr(number); + } + if (!value.is_string()) throw ValueError("setting '" + key + "' wants a string"); + const std::string& text = value.as_string(); + if (spec.required && text.empty()) throw ValueError("setting '" + key + "' is required and cannot be empty"); + if (spec.hasOptions && std::find(spec.options.begin(), spec.options.end(), text) == spec.options.end()) { + std::string joined; + for (size_t k = 0; k < spec.options.size(); k++) { if (k) joined += ", "; joined += spec.options[k]; } + throw ValueError("setting '" + key + "' must be one of: " + joined); + } + return text; +} + +static bool in_words(const std::vector& w, const std::string& t) { return std::find(w.begin(), w.end(), t) != w.end(); } + +SettingValue SettingsRules::ParseEnv(const SettingSpec& spec, const std::string& raw) { + if (spec.type == "bool") { + std::string text = PyLower(PyStrip(raw)); + if (in_words(spec.trueWords.empty() ? TrueWords() : spec.trueWords, text)) return SettingValue::Bool(true); + if (in_words(spec.falseWords.empty() ? FalseWords() : spec.falseWords, text)) return SettingValue::Bool(false); + return spec.def; + } + if (spec.type == "float") { + std::string t = PyStrip(raw); + // ^\d+(\.\d+)?$ (python "$" allows one trailing newline, already stripped here) + size_t i = 0; + while (i < t.size() && t[i] >= '0' && t[i] <= '9') i++; + bool ok = i > 0; + if (ok && i < t.size()) { + if (t[i] != '.') ok = false; + else { + size_t j = i + 1; + while (j < t.size() && t[j] >= '0' && t[j] <= '9') j++; + ok = j > i + 1 && j == t.size(); + } + } + if (ok) return SettingValue::Float(PyFloat(t)); + return spec.def; + } + if (spec.hasOptions && std::find(spec.options.begin(), spec.options.end(), raw) == spec.options.end()) { + std::string joined; + for (size_t k = 0; k < spec.options.size(); k++) { if (k) joined += ", "; joined += spec.options[k]; } + throw ValueError("setting '" + spec.key + "' must be one of: " + joined + " (from " + spec.env + ")"); + } + return SettingValue::String(raw); +} + +SettingValue SettingsRules::ParseStored(const SettingSpec& spec, const std::string& stored) { + if (spec.type == "bool") return SettingValue::Bool(stored == "1"); + if (spec.type == "float") return SettingValue::Float(PyFloat(stored)); + return SettingValue::String(stored); +} + +} // namespace detz diff --git a/taicpp/src/config/settings_rules.h b/taicpp/src/config/settings_rules.h new file mode 100644 index 0000000..5077cdc --- /dev/null +++ b/taicpp/src/config/settings_rules.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include +#include "config.h" +#include "store/json.h" +namespace detz { + +// The validation and parsing half of settings.md (tai.py validate_setting_scope, +// normalize_setting_value, parse_setting_env, parse_setting_stored). All +// failures throw store::ValueError, as python raises ValueError. +class SettingsRules { + public: + static constexpr const char* kScopeGlobal = "global"; + static const std::vector& TrueWords(); + static const std::vector& FalseWords(); + + // returns scope or throws "invalid settings scope" + static const std::string& ValidateScope(const std::string& scope); + // canonical stored string, or nullopt for a legal null; throws on invalid input + static std::optional Normalize(const std::string& key, const SettingSpec& spec, const store::Json& value); + static SettingValue ParseEnv(const SettingSpec& spec, const std::string& raw); + static SettingValue ParseStored(const SettingSpec& spec, const std::string& stored); + // python float(text): throws ValueError when text is not a float literal + static double PyFloat(const std::string& text); + static std::string PyLower(const std::string& s); + static std::string PyStrip(const std::string& s); +}; + +} // namespace detz diff --git a/taicpp/src/plexer/flow_validator.cpp b/taicpp/src/plexer/flow_validator.cpp new file mode 100644 index 0000000..58bd7f6 --- /dev/null +++ b/taicpp/src/plexer/flow_validator.cpp @@ -0,0 +1,269 @@ +#include "flow_validator.h" +#include +#include +#include "plexer_consts.h" +#include "store/utf8.h" +namespace detz::plexer { + +using store::Json; +using store::Utf8; + +namespace { + +bool truthy(const Json* v) { + if (!v || v->is_null()) return false; + if (v->is_bool()) return v->as_bool(); + if (v->is_int()) return v->as_int() != 0; + if (v->is_float()) return v->as_double() != 0; + if (v->is_string()) return !v->as_string().empty(); + return v->size() > 0; +} + +std::string repr_str(const std::string& s) { return Json(s).py_repr(); } + +// python int(value) for the (TypeError, ValueError) handlers; nullopt = it raised +std::optional py_int(const Json& v) { + if (v.is_bool()) return v.as_bool() ? 1 : 0; + if (v.is_int()) return static_cast(v.as_int()); + if (v.is_float()) { + double d = v.as_double(); + if (std::isnan(d) || std::isinf(d)) return std::nullopt; + return static_cast(d); + } + if (v.is_string()) { + std::string t = Utf8::strip(v.as_string()); + if (t.empty()) return std::nullopt; + size_t i = 0; + if (t[0] == '+' || t[0] == '-') i = 1; + if (i >= t.size()) return std::nullopt; + std::string digits; + bool last_us = true; + for (size_t k = i; k < t.size(); k++) { + char c = t[k]; + if (c >= '0' && c <= '9') { digits += c; last_us = false; } + else if (c == '_' && !last_us && k + 1 < t.size()) last_us = true; + else return std::nullopt; + } + if (last_us) return std::nullopt; + long long n = std::stoll(digits); + return t[0] == '-' ? -n : n; + } + return std::nullopt; +} + +bool scalar_or_none(const Json& v) { return v.is_string() || v.is_int() || v.is_float() || v.is_bool() || v.is_null(); } + +std::string join_types() { + std::string out; + for (auto& t : Plexer::FlowTypes()) { if (!out.empty()) out += "/"; out += t; } + return out; +} + +} // namespace + +std::pair FlowValidator::id(const Json& raw, const std::string& what) { + std::string text = truthy(&raw) ? Utf8::strip(raw.py_str()) : ""; + if (text.empty()) return {"", what + " id must be a non-empty string"}; + if (static_cast(Utf8::length(text)) > Plexer::kFlowIdMax) return {"", what + " id " + repr_str(Utf8::prefix(text, 72)) + " is longer than " + std::to_string(Plexer::kFlowIdMax) + " chars"}; + if (Utf8::has_space(text)) return {"", what + " id " + repr_str(text) + " must not contain whitespace"}; + return {text, ""}; +} + +std::pair FlowValidator::text(const Json& raw, const std::string& what, int limit) { + if (!raw.is_string()) return {"", what + " must be a string"}; + if (static_cast(Utf8::length(raw.as_string())) > limit) return {"", what + " is longer than " + std::to_string(limit) + " chars"}; + return {raw.as_string(), ""}; +} + +FlowValidator::Result FlowValidator::options(const Json& raw, const std::string& what) { + if (!raw.is_array() || raw.size() == 0) return {Json::array(), what + " needs 1 to " + std::to_string(Plexer::kFlowOptionsMax) + " options"}; + if (static_cast(raw.size()) > Plexer::kFlowOptionsMax) return {Json::array(), what + " has " + std::to_string(raw.size()) + " options, at most " + std::to_string(Plexer::kFlowOptionsMax)}; + Json out = Json::array(); + int pos = 0; + for (auto& item : raw.items()) { + Json label, value; + if (item.is_string()) { label = item; value = item; } + else if (item.is_object()) { + const Json* l = item.find("label"); + label = l ? *l : Json(); + const Json* v = item.find("value"); + value = v ? *v : label; + } else { + return {Json::array(), what + " option " + std::to_string(pos + 1) + " must be a string or {label, value}"}; + } + if (!label.is_string() || Utf8::strip(label.as_string()).empty()) return {Json::array(), what + " option " + std::to_string(pos + 1) + " needs a non-empty label"}; + if (static_cast(Utf8::length(label.as_string())) > Plexer::kFlowLabelMax) return {Json::array(), what + " option " + std::to_string(pos + 1) + " label is longer than " + std::to_string(Plexer::kFlowLabelMax) + " chars"}; + if (!scalar_or_none(value)) return {Json::array(), what + " option " + std::to_string(pos + 1) + " value must be a JSON scalar"}; + Json o = Json::object(); + o.set("label", label); + o.set("value", value); + out.push(o); + pos++; + } + return {out, ""}; +} + +FlowValidator::Result FlowValidator::condition(const Json& raw, const std::string& what, const std::set& known) { + if (raw.is_null()) return {Json::object(), ""}; + if (!raw.is_object() || raw.size() == 0) return {Json::object(), what + " show_if must be a non-empty {field id: expected} object"}; + Json clauses = Json::object(); + for (auto& m : raw.members()) { + if (!known.count(m.first)) return {Json::object(), what + " show_if refers to unknown field " + repr_str(m.first)}; + Json accepted = m.second.is_array() ? m.second : Json::array(); + if (!m.second.is_array()) accepted.push(m.second); + if (accepted.size() == 0) return {Json::object(), what + " show_if for " + repr_str(m.first) + " needs at least one expected value"}; + for (auto& v : accepted.items()) { + if (!scalar_or_none(v)) return {Json::object(), what + " show_if for " + repr_str(m.first) + " must hold JSON scalars"}; + } + clauses.set(m.first, accepted); + } + return {clauses, ""}; +} + +FlowValidator::Result FlowValidator::spec(const Json& sp) { + auto fail = [](const std::string& why) { return Result{Json(), why}; }; + if (!sp.is_object()) return fail("spec must be a JSON object"); + auto get = [&](const Json& obj, const char* key, const Json& dflt) { const Json* v = obj.find(key); return v ? *v : dflt; }; + Json version = get(sp, "v", Json(Plexer::kFlowVersion)); + if (!(version == Json(Plexer::kFlowVersion))) return fail("spec v must be " + std::to_string(Plexer::kFlowVersion) + ", got " + version.py_repr()); + auto [title, terr] = text(get(sp, "title", Json("")), "title", Plexer::kFlowTitleMax); + if (!terr.empty()) return fail(terr); + auto to = py_int(get(sp, "timeout", Json(Plexer::kFlowTimeoutDefault))); + if (!to) return fail("timeout must be an integer number of seconds"); + long long timeout = *to; + if (!(Plexer::kFlowTimeoutMin <= timeout && timeout <= Plexer::kFlowTimeoutMax)) { + return fail("timeout must be " + std::to_string(Plexer::kFlowTimeoutMin) + " to " + std::to_string(Plexer::kFlowTimeoutMax) + " seconds"); + } + const Json* raw_screens = sp.find("screens"); + if (!raw_screens || !raw_screens->is_array() || raw_screens->size() == 0) return fail("screens must be a non-empty list"); + if (static_cast(raw_screens->size()) > Plexer::kFlowScreensMax) return fail(std::to_string(raw_screens->size()) + " screens, at most " + std::to_string(Plexer::kFlowScreensMax)); + + Json screens = Json::array(); + std::set screen_ids, field_ids; + int pos = 0; + for (auto& item : raw_screens->items()) { + std::string what = "screen " + std::to_string(pos + 1); + pos++; + if (!item.is_object()) return fail(what + " must be an object"); + auto [ident, e1] = id(get(item, "id", Json()), what); + if (!e1.empty()) return fail(e1); + if (screen_ids.count(ident)) return fail("duplicate screen id " + repr_str(ident)); + screen_ids.insert(ident); + auto [title_s, e2] = text(get(item, "title", Json("")), ident + " title", Plexer::kFlowTitleMax); + if (!e2.empty()) return fail(e2); + auto [body, e3] = text(get(item, "body", Json("")), ident + " body", Plexer::kFlowTextMax); + if (!e3.empty()) return fail(e3); + Json raw_fields = get(item, "fields", Json::array()); + if (!raw_fields.is_array()) return fail(ident + " fields must be a list"); + if (static_cast(raw_fields.size()) > Plexer::kFlowFieldsMax) return fail(ident + " has " + std::to_string(raw_fields.size()) + " fields, at most " + std::to_string(Plexer::kFlowFieldsMax)); + Json fields = Json::array(); + int fpos = 0; + for (auto& field : raw_fields.items()) { + std::string fwhat = ident + " field " + std::to_string(fpos + 1); + fpos++; + if (!field.is_object()) return fail(fwhat + " must be an object"); + Json kind = get(field, "type", Json()); + bool known_kind = false; + for (auto& t : Plexer::FlowTypes()) if (kind.is_string() && kind.as_string() == t) known_kind = true; + if (!known_kind) return fail(fwhat + " type must be one of " + join_types() + ", got " + kind.py_repr()); + auto [fid, e4] = id(get(field, "id", Json()), fwhat); + if (!e4.empty()) return fail(e4); + if (field_ids.count(fid)) return fail("duplicate field id " + repr_str(fid)); + field_ids.insert(fid); + auto [label, e5] = text(get(field, "label", Json("")), fid + " label", Plexer::kFlowLabelMax); + if (!e5.empty()) return fail(e5); + Json entry = Json::object(); + entry.set("id", fid); entry.set("type", kind); entry.set("label", label); entry.set("show_if", get(field, "show_if", Json())); + const std::string& k = kind.as_string(); + if (k == "choice") { + auto [opts, e6] = options(get(field, "options", Json()), fid); + if (!e6.empty()) return fail(e6); + entry.set("options", opts); + auto def = py_int(get(field, "default", Json(0))); + if (!def) return fail(fid + " default must be an option index"); + if (!(0 <= *def && *def < static_cast(opts.size()))) return fail(fid + " default " + std::to_string(*def) + " is outside its " + std::to_string(opts.size()) + " options"); + entry.set("default", static_cast(*def)); + } else if (k == "text") { + auto [initial, e7] = text(get(field, "initial", Json("")), fid + " initial", Plexer::kFlowAnswerMax); + if (!e7.empty()) return fail(e7); + entry.set("initial", initial); + } else { + auto [txt, e8] = text(get(field, "text", Json("")), fid + " text", Plexer::kFlowTextMax); + if (!e8.empty()) return fail(e8); + entry.set("text", txt); + } + fields.push(entry); + } + Json raw_actions = get(item, "actions", Json::array()); + if (!raw_actions.is_array()) return fail(ident + " actions must be a list"); + if (static_cast(raw_actions.size()) > Plexer::kFlowActionsMax) return fail(ident + " has " + std::to_string(raw_actions.size()) + " actions, at most " + std::to_string(Plexer::kFlowActionsMax)); + Json actions = Json::array(); + std::set action_ids; + int apos = 0; + for (auto& action : raw_actions.items()) { + std::string awhat = ident + " action " + std::to_string(apos + 1); + apos++; + if (!action.is_object()) return fail(awhat + " must be an object"); + auto [aid, e9] = id(get(action, "id", Json()), awhat); + if (!e9.empty()) return fail(e9); + if (action_ids.count(aid)) return fail(ident + " has a duplicate action id " + repr_str(aid)); + action_ids.insert(aid); + auto [alabel, e10] = text(get(action, "label", Json("")), aid + " label", Plexer::kFlowLabelMax); + if (!e10.empty()) return fail(e10); + if (Utf8::strip(alabel).empty()) return fail(aid + " needs a non-empty label"); + Json a = Json::object(); + a.set("id", aid); a.set("label", alabel); + actions.push(a); + } + if (Utf8::strip(body).empty() && fields.size() == 0 && actions.size() == 0) return fail(ident + " has nothing to show (no body, fields or actions)"); + Json screen = Json::object(); + screen.set("id", ident); screen.set("title", title_s); screen.set("body", body); screen.set("fields", fields); screen.set("actions", actions); + screen.set("next", get(item, "next", Json())); screen.set("show_if", get(item, "show_if", Json())); + screens.push(screen); + } + std::set known_fields; + for (auto& screen : screens.items()) { + for (auto& field : screen.find("fields")->items()) { + std::string t = field.get_string("type"); + if (t == "choice" || t == "text") known_fields.insert(field.get_string("id")); + } + } + for (auto& screen : screens.items()) { + for (auto& field : const_cast(screen.find("fields"))->items()) { + auto [clauses, err] = condition(get(field, "show_if", Json()), field.get_string("id"), known_fields); + if (!err.empty()) return fail(err); + field.set("show_if", clauses); + } + auto [clauses, err] = condition(get(screen, "show_if", Json()), screen.get_string("id"), known_fields); + if (!err.empty()) return fail(err); + screen.set("show_if", clauses); + Json target = get(screen, "next", Json()); + std::string sid = screen.get_string("id"); + if (target.is_null()) { + screen.set("next", Json()); + } else if (target.is_string()) { + const std::string& t = target.as_string(); + if (t != "$done" && t != "$cancel" && !screen_ids.count(t)) return fail(sid + " next refers to unknown screen " + repr_str(t)); + screen.set("next", target); + } else if (target.is_object()) { + if (screen.find("actions")->size() == 0) return fail(sid + " next is a map but the screen has no actions"); + for (auto& m : target.members()) { + bool known_action = false; + for (auto& a : screen.find("actions")->items()) if (a.get_string("id") == m.first) known_action = true; + if (!known_action) return fail(sid + " next maps unknown action " + repr_str(m.first)); + bool ok_dest = m.second.is_string() && (m.second.as_string() == "$done" || m.second.as_string() == "$cancel" || screen_ids.count(m.second.as_string())); + if (!ok_dest) return fail(sid + " next for " + repr_str(m.first) + " refers to unknown screen " + m.second.py_repr()); + } + screen.set("next", target); + } else { + return fail(sid + " next must be a screen id or an {action: screen} map"); + } + } + Json start = get(sp, "start", Json(screens.items()[0].get_string("id"))); + if (!start.is_string() || !screen_ids.count(start.as_string())) return fail("start refers to unknown screen " + start.py_repr()); + Json app = Json::object(); + app.set("title", title); app.set("timeout", static_cast(timeout)); app.set("start", start); app.set("screens", screens); + return {app, ""}; +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/flow_validator.h b/taicpp/src/plexer/flow_validator.h new file mode 100644 index 0000000..bb1e920 --- /dev/null +++ b/taicpp/src/plexer/flow_validator.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include +#include "store/json.h" +namespace detz::plexer { + +// The plexer_show JSON contract (uiflow.md, version PLEXER_FLOW_VERSION): +// tai.py validate_flow_id, validate_flow_text, validate_flow_options, +// validate_flow_condition, validate_flow_spec. Every rejection is loud: the +// second member of each result is the reason, "" on success. +class FlowValidator { + public: + using Json = store::Json; + using Result = std::pair; + static std::pair id(const Json& raw, const std::string& what); + static std::pair text(const Json& raw, const std::string& what, int limit); + static Result options(const Json& raw, const std::string& what); + static Result condition(const Json& raw, const std::string& what, const std::set& known); + // normalized app {"title","timeout","start","screens"} or (null, reason) + static Result spec(const Json& spec); +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/native_plexer_controller.cpp b/taicpp/src/plexer/native_plexer_controller.cpp new file mode 100644 index 0000000..6fb5d96 --- /dev/null +++ b/taicpp/src/plexer/native_plexer_controller.cpp @@ -0,0 +1,164 @@ +#include "native_plexer_controller.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "plexer_consts.h" +#include "plexer_error.h" +#include "store/utf8.h" +namespace detz::plexer { + +namespace { +constexpr int kCallTimeoutMs = 10000; // conn.settimeout(10) + +std::string errno_text(int e) { return "[Errno " + std::to_string(e) + "] " + std::strerror(e); } + +// Waits for fd to become ready; false on the 10 s socket timeout. +bool wait_fd(int fd, short events) { + struct pollfd p = {fd, events, 0}; + while (true) { + int rc = ::poll(&p, 1, kCallTimeoutMs); + if (rc < 0 && errno == EINTR) continue; + return rc > 0; + } +} +} // namespace + +NativePlexerController::NativePlexerController(std::string socket_path, std::string pane) + : socket_path_(std::move(socket_path)), pane_(std::move(pane)) {} + +NativePlexerController::Json NativePlexerController::call(Json payload) { + if (!pane_.empty()) payload.set("pane", pane_); + int fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) throw PlexerError("cannot reach native plexer backend: " + errno_text(errno)); + ::fcntl(fd, F_SETFL, ::fcntl(fd, F_GETFL) | O_NONBLOCK); + struct sockaddr_un addr; + std::memset(&addr, 0, sizeof addr); + addr.sun_family = AF_UNIX; + if (socket_path_.size() >= sizeof addr.sun_path) { + ::close(fd); + throw PlexerError("cannot reach native plexer backend: AF_UNIX path too long"); + } + std::memcpy(addr.sun_path, socket_path_.c_str(), socket_path_.size() + 1); + int rc = ::connect(fd, reinterpret_cast(&addr), sizeof addr); + if (rc != 0 && (errno == EINPROGRESS || errno == EAGAIN)) { + if (!wait_fd(fd, POLLOUT)) { ::close(fd); throw PlexerError("cannot reach native plexer backend: timed out"); } + int soerr = 0; + socklen_t sl = sizeof soerr; + ::getsockopt(fd, SOL_SOCKET, SO_ERROR, &soerr, &sl); + if (soerr != 0) { ::close(fd); throw PlexerError("cannot reach native plexer backend: " + errno_text(soerr)); } + } else if (rc != 0) { + int e = errno; + ::close(fd); + throw PlexerError("cannot reach native plexer backend: " + errno_text(e)); + } + std::string chunks; + try { + std::string line = payload.dumps() + "\n"; + size_t sent = 0; + while (sent < line.size()) { + ssize_t n = ::send(fd, line.data() + sent, line.size() - sent, MSG_NOSIGNAL); + if (n < 0) { + if (errno == EAGAIN || errno == EINTR) { if (!wait_fd(fd, POLLOUT)) throw PlexerError("native plexer backend read/write failed: timed out"); continue; } + throw PlexerError("native plexer backend read/write failed: " + errno_text(errno)); + } + sent += static_cast(n); + } + char buf[65536]; + while (true) { + ssize_t n = ::recv(fd, buf, sizeof buf, 0); + if (n < 0) { + if (errno == EAGAIN || errno == EINTR) { if (!wait_fd(fd, POLLIN)) throw PlexerError("native plexer backend read/write failed: timed out"); continue; } + throw PlexerError("native plexer backend read/write failed: " + errno_text(errno)); + } + if (n == 0) break; + chunks.append(buf, static_cast(n)); + if (chunks.back() == '\n') break; + } + } catch (...) { + ::close(fd); + throw; + } + ::close(fd); + std::string raw = store::Utf8::strip(store::Utf8::sanitize(chunks)); + Json reply = Json::object(); + if (!raw.empty()) { + try { reply = Json::loads(raw); } catch (const store::JsonError&) { throw PlexerError("native plexer backend sent an unparseable reply"); } + } + if (!reply.is_object()) throw PlexerError("native plexer backend sent an unparseable reply"); + const Json* ok = reply.find("ok"); + bool truthy = ok && ((ok->is_bool() && ok->as_bool()) || (ok->is_number() && ok->as_double() != 0) || (ok->is_string() && !ok->as_string().empty()) || ((ok->is_array() || ok->is_object()) && ok->size() > 0)); + if (!truthy) { + const Json* err = reply.find("error"); + std::string msg = (err && err->is_string() && !err->as_string().empty()) ? err->as_string() : "native plexer backend refused the command"; + throw PlexerError(msg); + } + const Json* data = reply.find("data"); + return data ? *data : Json(); +} + +NativePlexerController::Json NativePlexerController::send_keys(const std::string& text, bool enter) { + Json p = Json::object(); + p.set("cmd", "send"); p.set("text", text); p.set("enter", enter); + return call(p); +} + +NativePlexerController::Json NativePlexerController::read_pane(int lines) { + Json p = Json::object(); + p.set("cmd", "read"); p.set("lines", lines); + return call(p); +} + +NativePlexerController::Json NativePlexerController::status() { + Json p = Json::object(); + p.set("cmd", "status"); + return call(p); +} + +NativePlexerController::Json NativePlexerController::ask(const Json& screen, const Json& timeout_json) { + double timeout = timeout_json.as_double(); + Json show = Json::object(); + show.set("cmd", "show"); show.set("screen", screen); show.set("timeout", timeout_json); + Json handle = call(show); + const Json* idp = handle.is_object() ? handle.find("id") : nullptr; + if (!idp) throw PlexerError("native plexer backend sent no flow id"); + Json flow_id = *idp; + auto deadline = std::chrono::steady_clock::now() + std::chrono::duration_cast(std::chrono::duration(std::max(1.0, timeout))); + while (true) { + Json q = Json::object(); + q.set("cmd", "result"); q.set("id", flow_id); + Json reply = call(q); + const Json* done = reply.is_object() ? reply.find("done") : nullptr; + if (done && ((done->is_bool() && done->as_bool()) || (done->is_number() && done->as_double() != 0))) { + std::string status = reply.get_string("status"); + Json out = Json::object(); + if (status == "ok") { + out.set("status", "ok"); + const Json* v = reply.find("value"); + out.set("value", v ? *v : Json()); + return out; + } + if (status == "cancelled" || status == "timeout") { out.set("status", status); return out; } + throw PlexerError("flow " + flow_id.py_str() + " ended: " + (status.empty() ? "unknown" : status)); + } + if (std::chrono::steady_clock::now() >= deadline) { + try { + Json c = Json::object(); + c.set("cmd", "cancel"); c.set("id", flow_id); + call(c); + } catch (const PlexerError&) {} + Json out = Json::object(); + out.set("status", "timeout"); + return out; + } + std::this_thread::sleep_for(std::chrono::duration(Plexer::kFlowPoll)); + } +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/native_plexer_controller.h b/taicpp/src/plexer/native_plexer_controller.h new file mode 100644 index 0000000..aeb004c --- /dev/null +++ b/taicpp/src/plexer/native_plexer_controller.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include "plexer_controller.h" +namespace detz::plexer { + +// Our own NativePlexerBackend already wraps this shell: talk to it over the +// unix socket named by TAI_PLEXER_SOCKET, one JSON line per request. +class NativePlexerController : public PlexerController { + public: + NativePlexerController(std::string socket_path, std::string pane = ""); + std::string kind() const override { return "native"; } + Json send_keys(const std::string& text, bool enter = false) override; + Json read_pane(int lines = 200) override; + Json status() override; + using PlexerController::ask; + Json ask(const Json& screen, const Json& timeout) override; + + const std::string& socket_path() const { return socket_path_; } + const std::string& pane() const { return pane_; } + + // the wire primitive: one JSON line out, the reply "data" back (PlexerError on any failure) + Json call(Json payload); + + private: + std::string socket_path_; + std::string pane_; +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer.cpp b/taicpp/src/plexer/plexer.cpp index 5f2d720..e571978 100644 --- a/taicpp/src/plexer/plexer.cpp +++ b/taicpp/src/plexer/plexer.cpp @@ -1,8 +1,35 @@ #include "plexer.h" -#include +#include +#include +#include "store/utf8.h" namespace detz::plexer { -static std::string sh(const std::string& c) { std::string o; char b[256]; FILE* p=::popen(c.c_str(),"r"); if(!p) return o; while(std::fgets(b,sizeof b,p)) o+=b; ::pclose(p); return o; } -std::vector Tmux::list() { std::vector v; for (auto& l : std::vector{sh("tmux list-panes -F '#{pane_id} #{pane_current_path} #{pane_current_command}' 2>/dev/null")}) {(void)l;} return v; } -std::string Tmux::capture(const std::string& id) { return sh("tmux capture-pane -p -t " + id + " 2>/dev/null"); } -void Tmux::send(const std::string& id, const std::string& k) { (void)sh("tmux send-keys -t " + id + " " + k + " 2>/dev/null"); } + +const std::vector& Plexer::FlowTypes() { static const std::vector v = {"choice", "text", "info"}; return v; } +const std::vector& Plexer::FlowKinds() { static const std::vector v = {"message", "choice", "text"}; return v; } + +std::string Plexer::dir(const std::string& home) { + std::string path = (std::filesystem::path(home) / kDirName).string(); + std::error_code ec; + std::filesystem::create_directories(path, ec); + ::chmod(path.c_str(), 0700); + return path; +} + +std::string Plexer::shlex_quote(const std::string& s) { + if (s.empty()) return "''"; + bool safe = true; + for (unsigned char c : s) { + bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '@' || c == '%' || c == '+' || c == '=' || c == ':' || c == ',' || c == '.' || c == '/' || c == '-'; + if (!ok) { safe = false; break; } + } + if (safe) return s; + std::string out = "'"; + for (char c : s) { if (c == '\'') out += "'\"'\"'"; else out += c; } + return out + "'"; +} + +std::string Plexer::short_error(const std::string& what, const std::string& fallback) { + return store::Utf8::prefix(what.empty() ? fallback : what, 300); +} + } // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer.h b/taicpp/src/plexer/plexer.h index b83a4ee..ba17d3f 100644 --- a/taicpp/src/plexer/plexer.h +++ b/taicpp/src/plexer/plexer.h @@ -1,27 +1,19 @@ #pragma once -#include -#include -namespace detz::plexer { -struct Pane { std::string id; std::string cwd; std::string cmd; }; -class Controller { - public: - virtual ~Controller() = default; - virtual std::vector list() = 0; - virtual std::string capture(const std::string& id) = 0; - virtual void send(const std::string& id, const std::string& keys) = 0; -}; -class Native : public Controller { - public: - std::vector list() override { return panes_; } - std::string capture(const std::string&) override { return ""; } - void send(const std::string&, const std::string&) override {} - private: - std::vector panes_; -}; -class Tmux : public Controller { - public: - std::vector list() override; - std::string capture(const std::string& id) override; - void send(const std::string& id, const std::string& keys) override; -}; -} // namespace detz::plexer +// Umbrella header of the plexer module (tai.py PlexerController family and +// PlexerFlow, uiflow.md). One class per header: +// plexer_error.h PlexerError +// plexer_consts.h Plexer (constants, plexer_dir, shlex_quote) +// plexer_controller.h PlexerController (abstract) +// native_plexer_controller.h NativePlexerController +// tmux_plexer_controller.h TmuxPlexerController +// plexer_controller_factory.h PlexerControllerFactory::detect +// flow_validator.h FlowValidator (the plexer_show JSON contract) +// plexer_flow.h PlexerFlow +#include "native_plexer_controller.h" +#include "plexer_consts.h" +#include "plexer_controller.h" +#include "plexer_controller_factory.h" +#include "plexer_error.h" +#include "flow_validator.h" +#include "plexer_flow.h" +#include "tmux_plexer_controller.h" diff --git a/taicpp/src/plexer/plexer_consts.h b/taicpp/src/plexer/plexer_consts.h new file mode 100644 index 0000000..a320fb6 --- /dev/null +++ b/taicpp/src/plexer/plexer_consts.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include +namespace detz::plexer { + +// PLEXER_* / MPLEX_FLOW_* constants of tai.py plus the two tiny helpers the +// controllers share (plexer_dir, shlex.quote, short_error). +class Plexer { + public: + static constexpr const char* kDirName = "plexer"; // PLEXER_DIR_NAME + static constexpr const char* kPaneEnv = "TAI_PLEXER_PANE"; // PLEXER_PANE_ENV + static constexpr const char* kSocketEnv = "TAI_PLEXER_SOCKET"; // PLEXER_SOCKET_ENV + static constexpr const char* kActiveEnv = "TAI_PLEXER_ACTIVE"; // PLEXER_ACTIVE_ENV + static constexpr int kRecvMax = 1 << 20; // PLEXER_RECV_MAX + + static constexpr int kFlowVersion = 1; + static constexpr int kFlowScreensMax = 32; + static constexpr int kFlowFieldsMax = 12; + static constexpr int kFlowOptionsMax = 12; + static constexpr int kFlowActionsMax = 12; + static constexpr int kFlowVisitsMax = 64; + static constexpr int kFlowIdMax = 64; + static constexpr int kFlowTitleMax = 200; + static constexpr int kFlowTextMax = 2000; + static constexpr int kFlowLabelMax = 200; + static constexpr int kFlowAnswerMax = 2000; + static constexpr int kFlowTimeoutDefault = 300; + static constexpr int kFlowTimeoutMin = 5; + static constexpr int kFlowTimeoutMax = 600; + static constexpr double kFlowPoll = 0.25; + static constexpr int kMplexFlowMax = 16; + static constexpr int kMplexFlowKeep = 300; + static const std::vector& FlowTypes(); // choice, text, info + static const std::vector& FlowKinds(); // message, choice, text + + // /plexer, created with mode 0700 (chmod failure ignored) + static std::string dir(const std::string& home); + // python shlex.quote + static std::string shlex_quote(const std::string& s); + // str(exc)[:300], or `fallback` (the exception type name) when empty + static std::string short_error(const std::string& what, const std::string& fallback = "Exception"); +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_controller.cpp b/taicpp/src/plexer/plexer_controller.cpp new file mode 100644 index 0000000..6dd680e --- /dev/null +++ b/taicpp/src/plexer/plexer_controller.cpp @@ -0,0 +1,22 @@ +#include "plexer_controller.h" +#include +#include +namespace detz::plexer { + +void PlexerController::type_text(const std::string& text, double delay, bool enter) { + size_t i = 0; + while (i < text.size()) { + size_t n = 1; + unsigned char c = static_cast(text[i]); + if ((c & 0xE0) == 0xC0) n = 2; + else if ((c & 0xF0) == 0xE0) n = 3; + else if ((c & 0xF8) == 0xF0) n = 4; + if (i + n > text.size()) n = text.size() - i; + send_keys(text.substr(i, n)); + std::this_thread::sleep_for(std::chrono::duration(delay)); + i += n; + } + if (enter) send_keys("", true); +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_controller.h b/taicpp/src/plexer/plexer_controller.h new file mode 100644 index 0000000..edf97cb --- /dev/null +++ b/taicpp/src/plexer/plexer_controller.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include "store/json.h" +namespace detz::plexer { + +// What Tools.run_plexer_* calls: one small interface, two backends chosen by +// PlexerControllerFactory::detect from the environment. +class PlexerController { + public: + using Json = store::Json; + virtual ~PlexerController() = default; + virtual std::string kind() const { return "abstract"; } + + virtual Json send_keys(const std::string& text, bool enter = false) = 0; + virtual Json read_pane(int lines = 200) = 0; + virtual Json status() = 0; + // One blocking question on the user's terminal. screen is + // {"title","text","kind" (message/choice/text),"options" ([labels]),"initial"}. + // Returns {"status":"ok","value":...} (choice: option index, text: string, + // message: true), {"status":"cancelled"} or {"status":"timeout"}. + // Transport or backend failures throw PlexerError. + // timeout is the caller's number as given (an int stays an int on the wire) + virtual Json ask(const Json& screen, const Json& timeout) = 0; + Json ask(const Json& screen, double timeout) { return ask(screen, Json(timeout)); } + + // sends one character at a time (a python str iterates code points), then Enter when asked + virtual void type_text(const std::string& text, double delay = 0.03, bool enter = true); +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_controller_factory.cpp b/taicpp/src/plexer/plexer_controller_factory.cpp new file mode 100644 index 0000000..498ed7f --- /dev/null +++ b/taicpp/src/plexer/plexer_controller_factory.cpp @@ -0,0 +1,19 @@ +#include "plexer_controller_factory.h" +#include +#include "native_plexer_controller.h" +#include "plexer_consts.h" +#include "tmux_plexer_controller.h" +namespace detz::plexer { + +std::unique_ptr PlexerControllerFactory::detect() { + const char* socket_path = std::getenv(Plexer::kSocketEnv); + if (socket_path && *socket_path) { + const char* pane = std::getenv(Plexer::kPaneEnv); + return std::make_unique(socket_path, pane ? pane : ""); + } + const char* tmux_pane = std::getenv("TMUX_PANE"); + if (tmux_pane && *tmux_pane) return std::make_unique(tmux_pane); + return nullptr; +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_controller_factory.h b/taicpp/src/plexer/plexer_controller_factory.h new file mode 100644 index 0000000..77131ac --- /dev/null +++ b/taicpp/src/plexer/plexer_controller_factory.h @@ -0,0 +1,15 @@ +#pragma once +#include +#include "plexer_controller.h" +namespace detz::plexer { + +// The whole detection flow lives here: which environment variable is set +// decides which backend already manages this shell. Nothing here ever spawns +// a backend; that only happens via --plexer-wrap from the shell hook. +class PlexerControllerFactory { + public: + // TAI_PLEXER_SOCKET (+ TAI_PLEXER_PANE) -> native, else TMUX_PANE -> tmux, else null + static std::unique_ptr detect(); +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_error.h b/taicpp/src/plexer/plexer_error.h new file mode 100644 index 0000000..786063b --- /dev/null +++ b/taicpp/src/plexer/plexer_error.h @@ -0,0 +1,12 @@ +#pragma once +#include +#include +namespace detz::plexer { + +// Raised by a PlexerController when the backend can't be reached or refuses a command. +class PlexerError : public std::runtime_error { + public: + explicit PlexerError(const std::string& m) : std::runtime_error(m) {} +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_flow.cpp b/taicpp/src/plexer/plexer_flow.cpp new file mode 100644 index 0000000..339ed92 --- /dev/null +++ b/taicpp/src/plexer/plexer_flow.cpp @@ -0,0 +1,197 @@ +#include "plexer_flow.h" +#include +#include "plexer_consts.h" +#include "plexer_error.h" +#include "store/utf8.h" +namespace detz::plexer { + +using store::Json; +using store::Utf8; + +namespace { +std::string first_nonempty(const std::string& a, const std::string& b) { return a.empty() ? b : a; } +const Json& get(const Json& obj, const char* key) { + static const Json null_json; + const Json* v = obj.find(key); + return v ? *v : null_json; +} +} // namespace + +PlexerFlow::PlexerFlow(Json app) : app_(std::move(app)) { + for (auto& screen : app_.find("screens")->items()) { + by_id_[screen.get_string("id")] = &screen; + order_.push_back(screen.get_string("id")); + } +} + +bool PlexerFlow::condition_holds(const Json& clauses, const Json& answers) { + for (auto& clause : clauses.members()) { + const Json* have = answers.find(clause.first); + if (!have) return false; // MISSING is never in accepted + bool found = false; + for (auto& accepted : clause.second.items()) if (accepted == *have) { found = true; break; } + if (!found) return false; + } + return true; +} + +std::string PlexerFlow::follow_up(const Json& screen) const { + const Json& target = get(screen, "next"); + if (target.is_string()) return target.as_string(); + auto pos = std::find(order_.begin(), order_.end(), screen.get_string("id")) - order_.begin(); + if (static_cast(pos) + 1 < order_.size()) return order_[static_cast(pos) + 1]; + return "$done"; +} + +int PlexerFlow::first_askable(const Json& screen) const { + int idx = 0; + for (auto& field : get(screen, "fields").items()) { + std::string t = field.get_string("type"); + if ((t == "choice" || t == "text") && condition_holds(get(field, "show_if"), answers_)) return idx; + idx++; + } + return -1; +} + +std::string PlexerFlow::screen_text(const Json& screen, int field_index) const { + const Json* field = field_index >= 0 ? &get(screen, "fields").items()[static_cast(field_index)] : nullptr; + std::string app_title = app_.get_string("title"); + if (field && field_index != first_askable(screen)) return first_nonempty(field->get_string("label"), first_nonempty(screen.get_string("title"), app_title)); + std::vector parts{first_nonempty(screen.get_string("title"), app_title), screen.get_string("body")}; + for (auto& item : get(screen, "fields").items()) { + if (item.get_string("type") == "info" && condition_holds(get(item, "show_if"), answers_)) parts.push_back(item.get_string("text")); + } + if (field && !field->get_string("label").empty()) parts.push_back(field->get_string("label")); + std::string out; + for (auto& p : parts) { + if (p.empty() || Utf8::strip(p).empty()) continue; + if (!out.empty()) out += "\n\n"; + out += p; + } + return out; +} + +double PlexerFlow::remaining() const { + return std::chrono::duration(deadline_ - std::chrono::steady_clock::now()).count(); +} + +PlexerFlow::Json PlexerFlow::result_with(const std::string& status, const std::string& at, const std::string& field) const { + Json o = Json::object(); + o.set("status", status); + o.set("at", at); + if (!field.empty()) o.set("field", field); + o.set("answers", answers_); + return o; +} + +PlexerFlow::Json PlexerFlow::run(PlexerController& controller) { + deadline_ = std::chrono::steady_clock::now() + std::chrono::duration_cast(std::chrono::duration(app_.find("timeout")->as_double())); + std::string current = app_.get_string("start"); + while (true) { + visits_++; + if (visits_ > Plexer::kFlowVisitsMax) { + Json o = Json::object(); + o.set("status", "error"); + o.set("error", "flow passed " + std::to_string(Plexer::kFlowVisitsMax) + " screens (loop?), aborting"); + o.set("at", current); + o.set("answers", answers_); + return o; + } + const Json& screen = *by_id_.at(current); + std::string target; + if (condition_holds(get(screen, "show_if"), answers_)) { + Outcome outcome = run_screen(controller, screen); + if (outcome.kind == Outcome::Kind::Final) return outcome.result; + target = outcome.kind == Outcome::Kind::Jump ? outcome.target : follow_up(screen); + } else { + target = follow_up(screen); + } + if (target == "$done") { + Json o = Json::object(); + o.set("status", "done"); o.set("answers", answers_); o.set("action", action_); + return o; + } + if (target == "$cancel") return result_with("cancelled", current); + current = target; + } +} + +PlexerFlow::Outcome PlexerFlow::run_screen(PlexerController& controller, const Json& screen) { + auto final_of = [&](Json j) { Outcome o; o.kind = Outcome::Kind::Final; o.result = std::move(j); return o; }; + std::string sid = screen.get_string("id"); + std::string title = first_nonempty(screen.get_string("title"), app_.get_string("title")); + bool asked = false; + const auto& fields = get(screen, "fields").items(); + for (size_t fi = 0; fi < fields.size(); fi++) { + const Json& field = fields[fi]; + std::string type = field.get_string("type"); + if (type == "info" || !condition_holds(get(field, "show_if"), answers_)) continue; + if (remaining() <= 0) return final_of(result_with("timeout", sid)); + Json ask = Json::object(); + ask.set("title", title); + ask.set("text", screen_text(screen, static_cast(fi))); + Json reply; + if (type == "choice") { + Json labels = Json::array(); + for (auto& opt : get(field, "options").items()) labels.push(Json(opt.get_string("label"))); + ask.set("kind", "choice"); ask.set("options", labels); ask.set("default", get(field, "default")); + reply = controller.ask(ask, remaining()); + if (reply.get_string("status") != "ok") return final_of(result_with(reply.get_string("status"), sid, field.get_string("id"))); + const Json* v = reply.find("value"); + auto idx = v ? v->as_int() : -1; + const auto& opts = get(field, "options").items(); + if (idx < 0 || idx >= static_cast(opts.size())) throw PlexerError("choice index out of range"); + answers_.set(field.get_string("id"), get(opts[static_cast(idx)], "value")); + } else { + ask.set("kind", "text"); ask.set("initial", get(field, "initial")); + reply = controller.ask(ask, remaining()); + if (reply.get_string("status") != "ok") return final_of(result_with(reply.get_string("status"), sid, field.get_string("id"))); + answers_.set(field.get_string("id"), get(reply, "value")); + } + asked = true; + } + const auto& actions = get(screen, "actions").items(); + if (actions.empty()) { + if (!asked) { + if (remaining() <= 0) return final_of(result_with("timeout", sid)); + Json ask = Json::object(); + ask.set("title", title); ask.set("text", screen_text(screen)); ask.set("kind", "message"); + Json reply = controller.ask(ask, remaining()); + if (reply.get_string("status") != "ok") return final_of(result_with(reply.get_string("status"), sid)); + } + return Outcome{}; + } + if (remaining() <= 0) return final_of(result_with("timeout", sid)); + std::string text; + if (first_askable(screen) < 0 && Utf8::strip(screen.get_string("body")).empty()) text = title; + else if (first_askable(screen) < 0) text = screen_text(screen); + else text = title; + Json labels = Json::array(); + for (auto& a : actions) labels.push(Json(a.get_string("label"))); + Json ask = Json::object(); + ask.set("title", title); ask.set("text", text); ask.set("kind", "choice"); ask.set("options", labels); ask.set("default", 0); + Json reply = controller.ask(ask, remaining()); + if (reply.get_string("status") != "ok") return final_of(result_with(reply.get_string("status"), sid)); + const Json* v = reply.find("value"); + auto idx = v ? v->as_int() : -1; + if (idx < 0 || idx >= static_cast(actions.size())) throw PlexerError("choice index out of range"); + action_ = Json(actions[static_cast(idx)].get_string("id")); + const Json& target = get(screen, "next"); + if (target.is_object()) { + const Json* dest = target.find(action_.as_string()); + if (!dest) return Outcome{}; + if (dest->is_string() && dest->as_string() == "$done") { + Json o = Json::object(); + o.set("status", "done"); o.set("answers", answers_); o.set("action", action_); + return final_of(o); + } + if (dest->is_string() && dest->as_string() == "$cancel") return final_of(result_with("cancelled", sid)); + Outcome o; + o.kind = Outcome::Kind::Jump; + o.target = dest->as_string(); + return o; + } + return Outcome{}; +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/plexer_flow.h b/taicpp/src/plexer/plexer_flow.h new file mode 100644 index 0000000..38857d7 --- /dev/null +++ b/taicpp/src/plexer/plexer_flow.h @@ -0,0 +1,48 @@ +#pragma once +#include +#include +#include +#include +#include "plexer_controller.h" +#include "store/json.h" +namespace detz::plexer { + +// The driver of one UI flow: the main agent hands over a whole validated app +// in a single plexer_show call, this decomposes it into PlexerController::ask +// calls, evaluates conditions and branches locally, and reports only the +// final outcome (uiflow.md). +class PlexerFlow { + public: + using Json = store::Json; + explicit PlexerFlow(Json app); + PlexerFlow(const PlexerFlow&) = delete; + PlexerFlow& operator=(const PlexerFlow&) = delete; + // {"status": done|cancelled|timeout|error, "answers": {...}, "action": ..., "at": ..., "field": ...} + Json run(PlexerController& controller); + + // answers.get(key, MISSING) in accepted for every clause + static bool condition_holds(const Json& clauses, const Json& answers); + + private: + struct Outcome { + enum class Kind { Continue, Jump, Final } kind{Kind::Continue}; + std::string target; + Json result; + }; + std::string follow_up(const Json& screen) const; + std::string screen_text(const Json& screen, int field_index = -1) const; + int first_askable(const Json& screen) const; // index into fields, -1 for none + double remaining() const; + Outcome run_screen(PlexerController& controller, const Json& screen); + Json result_with(const std::string& status, const std::string& at, const std::string& field = "") const; + + Json app_; + std::map by_id_; + std::vector order_; + Json answers_ = Json::object(); + Json action_; + int visits_{0}; + std::chrono::steady_clock::time_point deadline_; +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/proc_runner.cpp b/taicpp/src/plexer/proc_runner.cpp new file mode 100644 index 0000000..15f1768 --- /dev/null +++ b/taicpp/src/plexer/proc_runner.cpp @@ -0,0 +1,82 @@ +#include "proc_runner.h" +#include +#include +#include +#include +#include +#include +#include +#include +namespace detz::plexer { + +static void set_cloexec(int fd) { ::fcntl(fd, F_SETFD, ::fcntl(fd, F_GETFD) | FD_CLOEXEC); } + +static std::string universal_newlines(const std::string& raw) { + std::string out; + out.reserve(raw.size()); + for (size_t i = 0; i < raw.size(); i++) { + if (raw[i] == '\r') { out += '\n'; if (i + 1 < raw.size() && raw[i + 1] == '\n') i++; } + else out += raw[i]; + } + return out; +} + +ProcResult ProcRunner::run(const std::vector& argv, double timeout_seconds) { + int out_pipe[2], err_pipe[2], exec_pipe[2]; + if (::pipe(out_pipe) != 0 || ::pipe(err_pipe) != 0 || ::pipe(exec_pipe) != 0) { + throw SpawnError(std::string("[Errno ") + std::to_string(errno) + "] " + std::strerror(errno)); + } + set_cloexec(exec_pipe[1]); + pid_t pid = ::fork(); + if (pid < 0) throw SpawnError(std::string("[Errno ") + std::to_string(errno) + "] " + std::strerror(errno)); + if (pid == 0) { + ::dup2(out_pipe[1], 1); + ::dup2(err_pipe[1], 2); + ::close(out_pipe[0]); ::close(out_pipe[1]); ::close(err_pipe[0]); ::close(err_pipe[1]); ::close(exec_pipe[0]); + std::vector args; + for (auto& a : argv) args.push_back(const_cast(a.c_str())); + args.push_back(nullptr); + ::execvp(args[0], args.data()); + int e = errno; + ssize_t w = ::write(exec_pipe[1], &e, sizeof e); + (void)w; + ::_exit(127); + } + ::close(out_pipe[1]); ::close(err_pipe[1]); ::close(exec_pipe[1]); + int child_errno = 0; + ssize_t got = ::read(exec_pipe[0], &child_errno, sizeof child_errno); + ::close(exec_pipe[0]); + if (got == static_cast(sizeof child_errno)) { + ::waitpid(pid, nullptr, 0); + ::close(out_pipe[0]); ::close(err_pipe[0]); + throw SpawnError(std::string("[Errno ") + std::to_string(child_errno) + "] " + std::strerror(child_errno) + ": '" + argv[0] + "'"); + } + ProcResult res; + std::string out, err; + auto deadline = std::chrono::steady_clock::now() + std::chrono::duration_cast(std::chrono::duration(timeout_seconds)); + struct pollfd fds[2] = {{out_pipe[0], POLLIN, 0}, {err_pipe[0], POLLIN, 0}}; + int open_fds = 2; + char buf[8192]; + while (open_fds > 0) { + auto left = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()).count(); + if (left <= 0) { res.timed_out = true; break; } + int rc = ::poll(fds, 2, static_cast(left > 1000000 ? 1000000 : left)); + if (rc < 0) { if (errno == EINTR) continue; break; } + for (int k = 0; k < 2; k++) { + if (fds[k].fd < 0 || !(fds[k].revents & (POLLIN | POLLHUP | POLLERR))) continue; + ssize_t n = ::read(fds[k].fd, buf, sizeof buf); + if (n > 0) (k == 0 ? out : err).append(buf, static_cast(n)); + else { ::close(fds[k].fd); fds[k].fd = -1; open_fds--; } + } + } + if (res.timed_out) ::kill(pid, SIGKILL); + for (int k = 0; k < 2; k++) if (fds[k].fd >= 0) ::close(fds[k].fd); + int status = 0; + ::waitpid(pid, &status, 0); + res.returncode = WIFEXITED(status) ? WEXITSTATUS(status) : (WIFSIGNALED(status) ? -WTERMSIG(status) : 0); + res.out = universal_newlines(out); + res.err = universal_newlines(err); + return res; +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/proc_runner.h b/taicpp/src/plexer/proc_runner.h new file mode 100644 index 0000000..6a19735 --- /dev/null +++ b/taicpp/src/plexer/proc_runner.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include +#include +namespace detz::plexer { + +// python OSError raised by subprocess.run (exec failure); what() is "[Errno N] text: 'argv0'". +class SpawnError : public std::runtime_error { + public: + explicit SpawnError(const std::string& m) : std::runtime_error(m) {} +}; + +struct ProcResult { + int returncode{0}; + std::string out, err; + bool timed_out{false}; +}; + +// subprocess.run(argv, stdout=PIPE, stderr=PIPE, universal_newlines=True, timeout=...). +// Used for the tmux control calls (not user-facing commands, which go through +// the live shell stream). On timeout the child is killed and timed_out is set. +class ProcRunner { + public: + static ProcResult run(const std::vector& argv, double timeout_seconds); +}; + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/tmux_plexer_controller.cpp b/taicpp/src/plexer/tmux_plexer_controller.cpp new file mode 100644 index 0000000..3427ccf --- /dev/null +++ b/taicpp/src/plexer/tmux_plexer_controller.cpp @@ -0,0 +1,216 @@ +#include "tmux_plexer_controller.h" +#include +#include +#include +#include +#include +#include +#include +#include "plexer_consts.h" +#include "plexer_error.h" +#include "proc_runner.h" +#include "store/utf8.h" +namespace detz::plexer { + +namespace { +std::string temp_dir() { + for (const char* name : {"TMPDIR", "TEMP", "TMP"}) { + const char* v = std::getenv(name); + if (v && *v) return v; + } + return "/tmp"; +} +// tempfile.mkstemp(prefix, suffix): create a fresh 0600 file, return its path +std::string mkstemp_path(const std::string& prefix, const std::string& suffix) { + std::string templ = temp_dir() + "/" + prefix + "XXXXXX" + suffix; + std::vector buf(templ.begin(), templ.end()); + buf.push_back('\0'); + int fd = ::mkstemps(buf.data(), static_cast(suffix.size())); + if (fd < 0) throw PlexerError("cannot stage prompt text: could not create a temp file"); + ::close(fd); + return std::string(buf.data()); +} +std::string trim(const std::string& s) { return store::Utf8::strip(s); } +} // namespace + +TmuxPlexerController::TmuxPlexerController(std::string pane) : pane_(std::move(pane)) {} + +std::string TmuxPlexerController::tmux(const std::vector& args) { + std::vector argv{"tmux"}; + argv.insert(argv.end(), args.begin(), args.end()); + ProcResult done; + try { + done = ProcRunner::run(argv, 15.0); + } catch (const SpawnError& e) { + throw PlexerError("tmux call failed: " + Plexer::short_error(e.what(), "OSError")); + } + if (done.timed_out) { + Json cmd = Json::array(); + for (auto& a : argv) cmd.push(Json(a)); + throw PlexerError("tmux call failed: " + Plexer::short_error("Command '" + cmd.py_repr() + "' timed out after 15 seconds", "TimeoutExpired")); + } + if (done.returncode != 0) throw PlexerError("tmux error: " + store::Utf8::prefix(trim(done.err), 300)); + return done.out; +} + +TmuxPlexerController::Json TmuxPlexerController::send_keys(const std::string& text, bool enter) { + if (!text.empty()) tmux({"send-keys", "-t", pane_, "-l", text}); + if (enter) tmux({"send-keys", "-t", pane_, "Enter"}); + return Json(true); +} + +TmuxPlexerController::Json TmuxPlexerController::read_pane(int lines) { + return Json(tmux({"capture-pane", "-p", "-t", pane_, "-S", "-" + std::to_string(std::max(1, lines))})); +} + +TmuxPlexerController::Json TmuxPlexerController::status() { + // list-panes, not display-message -p: display-message silently falls back to + // the default pane for a missing target, list-panes validates it. + std::string info = tmux({"list-panes", "-t", pane_, "-F", "#{session_name}:#{window_index}.#{pane_index} #{pane_current_command}"}); + Json o = Json::object(); + o.set("backend", "tmux"); + o.set("pane", pane_); + o.set("info", trim(info)); + return o; +} + +TmuxPlexerController::Json TmuxPlexerController::ask(const Json& screen, const Json& timeout_json) { + double timeout = timeout_json.as_double(); + std::string kind = screen.get_string("kind"); + if (kind == "choice") return ask_choice(screen, timeout); + if (kind == "text") return ask_text(screen, timeout); + if (kind == "message") return ask_message(screen, timeout); + const Json* k = screen.find("kind"); + throw PlexerError("unknown screen kind " + (k ? k->py_repr() : std::string("None"))); +} + +std::string TmuxPlexerController::ask_result_path() { + std::string path = mkstemp_path("tai-flow-", ".answer"); + ::unlink(path.c_str()); + return path; +} + +std::optional TmuxPlexerController::await_answer(const std::string& path, double budget) { + auto deadline = std::chrono::steady_clock::now() + std::chrono::duration_cast(std::chrono::duration(std::max(0.0, budget))); + while (true) { + std::ifstream in(path, std::ios::binary); + if (in) { + std::stringstream ss; + ss << in.rdbuf(); + return store::Utf8::sanitize(ss.str()); + } + if (std::chrono::steady_clock::now() >= deadline) return std::nullopt; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } +} + +void TmuxPlexerController::ask_dismiss() { + try { ProcRunner::run({"tmux", "display-popup", "-C"}, 5.0); } catch (const SpawnError&) {} +} + +std::optional TmuxPlexerController::ask_run(const std::vector& argv, const std::string& path, double timeout, bool& timed_out) { + timed_out = false; + std::vector full{"tmux"}; + full.insert(full.end(), argv.begin(), argv.end()); + ProcResult done; + try { + done = ProcRunner::run(full, std::max(1.0, timeout)); + } catch (const SpawnError& e) { + throw PlexerError("tmux call failed: " + Plexer::short_error(e.what(), "OSError")); + } + if (done.timed_out) { ask_dismiss(); timed_out = true; return std::nullopt; } + if (done.returncode != 0 && !trim(done.err).empty()) throw PlexerError("tmux " + argv[0] + " failed: " + store::Utf8::prefix(trim(done.err), 300)); + if (done.returncode != 0) return std::nullopt; + return await_answer(path); +} + +TmuxPlexerController::Json TmuxPlexerController::ask_choice(const Json& screen, double timeout) { + std::vector options; + if (const Json* o = screen.find("options")) if (o->is_array()) for (auto& it : o->items()) options.push_back(it.py_str()); + std::string path = ask_result_path(); + std::optional raw; + bool timed_out = false; + try { + std::string title = screen.get_string("title"); + std::vector argv{"display-menu", "-T", title.empty() ? "tai" : title}; + static const std::string keys = "123456789abcdefghijklmnopqrstuvwxyz"; + for (size_t index = 0; index < options.size(); index++) { + std::string key = index < keys.size() ? std::string(1, keys[index]) : std::string(); + argv.push_back(options[index]); + argv.push_back(key); + argv.push_back("run-shell \"printf '%s' '" + std::to_string(index) + "' > " + Plexer::shlex_quote(path) + "\""); + } + raw = ask_run(argv, path, timeout, timed_out); + } catch (...) { + ::unlink(path.c_str()); + throw; + } + ::unlink(path.c_str()); + Json out = Json::object(); + if (timed_out) { out.set("status", "timeout"); return out; } + std::string t = raw ? trim(*raw) : ""; + bool digits = !t.empty() && std::all_of(t.begin(), t.end(), [](char c) { return c >= '0' && c <= '9'; }); + if (!raw || !digits) { out.set("status", "cancelled"); return out; } + long long index = std::stoll(t); + if (index >= 0 && index < static_cast(options.size())) { + out.set("status", "ok"); + out.set("value", index); + return out; + } + out.set("status", "cancelled"); + return out; +} + +std::optional TmuxPlexerController::ask_popup(const std::string& title, const std::string& body, const std::string& reader_fmt, double timeout, bool& timed_out) { + std::string prompt_path = mkstemp_path("tai-flow-", ".prompt"); + try { + std::ofstream out(prompt_path, std::ios::binary | std::ios::trunc); + if (!out) throw PlexerError("cannot stage prompt text: could not write " + prompt_path); + out << body; + } catch (...) { + ::unlink(prompt_path.c_str()); + throw; + } + std::string path = ask_result_path(); + std::optional raw; + try { + std::string reader = reader_fmt; + size_t at = reader.find("%PATH%"); + reader.replace(at, 6, Plexer::shlex_quote(path)); + std::string command = "cat " + Plexer::shlex_quote(prompt_path) + "; " + reader; + raw = ask_run({"display-popup", "-w", "62", "-h", "10", "-T", title.empty() ? "tai" : title, "-E", command}, path, timeout, timed_out); + } catch (...) { + ::unlink(prompt_path.c_str()); + ::unlink(path.c_str()); + throw; + } + ::unlink(prompt_path.c_str()); + ::unlink(path.c_str()); + return raw; +} + +TmuxPlexerController::Json TmuxPlexerController::ask_text(const Json& screen, double timeout) { + std::string body = screen.get_string("text") + "\n\n> "; + bool timed_out = false; + auto raw = ask_popup(screen.get_string("title"), body, "read -r TAI_V && printf '%s' \"$TAI_V\" > %PATH%", timeout, timed_out); + Json out = Json::object(); + if (timed_out) { out.set("status", "timeout"); return out; } + if (!raw) { out.set("status", "cancelled"); return out; } + out.set("status", "ok"); + out.set("value", store::Utf8::prefix(*raw, Plexer::kFlowAnswerMax)); + return out; +} + +TmuxPlexerController::Json TmuxPlexerController::ask_message(const Json& screen, double timeout) { + std::string body = screen.get_string("text") + "\n\n[Enter] "; + bool timed_out = false; + auto raw = ask_popup(screen.get_string("title"), body, "read -r _ && printf ok > %PATH%", timeout, timed_out); + Json out = Json::object(); + if (timed_out) { out.set("status", "timeout"); return out; } + if (!raw) { out.set("status", "cancelled"); return out; } + out.set("status", "ok"); + out.set("value", true); + return out; +} + +} // namespace detz::plexer diff --git a/taicpp/src/plexer/tmux_plexer_controller.h b/taicpp/src/plexer/tmux_plexer_controller.h new file mode 100644 index 0000000..701c1b5 --- /dev/null +++ b/taicpp/src/plexer/tmux_plexer_controller.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include +#include "plexer_controller.h" +namespace detz::plexer { + +// tmux is already the active manager of this shell (TMUX_PANE): drive it +// through the tmux CLI. Questions render as display-menu / display-popup and +// answer through a temp file. +class TmuxPlexerController : public PlexerController { + public: + explicit TmuxPlexerController(std::string pane); + std::string kind() const override { return "tmux"; } + Json send_keys(const std::string& text, bool enter = false) override; + Json read_pane(int lines = 200) override; + Json status() override; + using PlexerController::ask; + Json ask(const Json& screen, const Json& timeout) override; + + const std::string& pane() const { return pane_; } + + private: + // tmux , stdout on success; PlexerError on failure + std::string tmux(const std::vector& args); + std::string ask_result_path(); + std::optional await_answer(const std::string& path, double budget = 1.0); + void ask_dismiss(); + // answer file content, nullopt when dismissed, "timeout" sentinel via `timed_out` + std::optional ask_run(const std::vector& argv, const std::string& path, double timeout, bool& timed_out); + Json ask_choice(const Json& screen, double timeout); + std::optional ask_popup(const std::string& title, const std::string& body, const std::string& reader_fmt, double timeout, bool& timed_out); + Json ask_text(const Json& screen, double timeout); + Json ask_message(const Json& screen, double timeout); + std::string pane_; +}; + +} // namespace detz::plexer diff --git a/taicpp/src/repl/boot.cpp b/taicpp/src/repl/boot.cpp new file mode 100644 index 0000000..0af24cd --- /dev/null +++ b/taicpp/src/repl/boot.cpp @@ -0,0 +1,85 @@ +#include "boot.h" +#include +#include +#include "config/boot_settings.h" +#include "store/debuglog.h" +#include "store/patterns.h" +#include "store/utf8.h" +namespace detz::repl { + +namespace { +std::string lower_strip(const char* v) { + std::string s = store::Utf8::strip(v ? v : ""); + for (auto& c : s) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + return s; +} +std::string tai_home() { + const char* h = std::getenv("TAI_HOME"); + if (h && *h) return h; + const char* home = std::getenv("HOME"); + return std::string(home ? home : "~") + "/.tai"; +} +std::string join(const std::vector& v, const char* sep) { + std::string out; + for (size_t k = 0; k < v.size(); k++) { if (k) out += sep; out += v[k]; } + return out; +} +} // namespace + +void Boot::configure_debug(const std::optional& sink, const std::optional& format) { + std::string chosen = sink ? *sink : lower_strip(std::getenv("TAI_DEBUG")); + if (chosen != "file" && chosen != "stdout" && chosen != "stderr") return; + std::string fmt = format ? *format : lower_strip(std::getenv("TAI_DEBUG_FORMAT")); + if (fmt != "sentence" && fmt != "json") fmt = "sentence"; + store::DebugLog::Global().configure(chosen, fmt, tai_home()); +} + +bool Boot::valid_profile(const std::string& profile) { + if (store::Patterns::profile(profile)) return true; + std::fprintf(stderr, "invalid profile name\n"); + return false; +} + +std::pair Boot::resolve_passphrase() { + const char* raw = std::getenv("TAI_PASSPHRASE"); + if (!raw) return {"tai-default-insecure-change-me", true}; + return {raw, false}; +} + +std::unique_ptr Boot::open(const BootOptions& o) { + auto booted = std::make_unique(); + bool yes = o.yes || o.yolo || o.autonomous; + booted->config = Config::Build(o.profile.empty() ? "default" : o.profile, yes, o.no_opencode); + auto [passphrase, default_key] = resolve_passphrase(); + if (o.seal_factory) { + booted->seal = o.seal_factory(booted->config.home, passphrase, default_key); + } + if (!booted->seal) { + if (!passphrase.empty() && !default_key) throw store::SealError("TAI_PASSPHRASE is set but this build links no vault seal"); + booted->seal = std::make_unique(); + } + booted->store = std::make_unique(booted->config, *booted->seal); + auto& log = store::DebugLog::Global(); + if (log.enabled) log.attach_store(*booted->store); + if (!booted->store->schema_notes.empty()) { + std::string notes = join(booted->store->schema_notes, ", "); + std::printf("vault schema upgraded: %s\n", notes.c_str()); + store::Json meta = store::Json::object(); + meta.set("notes", notes); + log.event("INFO", "vault", "schema_upgraded", "vault schema upgraded: " + notes, booted->config.profile, "", "", meta); + } + auto applied = BootSettings::Boot(booted->config, *booted->store); + if (!applied.empty()) std::printf("settings from vault: %s\n", join(applied, ", ").c_str()); + booted->ctx = session::ShellContext::current(); + booted->ctx.register_in(*booted->store); + booted->ctx.export_env(); + booted->conversation = std::make_unique(*booted->store, booted->config, o.persist, o.yolo || o.autonomous, o.autonomous, booted->ctx.sid, o.catalogs); + store::Json meta = store::Json::object(); + meta.set("version", o.version); + meta.set("home", booted->config.home); + meta.set("mode", o.mode); + log.event("INFO", "app", "boot", "tai " + o.version + " booted (" + o.mode + ")", booted->conversation->profile(), "", "", meta); + return booted; +} + +} // namespace detz::repl diff --git a/taicpp/src/repl/boot.h b/taicpp/src/repl/boot.h new file mode 100644 index 0000000..ca528c4 --- /dev/null +++ b/taicpp/src/repl/boot.h @@ -0,0 +1,47 @@ +#pragma once +#include +#include +#include +#include +#include "config/config.h" +#include "session/conversation.h" +#include "session/shell_context.h" +#include "store/seal_port.h" +#include "store/store.h" +namespace detz::repl { + +struct BootOptions { + std::string profile; // "" = default + bool yes{false}, yolo{false}, autonomous{false}, no_opencode{false}; + bool persist{true}; + std::string mode{"repl"}; // reported in the boot log event + std::string version; + // Builds the vault seal for a passphrase. Unset: no passphrase means no seal; a passphrase with no + // factory is refused loudly (the crypto is not part of the foundation area). + std::function(const std::string& home, const std::string& passphrase, bool default_key)> seal_factory; + session::Conversation::Catalogs catalogs; +}; + +// Everything boot() of tai.py builds before an Agent exists, kept alive together. +class BootedSession { + public: + Config config; + std::unique_ptr seal; + std::unique_ptr store; + session::ShellContext ctx; + std::unique_ptr conversation; +}; + +class Boot { + public: + // main() prologue: --debug / --debug-format / TAI_DEBUG / TAI_DEBUG_FORMAT -> DebugLog::Global().configure + static void configure_debug(const std::optional& sink, const std::optional& format); + // PROFILE_RE check of main(); false prints "invalid profile name" to stderr + static bool valid_profile(const std::string& profile); + // resolve_passphrase(): TAI_PASSPHRASE when set (even empty), else the default key + static std::pair resolve_passphrase(); + // boot(): config, seal, store, vault settings, shell context, conversation. Throws store::SealError. + static std::unique_ptr open(const BootOptions& options); +}; + +} // namespace detz::repl diff --git a/taicpp/src/repl/interrupt.cpp b/taicpp/src/repl/interrupt.cpp new file mode 100644 index 0000000..5a21cfb --- /dev/null +++ b/taicpp/src/repl/interrupt.cpp @@ -0,0 +1,24 @@ +#include "interrupt.h" +#include +#include +namespace detz::repl { + +namespace { +std::atomic g_flag{false}; +std::atomic g_installed{false}; +void on_sigint(int) { g_flag.store(true); } +} // namespace + +void Interrupt::install() { + if (g_installed.exchange(true)) return; + struct sigaction sa; + sa.sa_handler = on_sigint; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; // no SA_RESTART: a pending read returns EINTR + sigaction(SIGINT, &sa, nullptr); +} +bool Interrupt::requested() { return g_flag.load(); } +void Interrupt::clear() { g_flag.store(false); } +void Interrupt::raise() { g_flag.store(true); } + +} // namespace detz::repl diff --git a/taicpp/src/repl/interrupt.h b/taicpp/src/repl/interrupt.h new file mode 100644 index 0000000..4999a4b --- /dev/null +++ b/taicpp/src/repl/interrupt.h @@ -0,0 +1,14 @@ +#pragma once +namespace detz::repl { + +// SIGINT as python's KeyboardInterrupt: a handler without SA_RESTART sets a +// flag and breaks a blocking read; the REPL and the turn runner poll it. +class Interrupt { + public: + static void install(); // idempotent + static bool requested(); + static void clear(); + static void raise(); // set the flag programmatically (tests) +}; + +} // namespace detz::repl diff --git a/taicpp/src/repl/line_input.cpp b/taicpp/src/repl/line_input.cpp new file mode 100644 index 0000000..4fba227 --- /dev/null +++ b/taicpp/src/repl/line_input.cpp @@ -0,0 +1,41 @@ +#include "line_input.h" +#include +#include +#include +#include "interrupt.h" +namespace detz::repl { + +LineInput::Status LineInput::read(const std::string& prompt, std::string& line) { + std::fputs(prompt.c_str(), stdout); + std::fflush(stdout); + while (true) { + size_t nl = pending_.find('\n'); + if (nl != std::string::npos) { + line = pending_.substr(0, nl); + pending_.erase(0, nl + 1); + if (!line.empty() && line.back() == '\r') line.pop_back(); + return Status::Line; + } + if (eof_) { + if (pending_.empty()) return Status::Eof; + line = pending_; + pending_.clear(); + return Status::Line; + } + if (Interrupt::requested()) { Interrupt::clear(); return Status::Interrupted; } + char buf[4096]; + ssize_t n = ::read(fd_, buf, sizeof buf); + if (n < 0) { + if (errno == EINTR) { + if (Interrupt::requested()) { Interrupt::clear(); pending_.clear(); return Status::Interrupted; } + continue; + } + eof_ = true; + continue; + } + if (n == 0) { eof_ = true; continue; } + pending_.append(buf, static_cast(n)); + } +} + +} // namespace detz::repl diff --git a/taicpp/src/repl/line_input.h b/taicpp/src/repl/line_input.h new file mode 100644 index 0000000..d04a0d3 --- /dev/null +++ b/taicpp/src/repl/line_input.h @@ -0,0 +1,20 @@ +#pragma once +#include +namespace detz::repl { + +// python input(prompt): writes the prompt to stdout, reads one line from fd 0. +// Eof maps to EOFError, Interrupted to KeyboardInterrupt. No line editing: +// the C++ port stays free of a readline dependency (documented gap). +class LineInput { + public: + enum class Status { Line, Eof, Interrupted }; + explicit LineInput(int fd = 0) : fd_(fd) {} + Status read(const std::string& prompt, std::string& line); + + private: + int fd_; + std::string pending_; + bool eof_{false}; +}; + +} // namespace detz::repl diff --git a/taicpp/src/repl/repl.cpp b/taicpp/src/repl/repl.cpp new file mode 100644 index 0000000..c9ad647 --- /dev/null +++ b/taicpp/src/repl/repl.cpp @@ -0,0 +1,174 @@ +#include "repl.h" +#include +#include "config/prompts.h" +#include "interrupt.h" +#include "repl_style.h" +#include "session/shell_context.h" +#include "session/term.h" +#include "store/patterns.h" +#include "store/utf8.h" +namespace detz::repl { + +using session::Term; +using store::Json; +using store::Utf8; + +namespace { +void say(const std::string& line) { std::printf("%s\n", line.c_str()); std::fflush(stdout); } +std::string red(const std::string& s) { return Term::paint(s, Term::kRed); } +std::string lower(std::string s) { for (auto& c : s) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); return s; } +} // namespace + +Repl::Repl(session::Conversation& conversation, ReplHost& host, Options options, LineInput* input) + : conv_(conversation), host_(host), options_(std::move(options)), input_(input ? input : &owned_input_) {} + +void Repl::banner() { + say(ReplStyle::title(options_.version, conv_.profile())); + if (conv_.yolo()) say(Term::paint("YOLO MODE: all approvals off", std::string(Term::kBold) + Term::kRed)); + if (conv_.autonomous()) say(Term::paint("AUTO MODE: never asks, researches instead", std::string(Term::kBold) + Term::kYellow)); + const auto& seal = conv_.store().seal(); + std::string seal_state = "off"; + if (seal.enabled()) seal_state = seal.default_key() ? "on (default key)" : "on (TAI_PASSPHRASE)"; + say(ReplStyle::field("seal", seal_state)); + if (!conv_.ctx_id().empty()) say(ReplStyle::field("ctx", conv_.ctx_id())); + for (auto& [label, status] : host_.backend_status()) say(ReplStyle::field(label, status)); + Json state = conv_.store().load_turn_state(conv_.profile(), conv_.bot(), conv_.ctx_id()); + const Json* open = state.find("open"); + bool is_open = open && ((open->is_bool() && open->as_bool()) || (open->is_number() && open->as_double() != 0)); + if (is_open) say(Term::paint("interrupted turn available, restart with --continue to pick it up", Term::kYellow)); + say(Term::paint("type /help for commands", Term::kDim)); +} + +std::string Repl::resume_turn() { + Json state = conv_.store().load_turn_state(conv_.profile(), conv_.bot(), conv_.ctx_id()); + const Json* open = state.find("open"); + bool is_open = open && ((open->is_bool() && open->as_bool()) || (open->is_number() && open->as_double() != 0)); + if (!is_open) { + say("nothing to resume: last turn for " + conv_.profile() + "/" + conv_.bot() + " completed normally"); + return ""; + } + const Json* g = state.find("goal"); + std::string goal = (g && g->is_string()) ? g->as_string() : ""; + const Json* steps = state.find("steps"); + const Json* tools = state.find("tools"); + say("resuming interrupted turn for " + conv_.profile() + "/" + conv_.bot() + ": " + std::to_string(steps ? steps->as_int() : 0) + " steps, " + + std::to_string(tools ? tools->as_int() : 0) + " tool calls, goal: " + Utf8::prefix(goal, 200)); + std::string message = "Continue where you left off" + (goal.empty() ? std::string() : ": the interrupted request was: " + goal) + + ". Do not repeat steps already completed above; carry on with the next step."; + return host_.run_turn(message); +} + +bool Repl::handle_command(const std::string& text) { + auto [first, remainder] = Utf8::split_first(text.substr(1)); + if (first.empty()) { say(red("unknown command, try /help")); return true; } + std::string name = lower(first); + std::string arg = Utf8::strip(remainder); + if (name == "quit" || name == "exit") return false; + if (name == "help") { + say(Prompts::HelpText()); + } else if (name == "profiles") { + auto names = conv_.store().list_profiles(); + for (auto& item : names) say(std::string(item == conv_.profile() ? "*" : " ") + " " + item); + if (names.empty()) say("(no profiles)"); + } else if (name == "bots") { + for (auto& item : conv_.store().list_bots(conv_.profile())) { + std::string nicks; + const Json* n = item.find("nicknames"); + if (n && n->is_array() && n->size() > 0) { + std::string joined; + for (auto& x : n->items()) { if (!joined.empty()) joined += ", "; joined += x.py_str(); } + nicks = " aka " + joined; + } + say(std::string(item.get_string("name") == conv_.bot() ? "*" : " ") + " " + item.get_string("name") + nicks); + } + } else if (name == "bot") { + if (arg.empty()) say(red("use /bot , see /bots")); + else say(conv_.switch_bot(arg)); + } else if (name == "profile") { + if (arg.empty()) say("current profile: " + conv_.profile() + " (" + std::to_string(Utf8::length(conv_.system_message())) + " chars)"); + else if (!store::Patterns::profile(arg)) say(red("invalid profile name")); + else conv_.switch_profile(arg); + } else if (name == "ctx") { + say(session::ShellContext::report(conv_.store(), conv_.ctx_id())); + } else if (name == "clear") { + conv_.reset_history(); + say("history cleared"); + } else { + auto handled = host_.handle_command(name, arg); + if (!handled) say(red("unknown command, try /help")); + else if (!*handled) return false; + } + return true; +} + +void Repl::loop() { + Interrupt::install(); + conv_.store().history_load(conv_.profile()); // readline.add_history source; also imports the old history file once + host_.on_start(); + try { + while (true) { + int busy = host_.running_agents(); + std::string status = ReplStyle::status_line(options_.version, conv_.profile(), session::ShellContext::dir_key(), conv_.yolo(), busy); + if (!status.empty()) say(status); + std::string line; + auto got = input_->read(ReplStyle::prompt(conv_.profile(), conv_.yolo(), busy), line); + if (got == LineInput::Status::Eof) { std::printf("\n"); break; } + if (got == LineInput::Status::Interrupted) { std::printf("\n"); continue; } + std::string text = Utf8::strip(line); + if (text.empty()) continue; + conv_.store().history_add(conv_.profile(), conv_.bot(), text); + if (text[0] == '/') { + if (!handle_command(text)) break; + continue; + } + try { + host_.run_turn(text); + if (Interrupt::requested()) throw TurnInterrupted(); + } catch (const TurnInterrupted&) { + Interrupt::clear(); + say(Term::paint("\ninterrupted", Term::kYellow)); + conv_.checkpoint(); + } + if (conv_.persist()) conv_.save(); + } + } catch (...) { + host_.on_stop(); + conv_.save(); + throw; + } + host_.on_stop(); + conv_.save(); +} + +int Repl::run() { + int code = 0; + try { + if (options_.resume && !options_.telegram && !options_.scheduler) resume_turn(); + banner(); + loop(); + } catch (...) { + conv_.store().close(); + throw; + } + conv_.store().close(); + return code; +} + +int Repl::oneshot(const std::string& prompt) { + Interrupt::install(); + bool verified = false; + int code = 0; + try { + std::string final = host_.run_turn(prompt); + verified = host_.ensure_verified(prompt, final); + code = verified ? 0 : 1; + } catch (const TurnInterrupted&) { + say(Term::paint("\ninterrupted", Term::kYellow)); + code = 130; + } + try { conv_.checkpoint(); } catch (...) {} + try { conv_.store().close(); } catch (...) {} + return code; +} + +} // namespace detz::repl diff --git a/taicpp/src/repl/repl.h b/taicpp/src/repl/repl.h new file mode 100644 index 0000000..34fc21e --- /dev/null +++ b/taicpp/src/repl/repl.h @@ -0,0 +1,41 @@ +#pragma once +#include +#include "line_input.h" +#include "repl_host.h" +#include "session/conversation.h" +namespace detz::repl { + +// tai.py banner(), resume_turn(), repl() and the one-shot tail of main(): +// loads the history that Conversation restored for this shell/pane context, +// reads lines, records input history, runs turns and saves the session after +// every turn and again on the way out. +class Repl { + public: + struct Options { + std::string version; + bool resume{false}; // --continue + bool telegram{false}; + bool scheduler{false}; + }; + Repl(session::Conversation& conversation, ReplHost& host, Options options, LineInput* input = nullptr); + + // main()'s tail: resume (unless telegram or scheduler), banner, loop. Returns the exit code. + int run(); + void banner(); + // "" when nothing to resume + std::string resume_turn(); + void loop(); + // handle_command(agent, text): false leaves the loop + bool handle_command(const std::string& text); + // one-shot prompt: run, ensure_verified, checkpoint; returns the process exit code (0 verified, 1 not) + int oneshot(const std::string& prompt); + + private: + session::Conversation& conv_; + ReplHost& host_; + Options options_; + LineInput owned_input_; + LineInput* input_; +}; + +} // namespace detz::repl diff --git a/taicpp/src/repl/repl_host.h b/taicpp/src/repl/repl_host.h new file mode 100644 index 0000000..c65ee39 --- /dev/null +++ b/taicpp/src/repl/repl_host.h @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include +#include +#include +namespace detz::repl { + +// python KeyboardInterrupt out of a turn: the REPL prints "interrupted" and checkpoints. +class TurnInterrupted : public std::exception { + public: + const char* what() const noexcept override { return "interrupted"; } +}; + +// What the REPL needs from the rest of the program: the model loop and the +// commands that live outside the session layer. The agent module implements +// this; tests implement it with a scripted host. Messages are appended to the +// Conversation by the host (user, assistant, tool); the REPL saves them. +class ReplHost { + public: + virtual ~ReplHost() = default; + // Agent.run_turn(text): runs the turn, returns the final text + virtual std::string run_turn(const std::string& text) = 0; + // Agent.ensure_verified(prompt, final) for the one-shot path + virtual bool ensure_verified(const std::string& /*prompt*/, const std::string& /*final*/) { return true; } + // Slash commands the session layer does not own (/env, /skills, /secret, /models, ...). + // nullopt = unknown command; false = leave the REPL; true = keep going. + virtual std::optional handle_command(const std::string& /*name*/, const std::string& /*arg*/) { return std::nullopt; } + // running_agents(): background subagents shown as "+N" in the prompt + virtual int running_agents() { return 0; } + // start_scheduler / scheduler_stop.set() + virtual void on_start() {} + virtual void on_stop() {} + // probe_backend lines of the banner: (label, status) + virtual std::vector> backend_status() { return {}; } + // complete_input tab completion is not available without readline; kept for the line editor + virtual std::vector complete(const std::string& /*text*/) { return {}; } +}; + +} // namespace detz::repl diff --git a/taicpp/src/repl/repl_style.cpp b/taicpp/src/repl/repl_style.cpp new file mode 100644 index 0000000..5e8f73f --- /dev/null +++ b/taicpp/src/repl/repl_style.cpp @@ -0,0 +1,151 @@ +#include "repl_style.h" +#include +#include +#include +#include +#include +#include "session/term.h" +#include "store/utf8.h" +namespace detz::repl { + +using session::Term; +using store::Utf8; + +bool ReplStyle::glow() { return Term::color_enabled(); } + +std::string ReplStyle::glow_title(const std::string& version, const std::string& profile) { + return std::string(Term::kBold) + "\xe2\x97\x86" + Term::kReset + " " + Term::kBold + "tai" + Term::kReset + " " + kFgFile + version + Term::kReset + " " + Term::kDim + "\xc2\xb7 profile " + profile + Term::kReset; +} + +std::string ReplStyle::title(const std::string& version, const std::string& profile) { + if (glow()) return glow_title(version, profile); + return Term::paint("tai " + version + " \xc2\xb7 profile " + profile, std::string(Term::kBold) + Term::kCyan); +} + +std::string ReplStyle::glow_prompt() { return std::string(Term::kBold) + kFgOrange + "\xe2\x80\xba " + Term::kReset; } + +std::string ReplStyle::field(const std::string& label, const std::string& value) { + if (glow()) return std::string(" ") + Term::kDim + label + ":" + Term::kReset + " " + value; + return " " + label + ": " + value; +} + +size_t ReplStyle::terminal_columns(size_t fallback) { + // shutil.get_terminal_size: COLUMNS first, then the stdout tty, then the fallback + if (const char* env = std::getenv("COLUMNS")) { + char* end = nullptr; + long v = std::strtol(env, &end, 10); + if (end != env && *end == '\0' && v > 0) return static_cast(v); + } + struct winsize ws; + if (::ioctl(1, TIOCGWINSZ, &ws) == 0 && ws.ws_col > 0) return ws.ws_col; + return fallback; +} + +std::string ReplStyle::strip_ansi(const std::string& text) { + std::string out; + size_t i = 0; + while (i < text.size()) { + if (text[i] == '\033' && i + 1 < text.size() && text[i + 1] == '[') { + size_t j = i + 2; + while (j < text.size() && ((text[j] >= '0' && text[j] <= '9') || text[j] == ';')) j++; + if (j < text.size() && text[j] == 'm') { i = j + 1; continue; } + } + out += text[i++]; + } + return out; +} + +std::string ReplStyle::truncate_ansi(const std::string& text, size_t width) { + std::string plain = strip_ansi(text); + if (Utf8::length(plain) <= width) return text; + std::string out; + size_t count = 0, pos = 0; + // walk the escape sequences, copying the text between them up to `width` code points + size_t i = 0; + bool stopped = false; + while (i < text.size()) { + size_t esc = std::string::npos, esc_end = 0; + for (size_t k = i; k < text.size(); k++) { + if (text[k] == '\033' && k + 1 < text.size() && text[k + 1] == '[') { + size_t j = k + 2; + while (j < text.size() && ((text[j] >= '0' && text[j] <= '9') || text[j] == ';')) j++; + if (j < text.size() && text[j] == 'm') { esc = k; esc_end = j + 1; break; } + } + } + if (esc == std::string::npos) break; + std::string chunk = text.substr(i, esc - i); + size_t take = std::min(Utf8::length(chunk), width - count); + out += Utf8::prefix(chunk, take); + count += take; + if (count >= width) { pos = esc; stopped = true; break; } + out += text.substr(esc, esc_end - esc); + i = esc_end; + pos = esc_end; + } + if (!stopped && count < width) { + std::string rest = strip_ansi(text.substr(pos)); + out += Utf8::prefix(rest, width - count); + } + return out + Term::kReset; +} + +std::string ReplStyle::status_line(const std::string& version, const std::string& profile, const std::string& directory, bool yolo, int busy) { + if (!glow()) return ""; + return statusbar(version, profile, directory, yolo, busy, 0); +} + +std::string ReplStyle::statusbar(const std::string& version, const std::string& profile, const std::string& directory, bool yolo, int busy, size_t width) { + const char* home_env = std::getenv("HOME"); + std::string home = home_env ? home_env : ""; + std::string shortdir = directory; + if (directory == home) shortdir = "~"; + else if (!home.empty() && directory.rfind(home + "/", 0) == 0) shortdir = "~" + directory.substr(home.size()); + std::vector segs = { + std::string(Term::kBold) + kFgFile + "tai " + version + Term::kReset, + std::string(kFgFile) + profile + Term::kReset, + std::string(Term::kDim) + shortdir + Term::kReset, + }; + if (yolo) segs.push_back(std::string(Term::kBold) + kFgYolo + "YOLO" + Term::kReset); + if (busy) segs.push_back(std::string(Term::kYellow) + "+" + std::to_string(busy) + Term::kReset); + std::string dot = std::string(Term::kDim) + "\xc2\xb7" + Term::kReset; + auto join = [&](const std::vector& v) { + std::string out; + for (size_t k = 0; k < v.size(); k++) { if (k) out += dot; out += " " + v[k] + " "; } + return Utf8::strip(out); + }; + if (width == 0) width = terminal_columns(); + std::string flow = join(segs); + if ((yolo || busy) && segs.size() > 3) { + std::string left = join({segs[0], segs[1], segs[2]}); + std::string right = join(std::vector(segs.begin() + 3, segs.end())); + long gap = static_cast(width) - static_cast(Utf8::length(strip_ansi(left))) - static_cast(Utf8::length(strip_ansi(right))); + if (gap >= 4) return left + std::string(static_cast(gap), ' ') + right; + std::vector keep = {segs[0], segs[1]}; + keep.insert(keep.end(), segs.begin() + 3, segs.end()); + flow = join(keep); + } + if (Utf8::length(strip_ansi(flow)) <= width) return flow; + std::string core = truncate_ansi(flow, width); + // re.sub(r"(?:\033\[[0-9;]*m|[ ·])+$", "", core) + while (true) { + if (!core.empty() && core.back() == 'm') { + size_t j = core.size() - 1; + size_t k = j; + while (k > 0 && ((core[k - 1] >= '0' && core[k - 1] <= '9') || core[k - 1] == ';')) k--; + if (k >= 2 && core[k - 1] == '[' && core[k - 2] == '\033') { core.erase(k - 2); continue; } + } + if (!core.empty() && core.back() == ' ') { core.pop_back(); continue; } + if (core.size() >= 2 && static_cast(core[core.size() - 2]) == 0xC2 && static_cast(core.back()) == 0xB7) { core.erase(core.size() - 2); continue; } + break; + } + return core + Term::kReset; +} + +std::string ReplStyle::prompt(const std::string& profile, bool yolo, int busy) { + if (glow()) return glow_prompt(); + std::string base = profile + (yolo ? "!" : ""); + std::string tag = busy ? base + "+" + std::to_string(busy) : base; + return Term::paint("tai[" + tag + "]\xe2\x80\xba ", std::string(Term::kBold) + Term::kGreen); +} + +} // namespace detz::repl diff --git a/taicpp/src/repl/repl_style.h b/taicpp/src/repl/repl_style.h new file mode 100644 index 0000000..83fe748 --- /dev/null +++ b/taicpp/src/repl/repl_style.h @@ -0,0 +1,33 @@ +#pragma once +#include +namespace detz::repl { + +// How the REPL looks: the plain format (pipes, NO_COLOR, tests) byte for +// byte, and the Glow format on a color terminal (banner diamond, status bar, +// orange prompt). tai.py gates on Glow.enabled() at every call, so does this. +class ReplStyle { + public: + static constexpr const char* kFgFile = "\033[38;5;111m"; + static constexpr const char* kFgYolo = "\033[38;5;211m"; + static constexpr const char* kFgOrange = "\033[38;5;214m"; + + static bool glow(); + static std::string glow_title(const std::string& version, const std::string& profile); + static std::string glow_prompt(); + // "tai VERSION · profile NAME" title line + static std::string title(const std::string& version, const std::string& profile); + // " label: value" detail line + static std::string field(const std::string& label, const std::string& value); + // status bar printed above the prompt; "" in plain mode + static std::string status_line(const std::string& version, const std::string& profile, const std::string& directory, bool yolo, int busy); + // Glow.statusbar(version, profile, directory, yolo, agents, width); width 0 = terminal width + static std::string statusbar(const std::string& version, const std::string& profile, const std::string& directory, bool yolo, int busy, size_t width); + static std::string prompt(const std::string& profile, bool yolo, int busy); + + // helpers shared with tests + static std::string strip_ansi(const std::string& text); + static std::string truncate_ansi(const std::string& text, size_t width); + static size_t terminal_columns(size_t fallback = 80); +}; + +} // namespace detz::repl diff --git a/taicpp/src/session/conversation.cpp b/taicpp/src/session/conversation.cpp new file mode 100644 index 0000000..5098921 --- /dev/null +++ b/taicpp/src/session/conversation.cpp @@ -0,0 +1,153 @@ +#include "conversation.h" +#include +#include "config/prompts.h" +#include "shell_context.h" +#include "store/clock.h" +#include "store/utf8.h" +#include "term.h" +namespace detz::session { + +using store::Json; + +namespace { +Json message(const std::string& role, const std::string& content) { + Json m = Json::object(); + m.set("role", role); + m.set("content", content); + return m; +} +std::string replace_all(std::string text, const std::string& from, const std::string& to) { + size_t pos = 0; + while ((pos = text.find(from, pos)) != std::string::npos) { text.replace(pos, from.size(), to); pos += to.size(); } + return text; +} +} // namespace + +Conversation::Conversation(store::Store& st, const Config& config, bool persist, bool yolo, bool autonomous, const std::string& ctx, Catalogs catalogs) + : store_(st), config_(config), catalogs_(std::move(catalogs)), persist_(persist), yolo_(yolo), auto_(autonomous), ctx_id_(ShellContext::sanitize(ctx)) { + switch_profile(config_.profile, true); +} + +void Conversation::apply_system() { + std::string content = system_message_; + if (catalogs_.skill_catalog) content += catalogs_.skill_catalog(); + if (catalogs_.tool_catalog) content += catalogs_.tool_catalog(); + content += Prompts::OrientationBlock("", env == "sandbox"); + content += Prompts::CapabilityBlock(yolo_, auto_); + if (auto_) content += "\n\n" + Prompts::AutonomousNote(); + messages_[0] = message("system", content); +} + +void Conversation::reset_history() { + messages_.clear(); + messages_.push_back(message("system", system_message_)); + apply_system(); + apply_system(); +} + +void Conversation::switch_profile(const std::string& name, bool silent) { + if (!messages_.empty() && persist_) store_.save_session(profile_, messages_, bot_, ctx_id_); + profile_ = name; + bot_ = "main"; + store_.profile = name; + if (catalogs_.profile_reset) catalogs_.profile_reset(); + auto loaded = store_.load_system(name); + std::string text; + if (!loaded) { + text = Prompts::DefaultSystem(); + store_.save_system(name, text); + } else { + text = *loaded; + } + if (text.find(Prompts::LegacyPasswordNote()) != std::string::npos) { + text = replace_all(text, Prompts::LegacyPasswordNote(), Prompts::VaultMemoryNote()); + store_.save_system(name, text); + } + system_message_ = text; + std::vector restored = persist_ ? store_.load_session(name, "main", ctx_id_) : std::vector(); + messages_.clear(); + messages_.push_back(message("system", system_message_)); + messages_.insert(messages_.end(), restored.begin(), restored.end()); + apply_system(); + if (catalogs_.scope_changed) catalogs_.scope_changed(); + if (!silent) { + std::printf("%s\n", Term::paint("profile: " + name + " (" + std::to_string(store::Utf8::length(system_message_)) + " chars knowledge, " + std::to_string(restored.size()) + " restored messages)", Term::kGreen).c_str()); + } +} + +std::string Conversation::switch_bot(const std::string& name, bool silent) { + auto target = store_.resolve_bot(profile_, name); + if (!target) { + std::string avail; + for (auto& b : store_.list_bots(profile_)) { if (!avail.empty()) avail += ", "; avail += b.get_string("name"); } + return "unknown bot @" + name + " (available: " + avail + ")"; + } + if (!messages_.empty() && persist_) store_.save_session(profile_, messages_, bot_, ctx_id_); + bot_ = *target; + auto loaded = store_.load_bot_system(profile_, *target); + std::string text; + if (!loaded) { + text = Prompts::DefaultSystem(); + if (*target == "main") store_.save_system(profile_, text); + else store_.save_bot_system(profile_, *target, text); + } else { + text = *loaded; + } + system_message_ = text; + std::vector restored = persist_ ? store_.load_session(profile_, *target, ctx_id_) : std::vector(); + messages_.clear(); + messages_.push_back(message("system", system_message_)); + messages_.insert(messages_.end(), restored.begin(), restored.end()); + apply_system(); + if (catalogs_.scope_changed) catalogs_.scope_changed(); + if (!silent) std::printf("%s\n", Term::paint("bot: " + *target + " (" + std::to_string(restored.size()) + " restored messages)", Term::kGreen).c_str()); + return "switched to bot '" + *target + "'"; +} + +void Conversation::update_system(const std::string& text) { + system_message_ = text; + if (bot_ == "main") store_.save_system(profile_, text); + else store_.save_bot_system(profile_, bot_, text); + apply_system(); + store_.log_event(profile_, "system", "remember", "system message updated (" + std::to_string(store::Utf8::length(text)) + " chars)"); +} + +Json Conversation::turn_state(bool open, const std::string& goal) const { + Json s = Json::object(); + s.set("open", open); + s.set("goal", goal); + s.set("steps", turn_steps_); + s.set("tools", turn_tools_); + s.set("updated", store::Clock::now_iso()); + return s; +} + +void Conversation::begin_turn(const std::string& goal) { + turn_open_ = true; + turn_goal_ = goal; + turn_steps_ = 0; + turn_tools_ = 0; +} + +void Conversation::checkpoint() { + if (!persist_) return; + store_.save_session(profile_, messages_, bot_, ctx_id_); + if (turn_open_) store_.save_turn_state(profile_, bot_, turn_state(true, turn_goal_), ctx_id_); +} + +void Conversation::close_turn() { + if (!persist_) { turn_open_ = false; return; } + store_.save_session(profile_, messages_, bot_, ctx_id_); + turn_open_ = false; + if (!turn_goal_.empty()) store_.save_turn_state(profile_, bot_, turn_state(false, turn_goal_), ctx_id_); +} + +void Conversation::mark_turn_open(const std::string& goal) { + if (persist_) store_.save_turn_state(profile_, bot_, turn_state(true, store::Utf8::prefix(goal, 500)), ctx_id_); +} + +void Conversation::save() { store_.save_session(profile_, messages_, bot_, ctx_id_); } + +void Conversation::add_message(const std::string& role, const std::string& content) { messages_.push_back(message(role, content)); } + +} // namespace detz::session diff --git a/taicpp/src/session/conversation.h b/taicpp/src/session/conversation.h new file mode 100644 index 0000000..6eef1b9 --- /dev/null +++ b/taicpp/src/session/conversation.h @@ -0,0 +1,81 @@ +#pragma once +#include +#include +#include +#include "config/config.h" +#include "store/json.h" +#include "store/store.h" +namespace detz::session { + +// The persistence half of tai.py class Agent: which profile and bot are +// active, the message list ([0] is the system message), and when the list is +// written to the vault (switch, checkpoint, close_turn). The model loop lives +// elsewhere and appends to messages(); every restore and save here is keyed by +// the ShellContext id (ctx_id), i.e. by the shell, hence by the pane. +class Conversation { + public: + using Json = store::Json; + // Pieces of the system message owned by other modules (skills, tool census). + struct Catalogs { + std::function skill_catalog; // skill_catalog(skills) + std::function tool_catalog; // tool_catalog() + std::function scope_changed; // sync_chat_scope(): chat.scope = (profile, bot) + std::function profile_reset; // tools.read_files.clear(), secret_grants.clear() + }; + + Conversation(store::Store& store, const Config& config, bool persist = true, bool yolo = false, bool autonomous = false, + const std::string& ctx_id = "", Catalogs catalogs = {}); + + store::Store& store() { return store_; } + const Config& config() const { return config_; } + const std::string& profile() const { return profile_; } + const std::string& bot() const { return bot_; } + const std::string& ctx_id() const { return ctx_id_; } + const std::string& system_message() const { return system_message_; } + std::vector& messages() { return messages_; } + const std::vector& messages() const { return messages_; } + bool persist() const { return persist_; } + bool yolo() const { return yolo_; } + bool autonomous() const { return auto_; } + std::string env{"home"}; + + void apply_system(); + void reset_history(); + // restored message count; prints the green status line unless silent + void switch_profile(const std::string& name, bool silent = false); + // "switched to bot 'x'" or the unknown-bot message + std::string switch_bot(const std::string& name, bool silent = false); + void update_system(const std::string& text); + + // turn bookkeeping (Agent._turn_*) + void begin_turn(const std::string& goal); + void note_step() { turn_steps_++; } + void note_tool() { turn_tools_++; } + bool turn_open() const { return turn_open_; } + void checkpoint(); + void close_turn(); + void mark_turn_open(const std::string& goal); + // save_session for the current profile/bot (the REPL's final write) + void save(); + + void add_message(const std::string& role, const std::string& content); + + private: + Json turn_state(bool open, const std::string& goal) const; + store::Store& store_; + Config config_; + Catalogs catalogs_; + bool persist_; + bool yolo_, auto_; + std::string ctx_id_; + std::string profile_; + std::string bot_{"main"}; + std::string system_message_; + std::vector messages_; + bool turn_open_{false}; + std::string turn_goal_; + int turn_steps_{0}; + int turn_tools_{0}; +}; + +} // namespace detz::session diff --git a/taicpp/src/session/dir_defaults.cpp b/taicpp/src/session/dir_defaults.cpp new file mode 100644 index 0000000..5997d72 --- /dev/null +++ b/taicpp/src/session/dir_defaults.cpp @@ -0,0 +1,36 @@ +#include "dir_defaults.h" +#include +#include "shell_context.h" +#include "term.h" +namespace detz::session { + +std::string DirDefaults::apply(Conversation& conv, const Hooks& hooks, const std::string& directory) { + std::string key = ShellContext::dir_key(directory); + auto saved = conv.store().load_dir_defaults(key); + if (saved.empty()) return ""; + std::vector bits; + auto has = [&](const char* k) { auto it = saved.find(k); return it != saved.end() && !it->second.empty(); }; + if (has("provider")) { + std::set offered = hooks.offered_providers ? hooks.offered_providers() : std::set(); + if (offered.count(saved["provider"])) { + if (hooks.pin_provider) hooks.pin_provider(saved["provider"]); + bits.push_back("provider=" + saved["provider"]); + } else { + std::fprintf(stderr, "%s\n", Term::paint("dir default provider '" + saved["provider"] + "' is not available here, ignoring it (@auto clears it)", Term::kDim).c_str()); + } + } + if (has("model") && conv.config().explicitModel.empty()) { + if (hooks.pin_model) hooks.pin_model(saved["model"]); + bits.push_back("model=" + saved["model"]); + } + if (has("bot") && saved["bot"] != conv.bot()) { + std::string switched = conv.switch_bot(saved["bot"], true); + if (switched.rfind("switched", 0) == 0) bits.push_back("bot=" + saved["bot"]); + } + if (bits.empty()) return ""; + std::string out = "dir defaults for " + key + ":"; + for (auto& b : bits) out += " " + b; + return out; +} + +} // namespace detz::session diff --git a/taicpp/src/session/dir_defaults.h b/taicpp/src/session/dir_defaults.h new file mode 100644 index 0000000..eeab9b3 --- /dev/null +++ b/taicpp/src/session/dir_defaults.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include +#include +#include "conversation.h" +namespace detz::session { + +// apply_dir_defaults of tai.py: per-directory provider, model and bot pins +// stored by save_dir_defaults. The chat client lives in the llm module, so +// the provider and model pins are applied through hooks. +class DirDefaults { + public: + struct Hooks { + std::function()> offered_providers; // lower-cased provider labels the chat client offers + std::function pin_provider; // chat.provider_pin + std::function pin_model; // chat.model_pin + }; + // The one-line summary ("dir defaults for : provider=x model=y bot=z"), "" when nothing applied. + static std::string apply(Conversation& conversation, const Hooks& hooks, const std::string& directory = ""); +}; + +} // namespace detz::session diff --git a/taicpp/src/session/md5.cpp b/taicpp/src/session/md5.cpp new file mode 100644 index 0000000..cdaba79 --- /dev/null +++ b/taicpp/src/session/md5.cpp @@ -0,0 +1,55 @@ +#include "md5.h" +#include +#include +namespace detz::session { + +static inline uint32_t rol(uint32_t x, int c) { return (x << c) | (x >> (32 - c)); } + +std::string Md5::hex(const std::string& data) { + static const uint32_t K[64] = { + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391}; + static const int S[64] = {7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21}; + uint32_t a0 = 0x67452301, b0 = 0xefcdab89, c0 = 0x98badcfe, d0 = 0x10325476; + std::string msg = data; + uint64_t bits = static_cast(data.size()) * 8; + msg += static_cast(0x80); + while (msg.size() % 64 != 56) msg += '\0'; + for (int k = 0; k < 8; k++) msg += static_cast((bits >> (k * 8)) & 0xff); + for (size_t off = 0; off < msg.size(); off += 64) { + uint32_t M[16]; + for (int i = 0; i < 16; i++) { + size_t b = off + static_cast(i) * 4; + M[i] = static_cast(static_cast(msg[b])) | (static_cast(static_cast(msg[b + 1])) << 8) | + (static_cast(static_cast(msg[b + 2])) << 16) | (static_cast(static_cast(msg[b + 3])) << 24); + } + uint32_t A = a0, B = b0, C = c0, D = d0; + for (int i = 0; i < 64; i++) { + uint32_t F; + int g; + if (i < 16) { F = (B & C) | (~B & D); g = i; } + else if (i < 32) { F = (D & B) | (~D & C); g = (5 * i + 1) % 16; } + else if (i < 48) { F = B ^ C ^ D; g = (3 * i + 5) % 16; } + else { F = C ^ (B | ~D); g = (7 * i) % 16; } + F = F + A + K[i] + M[g]; + A = D; D = C; C = B; + B = B + rol(F, S[i]); + } + a0 += A; b0 += B; c0 += C; d0 += D; + } + std::string out; + char buf[4]; + for (uint32_t v : {a0, b0, c0, d0}) { + for (int k = 0; k < 4; k++) { std::snprintf(buf, sizeof buf, "%02x", (v >> (k * 8)) & 0xff); out += buf; } + } + return out; +} + +} // namespace detz::session diff --git a/taicpp/src/session/md5.h b/taicpp/src/session/md5.h new file mode 100644 index 0000000..6b0eaab --- /dev/null +++ b/taicpp/src/session/md5.h @@ -0,0 +1,11 @@ +#pragma once +#include +namespace detz::session { + +// MD5 (RFC 1321), only for the boot-tag fallback hashlib.md5(...).hexdigest(). +class Md5 { + public: + static std::string hex(const std::string& data); +}; + +} // namespace detz::session diff --git a/taicpp/src/session/shell_context.cpp b/taicpp/src/session/shell_context.cpp new file mode 100644 index 0000000..7c7363a --- /dev/null +++ b/taicpp/src/session/shell_context.cpp @@ -0,0 +1,393 @@ +#include "shell_context.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "md5.h" +#include "store/clock.h" +#include "store/json.h" +#include "store/store.h" +#include "store/utf8.h" +namespace detz::session { + +using store::Cell; +using store::Json; + +static const std::set kShells = {"bash", "zsh", "fish", "sh", "dash", "ksh", "tcsh", "nu"}; + +std::string ShellContext::sanitize(const std::string& value) { + std::string out; + for (char c : value) { + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_' || c == '.' || c == '-') out += c; + } + if (out.size() > static_cast(kSidMax)) out.resize(static_cast(kSidMax)); + return out; +} + +bool ShellContext::is_shell(const std::string& name) { return kShells.count(name) > 0; } + +std::string ShellContext::read_text(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) return ""; + std::string raw; + char buf[4096]; + in.read(buf, sizeof buf); + raw.assign(buf, static_cast(in.gcount())); + // read(4096) counts characters, so decode to code points before cutting + raw = store::Utf8::sanitize(raw); + return store::Utf8::prefix(raw, 4096); +} + +std::optional ShellContext::parse_stat(const std::string& text) { + size_t close = text.rfind(')'); + if (close == std::string::npos || close == 0) return std::nullopt; + std::string head = text.substr(0, close); + size_t open = head.find('('); + if (open == std::string::npos) return std::nullopt; + std::string comm = head.substr(open + 1); + auto fields = store::Utf8::split_ws(text.substr(close + 1)); + if (fields.size() < 20) return std::nullopt; + auto to_int = [](const std::string& s, long& out) { + if (s.empty()) return false; + size_t i = 0; + if (s[0] == '-' || s[0] == '+') i = 1; + if (i >= s.size()) return false; + for (size_t k = i; k < s.size(); k++) if (s[k] < '0' || s[k] > '9') return false; + out = std::strtol(s.c_str(), nullptr, 10); + return true; + }; + long ppid, ticks; + if (!to_int(fields[1], ppid) || !to_int(fields[19], ticks)) return std::nullopt; + return Stat{comm, ppid, ticks}; +} + +std::optional ShellContext::boot_time(const std::string& proc_root) { + std::ifstream in(proc_root + "/stat"); + if (!in) return std::nullopt; + std::string line; + while (std::getline(in, line)) { + if (line.rfind("btime ", 0) == 0) { + auto parts = store::Utf8::split_ws(line); + if (parts.size() < 2) return std::nullopt; + char* end = nullptr; + double v = std::strtod(parts[1].c_str(), &end); + if (end == parts[1].c_str() || *end != '\0') return std::nullopt; + return v; + } + } + return std::nullopt; +} + +long ShellContext::clk_tick() { + long v = ::sysconf(_SC_CLK_TCK); + return v > 0 ? v : 100; +} + +std::string ShellContext::boot_tag(const std::string& proc_root) { + std::string text = read_text(proc_root + "/sys/kernel/random/boot_id"); + std::string digest; + for (char c : text) { + char l = (c >= 'A' && c <= 'F') ? static_cast(c + 32) : c; + if ((l >= '0' && l <= '9') || (l >= 'a' && l <= 'f')) digest += l; + } + if (digest.size() > 8) digest.resize(8); + if (!digest.empty()) return digest; + auto bt = boot_time(proc_root); + if (bt && *bt != 0.0) return Md5::hex(Json::float_repr(*bt)).substr(0, 8); + return "deadbeef"; +} + +std::string ShellContext::shell_name(const std::string& argv0, const std::string& comm) { + auto lower = [](std::string s) { + for (auto& c : s) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + return s; + }; + std::string first = argv0.substr(0, argv0.find('\0')); + size_t slash = first.rfind('/'); + std::string base = slash == std::string::npos ? first : first.substr(slash + 1); + size_t a = 0; + while (a < base.size() && base[a] == '-') a++; + std::string c1 = lower(base.substr(a)); + a = 0; + while (a < comm.size() && comm[a] == '-') a++; + std::string c2 = lower(comm.substr(a)); + if (is_shell(c1)) return c1; + if (is_shell(c2)) return c2; + return ""; +} + +static std::string iso_utc_seconds(double ts) { + double whole = std::floor(ts); + double us = std::nearbyint((ts - whole) * 1e6); + long long secs = static_cast(whole) + (us >= 1e6 ? 1 : 0); + std::time_t t = static_cast(secs); + std::tm tm{}; + gmtime_r(&t, &tm); + char b[96]; + std::snprintf(b, sizeof b, "%04d-%02d-%02dT%02d:%02d:%02d+00:00", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + return b; +} + +std::vector ShellContext::read_chain(long pid, const std::string& proc_root, std::optional btime, long tick, int cap) { + std::vector chain; + std::set seen; + long current = pid; + while (current != 0 && !seen.count(current) && static_cast(chain.size()) < cap) { + seen.insert(current); + auto parsed = parse_stat(read_text(proc_root + "/" + std::to_string(current) + "/stat")); + if (!parsed) break; + std::string cmd = read_text(proc_root + "/" + std::to_string(current) + "/cmdline"); + std::string first = cmd.substr(0, cmd.find('\0')); + size_t slash = first.rfind('/'); + std::string argv0 = store::Utf8::prefix(slash == std::string::npos ? first : first.substr(slash + 1), 64); + ChainEntry e; + e.pid = current; + e.ppid = parsed->ppid; + e.comm = store::Utf8::prefix(parsed->comm, 64); + e.argv0 = argv0; + e.name = shell_name(argv0, parsed->comm); + e.start = (btime && *btime != 0.0) ? (*btime + static_cast(parsed->ticks) / static_cast(tick)) : 0.0; + e.start_iso = e.start != 0.0 ? iso_utc_seconds(e.start) : ""; + chain.push_back(e); + if (parsed->ppid <= 0 || parsed->ppid == current) break; + current = parsed->ppid; + } + return chain; +} + +std::string ShellContext::shell_id(double start_ts, long pid, const std::string& boot) { + std::time_t t = static_cast(std::floor(start_ts)); + std::tm tm{}; + localtime_r(&t, &tm); + char b[32]; + std::strftime(b, sizeof b, "%Y%m%d-%H%M%S", &tm); + return std::string(b) + "-p" + std::to_string(pid) + "-" + boot; +} + +std::string ShellContext::ephemeral_id(double now_ts, long pid) { + std::time_t t = static_cast(std::floor(now_ts)); + std::tm tm{}; + localtime_r(&t, &tm); + char b[32]; + std::strftime(b, sizeof b, "%Y%m%d-%H%M%S", &tm); + return std::string(b) + "-e" + std::to_string(pid) + "-" + store::Clock::uuid_hex().substr(0, 4); +} + +// re.search(r"-p(\d+)-", raw) -> digits +static std::optional embedded_pid(const std::string& raw) { + for (size_t i = 0; i + 2 < raw.size(); i++) { + if (raw[i] == '-' && raw[i + 1] == 'p') { + size_t j = i + 2; + while (j < raw.size() && raw[j] >= '0' && raw[j] <= '9') j++; + if (j > i + 2 && j < raw.size() && raw[j] == '-') return std::strtol(raw.substr(i + 2, j - i - 2).c_str(), nullptr, 10); + } + } + return std::nullopt; +} + +ShellContext ShellContext::current(const std::map* env, long pid_in, const std::string& proc_root, double now_in, long clk_tck) { + std::string raw; + if (env) { + auto it = env->find(kEnv); + if (it != env->end()) raw = sanitize(it->second); + } else if (const char* v = std::getenv(kEnv)) { + raw = sanitize(v); + } + long pid = pid_in < 0 ? static_cast(::getpid()) : pid_in; + double now = now_in < 0 ? store::Clock::epoch() : now_in; + auto btime = boot_time(proc_root); + long tick = clk_tck ? clk_tck : clk_tick(); + std::string boot = boot_tag(proc_root); + auto chain = read_chain(pid, proc_root, btime, tick, kChainCap); + for (auto& e : chain) if (!e.name.empty() && e.start != 0.0) e.sid = shell_id(e.start, e.pid, boot); + std::vector shells; + for (auto& e : chain) if (!e.sid.empty()) shells.push_back(&e); + ChainEntry* inner = shells.empty() ? nullptr : shells[0]; + ChainEntry* outer = shells.size() > 1 ? shells[1] : nullptr; + std::optional outer_sid; + if (outer) outer_sid = outer->sid; + ShellContext c; + c.chain = chain; + if (!raw.empty()) { + auto embedded = embedded_pid(raw); + if (!embedded) { + c.sid = raw; c.shell = inner ? inner->name : ""; c.pid = inner ? inner->pid : pid; c.started = inner ? inner->start_iso : ""; + return c; + } + if (inner && *embedded == inner->pid) { + c.sid = raw; c.shell = inner->name; c.pid = inner->pid; c.started = inner->start_iso; c.parent = outer_sid; + return c; + } + c.sid = inner ? inner->sid : ephemeral_id(now, pid); + c.shell = inner ? inner->name : ""; + c.pid = inner ? inner->pid : pid; + c.started = inner ? inner->start_iso : iso_utc_seconds(now); + c.parent = raw; + return c; + } + if (inner) { + c.sid = inner->sid; c.shell = inner->name; c.pid = inner->pid; c.started = inner->start_iso; c.parent = outer_sid; + return c; + } + c.sid = ephemeral_id(now, pid); + c.pid = pid; + c.started = iso_utc_seconds(now); + return c; +} + +std::string ShellContext::dir_key(const std::string& directory) { + std::error_code ec; + std::filesystem::path p = directory.empty() ? std::filesystem::current_path(ec) : std::filesystem::path(directory); + if (ec) return ""; + auto r = std::filesystem::weakly_canonical(p, ec); + if (ec) return ""; + return r.string(); +} + +void ShellContext::export_env() const { ::setenv(kEnv, sid.c_str(), 1); } + +void ShellContext::register_in(store::Store& st) const { + try { + std::string stamp = store::Clock::now_iso(); + std::vector shells; + for (auto& e : chain) if (!e.sid.empty()) shells.push_back(&e); + for (size_t pos = 0; pos < shells.size(); pos++) { + Cell next = pos + 1 < shells.size() ? Cell(shells[pos + 1]->sid) : Cell(); + st.db().execute("INSERT OR IGNORE INTO shell_sessions (sid, parent, shell, pid, started) VALUES (?, ?, ?, ?, ?)", + {shells[pos]->sid, next, shells[pos]->name, static_cast(shells[pos]->pid), shells[pos]->start_iso}); + } + Json chain_json = Json::array(); + for (auto& e : chain) { + Json o = Json::object(); + o.set("pid", static_cast(e.pid)); o.set("ppid", static_cast(e.ppid)); o.set("comm", e.comm); o.set("argv0", e.argv0); + o.set("name", e.name); o.set("start", e.start); o.set("start_iso", e.start_iso); o.set("sid", e.sid); + chain_json.push(o); + } + std::string chain_s = chain_json.dumps(); + Cell par = parent ? Cell(*parent) : Cell(); + std::string cwd = dir_key(); + st.db().execute("INSERT OR IGNORE INTO shell_sessions (sid, parent, shell, pid, started, last_seen, cwd, chain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + {sid, par, shell, static_cast(pid), started, stamp, cwd, chain_s}); + st.db().execute("UPDATE shell_sessions SET last_seen = ?, cwd = ?, parent = COALESCE(parent, ?), chain = ? WHERE sid = ?", {stamp, cwd, par, chain_s, sid}); + st.db().execute("DELETE FROM shell_sessions WHERE sid NOT IN (SELECT sid FROM shell_sessions ORDER BY COALESCE(last_seen, started, '') DESC LIMIT ?)", {kRowsCap}); + st.db().commit(); + } catch (const std::exception&) { + } +} + +std::vector ShellContext::tree_lines(store::Store& st, const std::string& sid_in) { + struct Node { std::string sid; std::optional parent; std::string shell; std::optional pid; }; + std::vector rows; + try { + rows = st.db().execute("SELECT sid, parent, shell, pid FROM shell_sessions").rows; + } catch (const store::SqlError&) { + return {}; + } + std::map by_sid; + std::map> kids; + for (auto& r : rows) { + Node n{r[0].text(), r[1].is_null() ? std::nullopt : std::optional(r[1].text()), r[2].is_null() ? "" : r[2].text(), r[3].is_null() ? std::nullopt : std::optional(static_cast(r[3].as_int()))}; + by_sid[n.sid] = n; + if (n.parent && !n.parent->empty()) kids[*n.parent].push_back(n.sid); + } + for (auto& [k, v] : kids) std::sort(v.begin(), v.end()); + if (!by_sid.count(sid_in)) return {}; + auto label = [&](const std::string& key) { + auto it = by_sid.find(key); + if (it == by_sid.end()) return key + " (pruned)"; + const Node& n = it->second; + std::string shell = n.shell.empty() ? "?" : n.shell; + return key + " " + shell + ((n.pid && *n.pid) ? " pid " + std::to_string(*n.pid) : std::string()); + }; + std::vector upwards; + std::set seen{sid_in}; + std::string node = sid_in; + while (static_cast(upwards.size()) < kTreeDepth) { + auto it = by_sid.find(node); + std::string par = (it != by_sid.end() && it->second.parent) ? *it->second.parent : ""; + if (par.empty() || seen.count(par)) break; + seen.insert(par); + upwards.push_back(par); + node = par; + } + std::vector lines; + size_t depth = 0; + for (auto it = upwards.rbegin(); it != upwards.rend(); ++it, ++depth) lines.push_back(std::string(depth * 2, ' ') + label(*it)); + lines.push_back(std::string(upwards.size() * 2, ' ') + "* " + label(sid_in)); + std::function(const std::string&, int, std::set)> below = + [&](const std::string& key, int d, std::set seen_here) { + std::vector out; + if (d > kTreeDepth || static_cast(lines.size() + out.size()) >= kTreeLines) return out; + auto kit = kids.find(key); + if (kit == kids.end()) return out; + size_t cnt = 0; + for (auto& child : kit->second) { + if (cnt++ >= static_cast(kTreeKids)) break; + if (seen_here.count(child) || static_cast(lines.size() + out.size()) >= kTreeLines) continue; + out.push_back(std::string(static_cast(d) * 2, ' ') + label(child)); + auto sub_seen = seen_here; + sub_seen.insert(child); + for (auto& l : below(child, d + 1, sub_seen)) out.push_back(l); + } + return out; + }; + for (auto& l : below(sid_in, static_cast(upwards.size()) + 1, seen)) lines.push_back(l); + if (lines.size() > static_cast(kTreeLines)) lines.resize(static_cast(kTreeLines)); + return lines; +} + +std::string ShellContext::report(store::Store& st, const std::string& sid_in) { + std::vector lines; + try { + std::optional row; + if (!sid_in.empty()) { + auto cur = st.db().execute("SELECT sid, parent, shell, pid, started, last_seen, cwd FROM shell_sessions WHERE sid = ?", {sid_in}); + if (!cur.rows.empty()) row = cur.rows[0]; + } + auto s = [](const Cell& c, const char* fallback) { return (c.is_null() || c.text().empty() || c.text() == "0") ? std::string(fallback) : c.text(); }; + if (row) { + const store::Row& r = *row; + auto ss = [](const Cell& c, const char* fallback) { return (c.is_null() || c.text().empty()) ? std::string(fallback) : c.text(); }; + lines.push_back("ctx: " + r[0].text()); + lines.push_back("shell: " + ss(r[2], "?") + " (pid " + s(r[3], "?") + ", started " + ss(r[4], "?") + ")"); + lines.push_back("parent: " + ss(r[1], "(none)")); + lines.push_back("cwd: " + ss(r[6], "?")); + } else if (!sid_in.empty()) { + lines.push_back("ctx: " + sid_in + " (unregistered)"); + } else { + lines.push_back("ctx: (legacy, no session)"); + } + } catch (const store::SqlError&) { + return "ctx unavailable (vault error)"; + } + auto tree = tree_lines(st, sid_in); + if (!tree.empty()) { + lines.push_back("tree:"); + for (auto& l : tree) lines.push_back(l); + } + std::vector rows; + try { + rows = st.db().execute("SELECT sid, shell, last_seen FROM shell_sessions ORDER BY COALESCE(last_seen, started, '') DESC LIMIT 8").rows; + } catch (const store::SqlError&) { + rows.clear(); + } + if (!rows.empty()) { + lines.push_back("recent:"); + for (auto& r : rows) { + std::string shell = (r[1].is_null() || r[1].text().empty()) ? "?" : r[1].text(); + std::string seen = r[2].is_null() ? "" : r[2].text(); + lines.push_back(std::string(r[0].text() == sid_in ? "*" : " ") + " " + r[0].text() + " " + shell + " " + seen); + } + } + std::string out; + for (size_t k = 0; k < lines.size(); k++) { if (k) out += "\n"; out += lines[k]; } + return out; +} + +} // namespace detz::session diff --git a/taicpp/src/session/shell_context.h b/taicpp/src/session/shell_context.h new file mode 100644 index 0000000..f103c50 --- /dev/null +++ b/taicpp/src/session/shell_context.h @@ -0,0 +1,65 @@ +#pragma once +#include +#include +#include +#include +namespace detz::store { class Store; } +namespace detz::session { + +// tai.py class ShellSession ("one interactive shell is one conversation +// context"). Named ShellContext here because detz::shell::ShellSession is the +// command runner. The id is derived from the nearest shell ancestor +// (YYYYMMDD-HHMMSS-p-), so quitting and restarting tai inside the +// same shell, which is the same tmux or native-plexer pane, resolves to the +// same id and therefore the same session/turn files. TAI_SESSION pins it. +class ShellContext { + public: + static constexpr const char* kEnv = "TAI_SESSION"; + static constexpr int kSidMax = 64; + static constexpr int kChainCap = 64; + static constexpr int kFilesCap = 20; + static constexpr int kRowsCap = 1000; + static constexpr int kTreeDepth = 8; + static constexpr int kTreeKids = 10; + static constexpr int kTreeLines = 40; + + struct ChainEntry { + long pid{0}; + long ppid{0}; + std::string comm, argv0, name; + double start{0.0}; + std::string start_iso; + std::string sid; + }; + + std::string sid, shell; + long pid{0}; + std::string started; + std::optional parent; + std::vector chain; + + // re.sub(r"[^A-Za-z0-9_.-]", "", str(value or ""))[:SID_MAX] + static std::string sanitize(const std::string& value); + static bool is_shell(const std::string& name); + static std::string read_text(const std::string& path); // first 4096 characters, "" on error + struct Stat { std::string comm; long ppid; long ticks; }; + static std::optional parse_stat(const std::string& text); + static std::optional boot_time(const std::string& proc_root); + static long clk_tick(); + static std::string boot_tag(const std::string& proc_root); + static std::string shell_name(const std::string& argv0, const std::string& comm); + static std::vector read_chain(long pid, const std::string& proc_root, std::optional btime, long tick, int cap); + static std::string shell_id(double start_ts, long pid, const std::string& boot); + static std::string ephemeral_id(double now_ts, long pid); + // env: explicit environment (default: the process environment); pid < 0 = getpid(); now < 0 = time.time() + static ShellContext current(const std::map* env = nullptr, long pid = -1, const std::string& proc_root = "/proc", double now = -1.0, long clk_tck = 0); + // os.path.realpath(directory or os.getcwd()), "" on error + static std::string dir_key(const std::string& directory = ""); + + void export_env() const; // setenv TAI_SESSION=sid + void register_in(store::Store& store) const; + static std::vector tree_lines(store::Store& store, const std::string& sid); + static std::string report(store::Store& store, const std::string& sid); +}; + +} // namespace detz::session diff --git a/taicpp/src/session/term.cpp b/taicpp/src/session/term.cpp new file mode 100644 index 0000000..9f5830f --- /dev/null +++ b/taicpp/src/session/term.cpp @@ -0,0 +1,17 @@ +#include "term.h" +#include +#include +namespace detz::session { + +bool Term::color_enabled() { + const char* no_color = std::getenv("NO_COLOR"); + if (no_color && *no_color) return false; + return ::isatty(1) != 0; +} + +std::string Term::paint(const std::string& text, const std::string& codes) { + if (!color_enabled()) return text; + return codes + text + kReset; +} + +} // namespace detz::session diff --git a/taicpp/src/session/term.h b/taicpp/src/session/term.h new file mode 100644 index 0000000..7749bc7 --- /dev/null +++ b/taicpp/src/session/term.h @@ -0,0 +1,22 @@ +#pragma once +#include +namespace detz::session { + +// The Ansi codes and paint() of tai.py that the session and REPL layer use. +class Term { + public: + static constexpr const char* kReset = "\033[0m"; + static constexpr const char* kBold = "\033[1m"; + static constexpr const char* kDim = "\033[2m"; + static constexpr const char* kRed = "\033[31m"; + static constexpr const char* kGreen = "\033[32m"; + static constexpr const char* kYellow = "\033[33m"; + static constexpr const char* kCyan = "\033[36m"; + + // False when NO_COLOR is set or stdout is not a terminal. + static bool color_enabled(); + // paint(text, *codes): plain text when color is off + static std::string paint(const std::string& text, const std::string& codes = ""); +}; + +} // namespace detz::session diff --git a/taicpp/src/store/clock.cpp b/taicpp/src/store/clock.cpp new file mode 100644 index 0000000..8ae9762 --- /dev/null +++ b/taicpp/src/store/clock.cpp @@ -0,0 +1,40 @@ +#include "clock.h" +#include +#include +#include +#include +namespace detz::store { + +static std::string stamp(bool micro) { + using namespace std::chrono; + auto now = system_clock::now(); + auto secs = time_point_cast(now); + long us = static_cast(duration_cast(now - secs).count()); + std::time_t t = system_clock::to_time_t(secs); + std::tm tm{}; + gmtime_r(&t, &tm); + char b[64]; + std::snprintf(b, sizeof b, "%04d-%02d-%02dT%02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + std::string out = b; + if (micro && us != 0) { std::snprintf(b, sizeof b, ".%06ld", us); out += b; } + return out + "+00:00"; +} + +std::string Clock::now_iso() { return stamp(false); } +std::string Clock::now_iso_micro() { return stamp(true); } +double Clock::epoch() { + using namespace std::chrono; + return duration_cast>(system_clock::now().time_since_epoch()).count(); +} +std::string Clock::uuid_hex() { + static thread_local std::random_device rd; + std::string out; + char b[16]; + for (int k = 0; k < 4; k++) { std::snprintf(b, sizeof b, "%08x", static_cast(rd())); out += b; } + out[12] = '4'; // version nibble, as uuid4 + static const char* v = "89ab"; + out[16] = v[rd() & 3]; + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/clock.h b/taicpp/src/store/clock.h new file mode 100644 index 0000000..70fcd4a --- /dev/null +++ b/taicpp/src/store/clock.h @@ -0,0 +1,18 @@ +#pragma once +#include +namespace detz::store { + +// Timestamps in the exact python isoformat shapes tai.py writes. +class Clock { + public: + // datetime.now(timezone.utc).isoformat(timespec="seconds") -> 2026-10-25T12:34:56+00:00 + static std::string now_iso(); + // datetime.now(timezone.utc).isoformat() -> microseconds omitted when zero, like python + static std::string now_iso_micro(); + // time.time() seconds since the epoch + static double epoch(); + // uuid.uuid4().hex (32 lowercase hex chars) + static std::string uuid_hex(); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/debuglog.cpp b/taicpp/src/store/debuglog.cpp new file mode 100644 index 0000000..29c76e1 --- /dev/null +++ b/taicpp/src/store/debuglog.cpp @@ -0,0 +1,272 @@ +#include "debuglog.h" +#include +#include +#include +#include +#include +#include "clock.h" +#include "store.h" +#include "utf8.h" +namespace detz::store { + +namespace { +std::terminate_handler g_old_terminate = nullptr; +} + +DebugLog::DebugLog() : run(Clock::uuid_hex().substr(0, 8)) {} +DebugLog::~DebugLog() { + try { close(); } catch (...) {} +} + +DebugLog& DebugLog::Global() { + static DebugLog instance; + return instance; +} + +void DebugLog::configure(const std::string& s, const std::string& fmt, const std::string& h, Store* st) { + reset(); + sink = (s == "file" || s == "stdout" || s == "stderr") ? s : "file"; + format = (fmt == "sentence" || fmt == "json") ? fmt : "sentence"; + home = h; + enabled = true; + if (st) attach_store(*st); + g_old_terminate = std::set_terminate(&DebugLog::on_terminate); + hook_installed_ = true; + prune(); +} + +void DebugLog::reset() { + try { close(); } catch (...) {} + if (hook_installed_) { + if (g_old_terminate) std::set_terminate(g_old_terminate); + hook_installed_ = false; + } + enabled = false; + sink = "file"; + format = "sentence"; + home.clear(); + handle_ = nullptr; + wiped_ = false; + run = Clock::uuid_hex().substr(0, 8); + store_ = nullptr; + db_.reset(); + db_count = 0; + dropped = 0; + context.clear(); + g_old_terminate = nullptr; +} + +void DebugLog::attach_store(Store& st) { + store_ = &st; + try { + db_ = std::make_unique(st.config().dbPath, 30.0); + db_->execute("PRAGMA journal_mode=WAL"); + db_->execute("PRAGMA busy_timeout=30000"); + const SealPort* sp = &st.seal(); + db_->create_function("tai_enc", [sp](const std::string& t) { return sp->lock(t); }); + } catch (const SqlError&) { + db_.reset(); + } +} + +void DebugLog::close() { + std::FILE* h = handle_; + handle_ = nullptr; + if (h && sink == "file") std::fclose(h); + if (db_) { + try { db_->commit(); } catch (const SqlError&) {} + db_.reset(); + } +} + +std::FILE* DebugLog::open_handle() { + if (sink == "stdout") return stdout; + if (sink == "stderr") return stderr; + std::error_code ec; + std::filesystem::create_directories(home, ec); + const char* child = std::getenv("TAI_CHILD"); + bool truncate = !wiped_ && !(child && *child); + wiped_ = true; + std::FILE* f = std::fopen((std::filesystem::path(home) / "debug.log").string().c_str(), truncate ? "w" : "a"); + if (!f) throw std::runtime_error("cannot open debug.log"); + return f; +} + +std::string DebugLog::flat(const std::string& text) { + auto parts = Utf8::split_ws(text); + std::string out; + for (size_t k = 0; k < parts.size(); k++) { if (k) out += ' '; out += parts[k]; } + return out; +} + +std::string DebugLog::scrub(const std::string& text, const std::string& profile) { + if (text.empty() || !store_) return text; + try { return store_->redact(text, profile); } catch (...) { return text; } +} + +std::string DebugLog::short_error(const Json& value) { + std::string t = value.py_str(); + if (t.empty()) t = value.is_array() ? "list" : value.is_object() ? "dict" : "str"; + return Utf8::prefix(t, 300); +} + +Json DebugLog::scrub_deep(const Json& value, const std::string& profile, size_t limit, int depth) { + if (value.is_string()) { + std::string clean = scrub(value.as_string(), profile); + if (Utf8::length(clean) > limit) clean = Utf8::prefix(clean, limit) + "..."; + return Json(flat(clean)); + } + if (depth >= 4) return Json(short_error(value)); + if (value.is_array()) { + Json out = Json::array(); + size_t n = 0; + for (auto& item : value.items()) { if (n++ >= 100) break; out.push(scrub_deep(item, profile, limit, depth + 1)); } + return out; + } + if (value.is_object()) { + std::vector order; + for (auto& m : value.members()) order.push_back(&m); + std::stable_sort(order.begin(), order.end(), [](const Json::Member* a, const Json::Member* b) { return a->first < b->first; }); + Json out = Json::object(); + for (auto* m : order) out.set(m->first, scrub_deep(m->second, profile, limit, depth + 1)); + return out; + } + return value; +} + +Json DebugLog::cut_meta(const Json& meta, const std::string& profile, size_t limit) { + Json out = Json::object(); + if (!meta.is_object()) return out; + std::vector order; + for (auto& m : meta.members()) order.push_back(&m); + std::stable_sort(order.begin(), order.end(), [](const Json::Member* a, const Json::Member* b) { return a->first < b->first; }); + for (auto* m : order) out.set(m->first, scrub_deep(m->second, profile, limit)); + return out; +} + +std::string DebugLog::pack_meta(const Json& meta) { + std::string blob = meta.dumps(true); + if (blob.size() <= kMetaMax) return blob; + Json slim = meta; + for (int pass = 0; pass < 4; pass++) { + std::vector big; + for (auto& m : slim.members()) if (m.second.is_string() && Utf8::length(m.second.as_string()) > 100) big.push_back(m.first); + if (big.empty()) break; + for (auto& key : big) { + const std::string& v = slim.find(key)->as_string(); + slim.set(key, Json(Utf8::prefix(v, Utf8::length(v) / 2) + "...")); + } + blob = slim.dumps(true); + if (blob.size() <= kMetaMax) return blob; + } + Json fallback = Json::object(); + fallback.set("truncated", true); + Json keys = Json::array(); + std::vector names; + for (auto& m : meta.members()) names.push_back(m.first); + std::sort(names.begin(), names.end()); + for (auto& n : names) keys.push(Json(n)); + fallback.set("keys", keys); + return fallback.dumps(true); +} + +std::string DebugLog::quote(const Json& value) { + if (!value.is_string()) return value.dumps(true); + const std::string& s = value.as_string(); + bool special = s.empty() || Utf8::has_space(s) || s.find('"') != std::string::npos || s.find('\\') != std::string::npos; + if (!special) return s; + std::string out = "\""; + for (char c : s) { + if (c == '\\') out += "\\\\"; + else if (c == '"') out += "\\\""; + else out += c; + } + return out + "\""; +} + +std::string DebugLog::sentence(const std::string& ts, const std::string& level, const std::string& run_id, const std::string& profile, const std::string& bot, const std::string& chat, + const std::string& component, const std::string& event_name, const std::string& message, const Json& meta) { + std::string line = ts + " " + level + " run=" + run_id + " profile=" + quote(Json(profile)) + " bot=" + quote(Json(bot)) + " chat=" + quote(Json(chat)) + " " + component + "." + event_name; + std::vector order; + for (auto& m : meta.members()) order.push_back(&m); + std::stable_sort(order.begin(), order.end(), [](const Json::Member* a, const Json::Member* b) { return a->first < b->first; }); + for (auto* m : order) line += " " + m->first + "=" + quote(m->second); + return line + " :: " + message; +} + +std::string DebugLog::json_line(const std::string& ts, const std::string& level, const std::string& run_id, const std::string& profile, const std::string& bot, const std::string& chat, + const std::string& component, const std::string& event_name, const std::string& message, const Json& meta) { + Json o = Json::object(); + o.set("ts", ts); o.set("level", level); o.set("run", run_id); o.set("profile", profile); o.set("bot", bot); o.set("chat", chat); + o.set("component", component); o.set("event", event_name); o.set("message", message); o.set("meta", meta); + return o.dumps(true); +} + +void DebugLog::event(const std::string& level, const std::string& component, const std::string& event_name, const std::string& message, + const std::string& profile_in, const std::string& bot_in, const std::string& chat_in, const Json& meta) { + if (!enabled) return; + try { + std::lock_guard guard(lock_); + auto ctx = [&](const std::string& given, const char* key) { + if (!given.empty()) return given; + auto it = context.find(key); + return it == context.end() ? std::string() : it->second; + }; + std::string profile = ctx(profile_in, "profile"); + std::string bot = ctx(bot_in, "bot"); + std::string chat = ctx(chat_in, "chat"); + std::string clean = Utf8::prefix(flat(scrub(message, profile)), kMsgMax); + Json db_meta = cut_meta(meta, profile, kTextMax); + Json show_meta = cut_meta(meta, profile, kPreview); + std::string ts = Clock::now_iso_micro(); + std::string line = format == "json" ? json_line(ts, level, run, profile, bot, chat, component, event_name, clean, show_meta) + : sentence(ts, level, run, profile, bot, chat, component, event_name, clean, show_meta); + if (!handle_) handle_ = open_handle(); + std::fputs((line + "\n").c_str(), handle_); + std::fflush(handle_); + if (db_) { + std::string blob = pack_meta(db_meta); + bool sealed = store_ && store_->seal().enabled(); + if (sealed) { + db_->execute("INSERT INTO logs (profile, bot, chat, ts, run, level, component, event, message, meta_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, tai_enc(?), tai_enc(?))", + {profile, bot, chat, ts, run, level, component, event_name, clean, blob}); + } else { + db_->execute("INSERT INTO logs (profile, bot, chat, ts, run, level, component, event, message, meta_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + {profile, bot, chat, ts, run, level, component, event_name, clean, blob}); + } + db_count++; + db_->commit(); + if (db_count % kDbPruneEvery == 0) prune(); + } + } catch (...) { + dropped++; + } +} + +void DebugLog::prune() { + if (!db_) return; + try { + std::lock_guard guard(lock_); + db_->execute("DELETE FROM logs WHERE id NOT IN (SELECT id FROM logs ORDER BY id DESC LIMIT ?)", {kDbKeep}); + db_->commit(); + } catch (const SqlError&) {} +} + +void DebugLog::on_terminate() { + DebugLog& log = Global(); + try { + std::string what = "unknown"; + try { + if (auto ep = std::current_exception()) std::rethrow_exception(ep); + } catch (const std::exception& e) { + what = e.what(); + } catch (...) {} + Json m = Json::object(); + m.set("error", "terminate: " + what); + log.event("ERROR", "app", "exception", "uncaught " + Utf8::prefix(what, 300), "", "", "", m); + } catch (...) {} + if (g_old_terminate) g_old_terminate(); + std::abort(); +} + +} // namespace detz::store diff --git a/taicpp/src/store/debuglog.h b/taicpp/src/store/debuglog.h new file mode 100644 index 0000000..ce1a8db --- /dev/null +++ b/taicpp/src/store/debuglog.h @@ -0,0 +1,73 @@ +#pragma once +#include +#include +#include +#include +#include +#include "json.h" +#include "sqldb.h" +namespace detz::store { + +class Store; + +// tai.py DebugLog (logging.md): one line per event to a sink (file, stdout or +// stderr) plus the vault `logs` table. Everything is bounded: message, text +// and preview caps, a row ceiling, 16 KiB meta. Disabled by default; the +// global instance Global() is the python module-level LOG. +class DebugLog { + public: + static constexpr int kDbKeep = 20000; // LOG_DB_KEEP + static constexpr int kDbPruneEvery = 256; // LOG_DB_PRUNE_EVERY + static constexpr int kMsgMax = 2000; // LOG_MSG_MAX + static constexpr int kTextMax = 4000; // LOG_TEXT_MAX + static constexpr int kPreview = 300; // LOG_PREVIEW + static constexpr size_t kMetaMax = 16384; // LOG_META_MAX + + DebugLog(); + ~DebugLog(); + static DebugLog& Global(); + + // python configure(sink, fmt, home, store) + void configure(const std::string& sink = "file", const std::string& fmt = "sentence", const std::string& home = "", Store* store = nullptr); + void reset(); + void attach_store(Store& store); + void close(); + void event(const std::string& level, const std::string& component, const std::string& event, const std::string& message, + const std::string& profile = "", const std::string& bot = "", const std::string& chat = "", const Json& meta = Json()); + void prune(); + + bool enabled{false}; + std::string sink{"file"}; + std::string format{"sentence"}; + std::string home; + std::string run; + int64_t db_count{0}; + int64_t dropped{0}; + std::map context; + + // formatting helpers (public for the doc and unit tests) + static std::string flat(const std::string& text); + static std::string quote(const Json& value); + static std::string sentence(const std::string& ts, const std::string& level, const std::string& run, const std::string& profile, const std::string& bot, const std::string& chat, + const std::string& component, const std::string& event, const std::string& message, const Json& meta); + static std::string json_line(const std::string& ts, const std::string& level, const std::string& run, const std::string& profile, const std::string& bot, const std::string& chat, + const std::string& component, const std::string& event, const std::string& message, const Json& meta); + static std::string pack_meta(const Json& meta); + static std::string short_error(const Json& value); + + private: + std::string scrub(const std::string& text, const std::string& profile); + Json scrub_deep(const Json& value, const std::string& profile, size_t limit, int depth = 0); + Json cut_meta(const Json& meta, const std::string& profile, size_t limit); + std::FILE* open_handle(); + static void on_terminate(); + + Store* store_{nullptr}; + std::unique_ptr db_; + std::FILE* handle_{nullptr}; + bool wiped_{false}; + bool hook_installed_{false}; + std::recursive_mutex lock_; +}; + +} // namespace detz::store diff --git a/taicpp/src/store/errors.h b/taicpp/src/store/errors.h new file mode 100644 index 0000000..5fe5b01 --- /dev/null +++ b/taicpp/src/store/errors.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +namespace detz::store { + +// python ValueError. +class ValueError : public std::runtime_error { + public: + explicit ValueError(const std::string& m) : std::runtime_error(m) {} +}; + +// python sqlite3.Error. +class SqlError : public std::runtime_error { + public: + explicit SqlError(const std::string& m) : std::runtime_error(m) {} +}; + +// python SealError. +class SealError : public std::runtime_error { + public: + explicit SealError(const std::string& m) : std::runtime_error(m) {} +}; + +} // namespace detz::store diff --git a/taicpp/src/store/fts.cpp b/taicpp/src/store/fts.cpp new file mode 100644 index 0000000..2aeb0a9 --- /dev/null +++ b/taicpp/src/store/fts.cpp @@ -0,0 +1,43 @@ +#include "fts.h" +#include +#include "json.h" +#include "tag_rules.h" +namespace detz::store { + +std::optional> Fts::escape(const std::string& query) { + auto toks = TagRules::tokens(query); + if (toks.empty()) return std::nullopt; + std::vector out; + for (auto& t : toks) out.push_back("\"" + t + "\""); + return out; +} + +static std::string join(const std::vector& terms, const char* sep, bool prefix) { + std::string out; + for (size_t k = 0; k < terms.size(); k++) { + if (k) out += sep; + std::string t = terms[k]; + if (prefix) { // term.strip('"') then "*" + size_t a = 0, b = t.size(); + while (a < b && t[a] == '"') a++; + while (b > a && t[b - 1] == '"') b--; + t = t.substr(a, b - a) + "*"; + } + out += "{title body} : " + t; + } + return out; +} + +std::string Fts::match(const std::vector& terms) { return join(terms, " AND ", false); } +std::string Fts::match_or(const std::vector& terms) { return join(terms, " OR ", false); } +std::string Fts::match_prefix(const std::vector& terms) { return join(terms, " AND ", true); } +std::vector Fts::match_variants(const std::vector& terms) { + std::vector v{match(terms), match_prefix(terms)}; + if (terms.size() > 1) v.push_back(match_or(terms)); + return v; +} +std::string Fts::rank(const std::string& table) { + return "bm25(" + table + ", 1.0, 1.0, 1.0, " + Json::float_repr(kTitleWeight) + ", " + Json::float_repr(kBodyWeight) + ")"; +} + +} // namespace detz::store diff --git a/taicpp/src/store/fts.h b/taicpp/src/store/fts.h new file mode 100644 index 0000000..a131ca5 --- /dev/null +++ b/taicpp/src/store/fts.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include +#include +namespace detz::store { + +// FTS5 query builders (tai.py fts_escape, fts_match, fts_match_or, +// fts_match_prefix, fts_match_variants, fts_rank). +class Fts { + public: + static constexpr double kTitleWeight = 3.0; // FTS_TITLE_WEIGHT + static constexpr double kBodyWeight = 1.0; // FTS_BODY_WEIGHT + // quoted lowercase terms, nullopt when the query holds none + static std::optional> escape(const std::string& query); + static std::string match(const std::vector& terms); + static std::string match_or(const std::vector& terms); + static std::string match_prefix(const std::vector& terms); + // strict to lenient: AND, prefix-AND, OR (the OR only with several terms) + static std::vector match_variants(const std::vector& terms); + static std::string rank(const std::string& table); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/json.cpp b/taicpp/src/store/json.cpp new file mode 100644 index 0000000..afa0cc5 --- /dev/null +++ b/taicpp/src/store/json.cpp @@ -0,0 +1,350 @@ +#include "json.h" +#include +#include +#include +#include +#include +#include +namespace detz::store { + +void Json::push(Json v) { if (kind_ != Kind::Array) { kind_ = Kind::Array; a_.clear(); } a_.push_back(std::move(v)); } +bool Json::has(const std::string& key) const { return find(key) != nullptr; } +const Json* Json::find(const std::string& key) const { + for (auto& m : o_) if (m.first == key) return &m.second; + return nullptr; +} +Json& Json::set(const std::string& key, Json v) { + if (kind_ != Kind::Object) { kind_ = Kind::Object; o_.clear(); } + for (auto& m : o_) if (m.first == key) { m.second = std::move(v); return m.second; } + o_.emplace_back(key, std::move(v)); + return o_.back().second; +} +std::string Json::get_string(const std::string& key, const std::string& fallback) const { + const Json* v = find(key); + return (v && v->is_string()) ? v->s_ : fallback; +} +void Json::erase(const std::string& key) { + o_.erase(std::remove_if(o_.begin(), o_.end(), [&](const Member& m) { return m.first == key; }), o_.end()); +} +bool Json::operator==(const Json& o) const { + // python: True == 1 == 1.0 + auto numeric = [](const Json& j) { return j.is_number() || j.is_bool(); }; + auto value = [](const Json& j) { return j.is_bool() ? (j.as_bool() ? 1.0 : 0.0) : j.as_double(); }; + if (numeric(*this) && numeric(o)) return value(*this) == value(o); + if (kind_ != o.kind_) return false; + switch (kind_) { + case Kind::Null: return true; + case Kind::Bool: return b_ == o.b_; + case Kind::String: return s_ == o.s_; + case Kind::Array: return a_ == o.a_; + case Kind::Object: { + if (o_.size() != o.o_.size()) return false; + for (auto& m : o_) { const Json* x = o.find(m.first); if (!x || !(*x == m.second)) return false; } + return true; + } + default: return false; + } +} + +std::string Json::float_repr(double v) { + if (std::isnan(v)) return "NaN"; + if (std::isinf(v)) return v < 0 ? "-Infinity" : "Infinity"; + if (v == 0) return std::signbit(v) ? "-0.0" : "0.0"; + char buf[64]; + auto r = std::to_chars(buf, buf + sizeof buf, v, std::chars_format::scientific); + std::string sci(buf, r.ptr); // d.ddde[+-]XX, shortest round trip digits + bool neg = sci[0] == '-'; + if (neg) sci.erase(0, 1); + size_t e = sci.find('e'); + std::string mant = sci.substr(0, e); + int exp10 = std::atoi(sci.c_str() + e + 1); + std::string digits; + for (char c : mant) if (c != '.') digits += c; + int decpt = exp10 + 1; + std::string out; + if (decpt > 16 || decpt < -3) { // repr switches to exponent form + out = digits.substr(0, 1); + if (digits.size() > 1) out += "." + digits.substr(1); + int x = decpt - 1; + char eb[16]; + std::snprintf(eb, sizeof eb, "e%c%02d", x < 0 ? '-' : '+', std::abs(x)); + out += eb; + } else if (decpt <= 0) { + out = "0." + std::string(static_cast(-decpt), '0') + digits; + } else if (static_cast(decpt) >= digits.size()) { + out = digits + std::string(static_cast(decpt) - digits.size(), '0') + ".0"; + } else { + out = digits.substr(0, static_cast(decpt)) + "." + digits.substr(static_cast(decpt)); + } + return neg ? "-" + out : out; +} + +static void escape_to(std::string& out, const std::string& s) { + out += '"'; + size_t i = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + if (c == '"') { out += "\\\""; i++; } + else if (c == '\\') { out += "\\\\"; i++; } + else if (c == '\n') { out += "\\n"; i++; } + else if (c == '\r') { out += "\\r"; i++; } + else if (c == '\t') { out += "\\t"; i++; } + else if (c == '\b') { out += "\\b"; i++; } + else if (c == '\f') { out += "\\f"; i++; } + else if (c < 0x20 || c == 0x7f) { char b[8]; std::snprintf(b, sizeof b, "\\u%04x", c); out += b; i++; } + else if (c < 0x80) { out += static_cast(c); i++; } + else { + uint32_t cp = 0xFFFD; int n = 1; + auto cont = [&](size_t k) { return static_cast(static_cast(s[i + k])) & 0x3Fu; }; + if ((c & 0xE0) == 0xC0 && i + 1 < s.size()) { cp = ((c & 0x1Fu) << 6) | cont(1); n = 2; } + else if ((c & 0xF0) == 0xE0 && i + 2 < s.size()) { cp = ((c & 0x0Fu) << 12) | (cont(1) << 6) | cont(2); n = 3; } + else if ((c & 0xF8) == 0xF0 && i + 3 < s.size()) { cp = ((c & 0x07u) << 18) | (cont(1) << 12) | (cont(2) << 6) | cont(3); n = 4; } + char b[16]; + if (cp >= 0x10000) { + uint32_t v = cp - 0x10000; + std::snprintf(b, sizeof b, "\\u%04x\\u%04x", 0xD800 + (v >> 10), 0xDC00 + (v & 0x3FF)); + } else { + std::snprintf(b, sizeof b, "\\u%04x", cp); + } + out += b; + i += static_cast(n); + } + } + out += '"'; +} + +void Json::dump_to(std::string& out, bool sort_keys) const { + switch (kind_) { + case Kind::Null: out += "null"; break; + case Kind::Bool: out += b_ ? "true" : "false"; break; + case Kind::Int: out += std::to_string(i_); break; + case Kind::Float: out += float_repr(d_); break; + case Kind::String: escape_to(out, s_); break; + case Kind::Array: { + out += '['; + for (size_t k = 0; k < a_.size(); k++) { if (k) out += ", "; a_[k].dump_to(out, sort_keys); } + out += ']'; + break; + } + case Kind::Object: { + out += '{'; + std::vector order; + for (auto& m : o_) order.push_back(&m); + if (sort_keys) std::stable_sort(order.begin(), order.end(), [](const Member* a, const Member* b) { return a->first < b->first; }); + bool first = true; + for (auto* m : order) { + if (!first) out += ", "; + first = false; + escape_to(out, m->first); + out += ": "; + m->second.dump_to(out, sort_keys); + } + out += '}'; + break; + } + } +} +std::string Json::dumps(bool sort_keys) const { std::string out; dump_to(out, sort_keys); return out; } + +static std::string py_str_repr(const std::string& s) { + bool has_single = s.find('\'') != std::string::npos, has_double = s.find('"') != std::string::npos; + char q = (has_single && !has_double) ? '"' : '\''; + std::string out(1, q); + for (size_t i = 0; i < s.size(); i++) { + unsigned char c = static_cast(s[i]); + if (c == static_cast(q) || c == '\\') { out += '\\'; out += static_cast(c); } + else if (c == '\n') out += "\\n"; + else if (c == '\r') out += "\\r"; + else if (c == '\t') out += "\\t"; + else if (c < 0x20 || c == 0x7f) { char b[8]; std::snprintf(b, sizeof b, "\\x%02x", c); out += b; } + else out += static_cast(c); + } + out += q; + return out; +} + +std::string Json::py_repr() const { + switch (kind_) { + case Kind::Null: return "None"; + case Kind::Bool: return b_ ? "True" : "False"; + case Kind::Int: return std::to_string(i_); + case Kind::Float: { + if (std::isnan(d_)) return "nan"; + if (std::isinf(d_)) return d_ < 0 ? "-inf" : "inf"; + return float_repr(d_); + } + case Kind::String: return py_str_repr(s_); + case Kind::Array: { + std::string out = "["; + for (size_t k = 0; k < a_.size(); k++) { if (k) out += ", "; out += a_[k].py_repr(); } + return out + "]"; + } + case Kind::Object: { + std::string out = "{"; + for (size_t k = 0; k < o_.size(); k++) { if (k) out += ", "; out += py_str_repr(o_[k].first) + ": " + o_[k].second.py_repr(); } + return out + "}"; + } + } + return ""; +} +std::string Json::py_str() const { return kind_ == Kind::String ? s_ : py_repr(); } + +namespace { +class Parser { + public: + explicit Parser(const std::string& t) : t_(t) {} + Json run() { + ws(); + Json v = value(0); + ws(); + if (p_ != t_.size()) fail("Extra data"); + return v; + } + private: + [[noreturn]] void fail(const std::string& m) { throw JsonError(m + ": char " + std::to_string(p_)); } + void ws() { while (p_ < t_.size() && (t_[p_] == ' ' || t_[p_] == '\t' || t_[p_] == '\n' || t_[p_] == '\r')) p_++; } + bool lit(const char* w) { size_t n = std::char_traits::length(w); if (t_.compare(p_, n, w) == 0) { p_ += n; return true; } return false; } + static bool dig(char c) { return c >= '0' && c <= '9'; } + Json value(int depth) { + if (depth > 200) fail("Recursion"); + if (p_ >= t_.size()) fail("Expecting value"); + char c = t_[p_]; + if (c == '{') return object(depth); + if (c == '[') return arr(depth); + if (c == '"') return Json(str()); + if (lit("true")) return Json(true); + if (lit("false")) return Json(false); + if (lit("null")) return Json(); + if (lit("NaN")) return Json(std::nan("")); + if (lit("Infinity")) return Json(HUGE_VAL); + if (lit("-Infinity")) return Json(-HUGE_VAL); + if (c == '-' || dig(c)) return number(); + fail("Expecting value"); + } + Json number() { + size_t s = p_; + if (t_[p_] == '-') p_++; + if (p_ >= t_.size()) fail("Expecting value"); + if (t_[p_] == '0') p_++; + else if (t_[p_] >= '1' && t_[p_] <= '9') { while (p_ < t_.size() && dig(t_[p_])) p_++; } + else fail("Expecting value"); + bool flt = false; + if (p_ + 1 < t_.size() && t_[p_] == '.' && dig(t_[p_ + 1])) { + flt = true; p_++; + while (p_ < t_.size() && dig(t_[p_])) p_++; + } + if (p_ < t_.size() && (t_[p_] == 'e' || t_[p_] == 'E')) { + size_t q = p_ + 1; + if (q < t_.size() && (t_[q] == '+' || t_[q] == '-')) q++; + if (q < t_.size() && dig(t_[q])) { + flt = true; p_ = q; + while (p_ < t_.size() && dig(t_[p_])) p_++; + } + } + std::string tok = t_.substr(s, p_ - s); + if (!flt) { + int64_t v = 0; + auto r = std::from_chars(tok.data(), tok.data() + tok.size(), v); + if (r.ec == std::errc()) return Json(static_cast(v)); + } + return Json(std::strtod(tok.c_str(), nullptr)); + } + static void put_utf8(std::string& o, uint32_t cp) { + if (cp < 0x80) o += static_cast(cp); + else if (cp < 0x800) { o += static_cast(0xC0 | (cp >> 6)); o += static_cast(0x80 | (cp & 0x3F)); } + else if (cp < 0x10000) { o += static_cast(0xE0 | (cp >> 12)); o += static_cast(0x80 | ((cp >> 6) & 0x3F)); o += static_cast(0x80 | (cp & 0x3F)); } + else { o += static_cast(0xF0 | (cp >> 18)); o += static_cast(0x80 | ((cp >> 12) & 0x3F)); o += static_cast(0x80 | ((cp >> 6) & 0x3F)); o += static_cast(0x80 | (cp & 0x3F)); } + } + uint32_t hex4() { + if (p_ + 4 > t_.size()) fail("Invalid \\uXXXX escape"); + uint32_t v = 0; + for (int k = 0; k < 4; k++) { + char c = t_[p_++]; + v <<= 4; + if (c >= '0' && c <= '9') v |= static_cast(c - '0'); + else if (c >= 'a' && c <= 'f') v |= static_cast(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v |= static_cast(c - 'A' + 10); + else fail("Invalid \\uXXXX escape"); + } + return v; + } + std::string str() { + p_++; + std::string o; + while (true) { + if (p_ >= t_.size()) fail("Unterminated string"); + unsigned char c = static_cast(t_[p_++]); + if (c == '"') return o; + if (c < 0x20) fail("Invalid control character"); + if (c != '\\') { o += static_cast(c); continue; } + if (p_ >= t_.size()) fail("Unterminated string"); + char e = t_[p_++]; + switch (e) { + case '"': o += '"'; break; + case '\\': o += '\\'; break; + case '/': o += '/'; break; + case 'b': o += '\b'; break; + case 'f': o += '\f'; break; + case 'n': o += '\n'; break; + case 'r': o += '\r'; break; + case 't': o += '\t'; break; + case 'u': { + uint32_t cp = hex4(); + if (cp >= 0xD800 && cp < 0xDC00 && t_.compare(p_, 2, "\\u") == 0) { + size_t save = p_; + p_ += 2; + uint32_t lo = hex4(); + if (lo >= 0xDC00 && lo < 0xE000) cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + else p_ = save; + } + put_utf8(o, cp); + break; + } + default: fail("Invalid \\escape"); + } + } + } + Json arr(int depth) { + p_++; + Json a = Json::array(); + ws(); + if (p_ < t_.size() && t_[p_] == ']') { p_++; return a; } + while (true) { + ws(); + a.push(value(depth + 1)); + ws(); + if (p_ >= t_.size()) fail("Expecting ',' delimiter"); + if (t_[p_] == ',') { p_++; continue; } + if (t_[p_] == ']') { p_++; return a; } + fail("Expecting ',' delimiter"); + } + } + Json object(int depth) { + p_++; + Json o = Json::object(); + ws(); + if (p_ < t_.size() && t_[p_] == '}') { p_++; return o; } + while (true) { + ws(); + if (p_ >= t_.size() || t_[p_] != '"') fail("Expecting property name enclosed in double quotes"); + std::string k = str(); + ws(); + if (p_ >= t_.size() || t_[p_] != ':') fail("Expecting ':' delimiter"); + p_++; + ws(); + o.set(k, value(depth + 1)); + ws(); + if (p_ >= t_.size()) fail("Expecting ',' delimiter"); + if (t_[p_] == ',') { p_++; continue; } + if (t_[p_] == '}') { p_++; return o; } + fail("Expecting ',' delimiter"); + } + } + const std::string& t_; + size_t p_{0}; +}; +} // namespace + +Json Json::loads(const std::string& text) { Parser p(text); return p.run(); } + +} // namespace detz::store diff --git a/taicpp/src/store/json.h b/taicpp/src/store/json.h new file mode 100644 index 0000000..ad8a25b --- /dev/null +++ b/taicpp/src/store/json.h @@ -0,0 +1,86 @@ +#pragma once +#include +#include +#include +#include +#include +namespace detz::store { + +// Raised where python raises ValueError out of json.loads. +class JsonError : public std::runtime_error { + public: + explicit JsonError(const std::string& m) : std::runtime_error(m) {} +}; + +// Minimal JSON value with python json semantics: dumps() matches +// json.dumps defaults (ensure_ascii, ", " and ": " separators, insertion +// ordered objects) and optional sort_keys; loads() is strict RFC 8259 plus +// the NaN/Infinity/-Infinity literals python accepts. +class Json { + public: + enum class Kind { Null, Bool, Int, Float, String, Array, Object }; + using Member = std::pair; + + Json() = default; + Json(std::nullptr_t) {} + Json(bool v) : kind_(Kind::Bool), b_(v) {} + Json(int v) : kind_(Kind::Int), i_(v) {} + Json(long v) : kind_(Kind::Int), i_(v) {} + Json(long long v) : kind_(Kind::Int), i_(v) {} + Json(unsigned long v) : kind_(Kind::Int), i_(static_cast(v)) {} + Json(double v) : kind_(Kind::Float), d_(v) {} + Json(const char* v) : kind_(Kind::String), s_(v) {} + Json(std::string v) : kind_(Kind::String), s_(std::move(v)) {} + + static Json array() { Json j; j.kind_ = Kind::Array; return j; } + static Json object() { Json j; j.kind_ = Kind::Object; return j; } + + Kind kind() const { return kind_; } + bool is_null() const { return kind_ == Kind::Null; } + bool is_bool() const { return kind_ == Kind::Bool; } + bool is_int() const { return kind_ == Kind::Int; } + bool is_float() const { return kind_ == Kind::Float; } + bool is_number() const { return is_int() || is_float(); } + bool is_string() const { return kind_ == Kind::String; } + bool is_array() const { return kind_ == Kind::Array; } + bool is_object() const { return kind_ == Kind::Object; } + + bool as_bool() const { return b_; } + int64_t as_int() const { return kind_ == Kind::Float ? static_cast(d_) : i_; } + double as_double() const { return kind_ == Kind::Int ? static_cast(i_) : d_; } + const std::string& as_string() const { return s_; } + + // array + const std::vector& items() const { return a_; } + std::vector& items() { return a_; } + void push(Json v); + size_t size() const { return kind_ == Kind::Object ? o_.size() : a_.size(); } + + // object + const std::vector& members() const { return o_; } + bool has(const std::string& key) const; + const Json* find(const std::string& key) const; + Json& set(const std::string& key, Json v); // replaces in place, keeps position + std::string get_string(const std::string& key, const std::string& fallback = "") const; + void erase(const std::string& key); + + std::string dumps(bool sort_keys = false) const; + static Json loads(const std::string& text); + static std::string float_repr(double v); // python repr(float) + std::string py_repr() const; // python repr() of the equivalent value (dict, list, str, int, float, bool, None) + std::string py_str() const; // python str(): strings raw, everything else repr + + bool operator==(const Json& o) const; + + private: + void dump_to(std::string& out, bool sort_keys) const; + Kind kind_{Kind::Null}; + bool b_{false}; + int64_t i_{0}; + double d_{0}; + std::string s_; + std::vector a_; + std::vector o_; +}; + +} // namespace detz::store diff --git a/taicpp/src/store/patterns.cpp b/taicpp/src/store/patterns.cpp new file mode 100644 index 0000000..c88572b --- /dev/null +++ b/taicpp/src/store/patterns.cpp @@ -0,0 +1,25 @@ +#include "patterns.h" +namespace detz::store { + +bool Patterns::match(const std::string& in, bool lower_only, bool allow_dot, size_t max_rest) { + std::string s = in; + if (!s.empty() && s.back() == '\n') s.pop_back(); // "$" before one trailing newline + if (s.empty() || s.size() > max_rest + 1) return false; + auto alnum = [&](char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (!lower_only && c >= 'A' && c <= 'Z'); + }; + if (!alnum(s[0])) return false; + for (size_t k = 1; k < s.size(); k++) { + char c = s[k]; + if (alnum(c) || c == '_' || c == '-' || (allow_dot && c == '.')) continue; + return false; + } + return true; +} + +bool Patterns::profile(const std::string& s) { return match(s, false, false, 31); } +bool Patterns::bot(const std::string& s) { return match(s, true, false, 31); } +bool Patterns::secret(const std::string& s) { return match(s, false, true, 63); } +bool Patterns::mcp_server(const std::string& s) { return match(s, false, true, 63); } + +} // namespace detz::store diff --git a/taicpp/src/store/patterns.h b/taicpp/src/store/patterns.h new file mode 100644 index 0000000..1241de9 --- /dev/null +++ b/taicpp/src/store/patterns.h @@ -0,0 +1,22 @@ +#pragma once +#include +namespace detz::store { + +// The anchored name regexes of tai.py. Each mirrors re.match with "^...$": +// python's "$" also accepts one trailing newline, so these do too. +class Patterns { + public: + // PROFILE_RE ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ + static bool profile(const std::string& s); + // BOT_RE ^[a-z0-9][a-z0-9_-]{0,31}$ + static bool bot(const std::string& s); + // SECRET_RE ^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$ + static bool secret(const std::string& s); + // MCP_SERVER_RE ^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$ + static bool mcp_server(const std::string& s); + + private: + static bool match(const std::string& s, bool lower_only, bool allow_dot, size_t max_rest); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/schema.cpp b/taicpp/src/store/schema.cpp new file mode 100644 index 0000000..3e7a6c2 --- /dev/null +++ b/taicpp/src/store/schema.cpp @@ -0,0 +1,193 @@ +#include "schema.h" +#include +#include +namespace detz::store { + +const std::vector& Schema::tables() { + static const std::vector t = { + {"events", {"id INTEGER PRIMARY KEY", "profile TEXT", "ts TEXT", "role TEXT", "kind TEXT", "text TEXT", "tags TEXT"}, {}}, + {"secrets", {"profile TEXT", "name TEXT", "value TEXT", "meta TEXT", "updated TEXT"}, {"profile", "name"}}, + {"schedules", {"id INTEGER PRIMARY KEY", "name TEXT", "prompt TEXT", "profile TEXT", "every_sec INTEGER", "next_run TEXT", "timeout INTEGER", "status TEXT", "last_status TEXT", "last_result TEXT", "created TEXT", "updated TEXT"}, {}}, + {"records", {"id TEXT PRIMARY KEY", "kind TEXT", "profile TEXT", "title TEXT", "content TEXT", "size INTEGER", "reads INTEGER", "created TEXT", "updated TEXT"}, {}}, + {"tags", {"item TEXT", "tag TEXT", "profile TEXT"}, {"item", "tag", "profile"}}, + {"edges", {"src TEXT", "dst TEXT", "relation TEXT", "profile TEXT", "created TEXT"}, {"src", "dst", "relation", "profile"}}, + {"audit", {"id INTEGER PRIMARY KEY", "profile TEXT", "ts TEXT", "actor TEXT", "action TEXT", "path TEXT", "message TEXT", "old_size INTEGER", "new_size INTEGER", "old TEXT", "new TEXT", "tags TEXT"}, {}}, + {"chunks", {"id INTEGER PRIMARY KEY", "parent_kind TEXT", "parent_id TEXT", "idx INTEGER", "text TEXT", "profile TEXT", "created TEXT"}, {}}, + {"dir_defaults", {"dir TEXT PRIMARY KEY", "provider TEXT", "model TEXT", "bot TEXT", "updated TEXT"}, {}}, + {"input_history", {"id INTEGER PRIMARY KEY", "profile TEXT", "bot TEXT", "ts TEXT", "text TEXT"}, {}}, + {"shell_sessions", {"sid TEXT PRIMARY KEY", "parent TEXT", "shell TEXT", "pid INTEGER", "started TEXT", "last_seen TEXT", "cwd TEXT", "chain TEXT"}, {}}, + {"settings", {"key TEXT", "scope TEXT", "value TEXT", "created TEXT", "updated TEXT"}, {"key", "scope"}}, + {"mcp_servers", {"name TEXT PRIMARY KEY", "transport TEXT", "command TEXT", "args_json TEXT", "env_json TEXT", "url TEXT", "headers_json TEXT", "cwd TEXT", "allow_json TEXT", "deny_json TEXT", "source TEXT", "created TEXT", "updated TEXT"}, {}}, + {"mcp_server_profiles", {"server TEXT", "profile TEXT", "enabled INTEGER"}, {"server", "profile"}}, + {"mcp_tool_pins", {"server TEXT", "tool TEXT", "hash TEXT", "updated TEXT"}, {"server", "tool"}}, + {"mcp_server_tools", {"server TEXT", "tool TEXT", "description TEXT", "schema_json TEXT", "updated TEXT"}, {"server", "tool"}}, + {"logs", {"id INTEGER PRIMARY KEY", "profile TEXT", "bot TEXT", "chat TEXT", "ts TEXT", "run TEXT", "level TEXT", "component TEXT", "event TEXT", "message TEXT", "meta_json TEXT"}, {}}, + }; + return t; +} + +const std::vector& Schema::indexes() { + static const std::vector v = { + {"idx_events_profile", "events(profile)"}, + {"idx_tags_tag", "tags(tag)"}, + {"idx_edges_src", "edges(src)"}, + {"idx_edges_dst", "edges(dst)"}, + {"idx_audit_path", "audit(path)"}, + {"idx_tags_profile", "tags(profile)"}, + {"idx_edges_profile", "edges(profile)"}, + {"idx_records_profile", "records(profile)"}, + {"idx_audit_profile", "audit(profile)"}, + {"idx_chunks_parent", "chunks(parent_kind, parent_id)"}, + {"idx_chunks_profile", "chunks(profile)"}, + {"idx_input_history_profile", "input_history(profile, id)"}, + {"idx_shell_sessions_parent", "shell_sessions(parent)"}, + {"idx_logs_profile", "logs(profile, id)"}, + {"idx_logs_bot", "logs(bot)"}, + {"idx_logs_chat", "logs(chat)"}, + {"idx_logs_level", "logs(level)"}, + {"idx_logs_event", "logs(component, event)"}, + {"idx_logs_run", "logs(run)"}, + }; + return v; +} + +static std::string join(const std::vector& v, const char* sep) { + std::string out; + for (size_t k = 0; k < v.size(); k++) { if (k) out += sep; out += v[k]; } + return out; +} + +static std::vector split_ws(const std::string& s) { + std::vector out; + std::string cur; + for (char c : s) { + if (c == ' ' || c == '\t' || c == '\n') { if (!cur.empty()) { out.push_back(cur); cur.clear(); } } + else cur += c; + } + if (!cur.empty()) out.push_back(cur); + return out; +} + +static std::set column_names(SqlDb& db, const std::string& table) { + std::set have; + for (auto& row : db.execute("PRAGMA table_info(" + table + ")").rows) have.insert(row[1].text()); + return have; +} + +std::vector Schema::ensure_schema(SqlDb& db) { + std::vector notes; + for (auto& t : tables()) { + auto exists = db.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", {t.name}); + if (exists.rows.empty()) { + std::string extra = t.pk.empty() ? "" : ", PRIMARY KEY (" + join(t.pk, ", ") + ")"; + db.execute("CREATE TABLE " + t.name + " (" + join(t.columns, ", ") + extra + ")"); + continue; + } + auto have = column_names(db, t.name); + for (auto& column : t.columns) { + auto parts = split_ws(column); + if (!have.count(parts[0])) { + db.execute("ALTER TABLE " + t.name + " ADD COLUMN " + parts[0] + " " + parts[1]); + notes.push_back("added " + t.name + "." + parts[0]); + } + } + } + for (auto& ix : indexes()) db.execute("CREATE INDEX IF NOT EXISTS " + ix.name + " ON " + ix.target); + db.commit(); + return notes; +} + +void Schema::ensure_column(SqlDb& db, const std::string& table, const std::string& column, const std::string& ddl) { + if (!column_names(db, table).count(column)) db.execute("ALTER TABLE " + table + " ADD COLUMN " + column + " " + ddl); +} + +std::vector Schema::table_pk_columns(SqlDb& db, const std::string& table) { + for (auto& row : db.execute("PRAGMA index_list(" + table + ")").rows) { + if (row[3].text() == "pk") { + std::vector cols; + for (auto& info : db.execute("PRAGMA index_info(" + row[1].text() + ")").rows) cols.push_back(info[2].text()); + return cols; + } + } + return {}; +} + +bool Schema::rebuild_with_profile(SqlDb& db, const std::string& table, const std::vector& columns, const std::vector& pk) { + if (table_pk_columns(db, table) == pk) return false; + std::vector names; + for (auto& c : columns) names.push_back(split_ws(c)[0]); + auto have = column_names(db, table); + std::vector keep; + for (auto& n : names) if (have.count(n)) keep.push_back(n); + db.execute("CREATE TABLE " + table + "_new (" + join(columns, ", ") + ", PRIMARY KEY (" + join(pk, ", ") + "))"); + if (!keep.empty()) db.execute("INSERT OR IGNORE INTO " + table + "_new (" + join(keep, ", ") + ") SELECT " + join(keep, ", ") + " FROM " + table); + db.execute("DROP TABLE " + table); + db.execute("ALTER TABLE " + table + "_new RENAME TO " + table); + return true; +} + +void Schema::migrate_profiles(SqlDb& db) { + for (const char* table : {"secrets", "records", "tags", "edges", "audit", "events", "schedules"}) { + ensure_column(db, table, "profile", "TEXT"); + db.execute(std::string("UPDATE ") + table + " SET profile = 'default' WHERE profile IS NULL"); + } + rebuild_with_profile(db, "secrets", {"profile TEXT", "name TEXT", "value TEXT", "meta TEXT", "updated TEXT"}, {"profile", "name"}); + rebuild_with_profile(db, "tags", {"item TEXT", "tag TEXT", "profile TEXT"}, {"item", "tag", "profile"}); + rebuild_with_profile(db, "edges", {"src TEXT", "dst TEXT", "relation TEXT", "profile TEXT", "created TEXT"}, {"src", "dst", "relation", "profile"}); +} + +bool Schema::setup_fts(SqlDb& db) { + try { + auto exists = db.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'fts_docs'"); + if (exists.rows.empty()) { + try { + db.execute("CREATE VIRTUAL TABLE fts_docs USING fts5(item, kind, sub, title, body, profile, tokenize='porter unicode61 remove_diacritics 2')"); + } catch (const SqlError&) { + db.execute("CREATE VIRTUAL TABLE fts_docs USING fts5(item, kind, sub, title, body, profile)"); + } + } + db.execute("CREATE TRIGGER IF NOT EXISTS trg_records_ai AFTER INSERT ON records BEGIN\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'record', new.kind, new.title, new.content, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_records_au AFTER UPDATE ON records WHEN old.title != new.title OR old.content != new.content OR old.profile != new.profile BEGIN\n" + " DELETE FROM fts_docs WHERE item = old.id AND kind = 'record';\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'record', new.kind, new.title, new.content, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_records_ad AFTER DELETE ON records BEGIN\n" + " DELETE FROM fts_docs WHERE item = old.id AND kind = 'record';\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_events_ai AFTER INSERT ON events BEGIN\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'event', new.role || '/' || new.kind, new.ts || ' ' || new.role || '/' || new.kind, new.text, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_events_au AFTER UPDATE ON events WHEN old.text != new.text OR old.profile != new.profile BEGIN\n" + " DELETE FROM fts_docs WHERE item = old.id AND kind = 'event';\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'event', new.role || '/' || new.kind, new.ts || ' ' || new.role || '/' || new.kind, new.text, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_audit_ai AFTER INSERT ON audit BEGIN\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES('audit:' || new.id, 'audit', new.action, new.path, new.message, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_chunks_ai AFTER INSERT ON chunks BEGIN\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES('chunk:' || new.id, 'chunk', new.parent_kind, new.parent_id, new.text, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_chunks_ad AFTER DELETE ON chunks BEGIN\n" + " DELETE FROM fts_docs WHERE item = 'chunk:' || old.id AND kind = 'chunk';\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_logs_ai AFTER INSERT ON logs BEGIN\n" + " INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES('log:' || new.id, 'log', new.component || '/' || new.event, new.ts || ' ' || new.level || ' ' || new.component || '/' || new.event, new.message, new.profile);\n" + " END"); + db.execute("CREATE TRIGGER IF NOT EXISTS trg_logs_ad AFTER DELETE ON logs BEGIN\n" + " DELETE FROM fts_docs WHERE item = 'log:' || old.id AND kind = 'log';\n" + " END"); + if (db.execute("SELECT COUNT(*) FROM fts_docs").rows[0][0].as_int() == 0) { + db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT id, 'record', kind, title, content, profile FROM records"); + db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT id, 'event', role || '/' || kind, ts || ' ' || role || '/' || kind, text, profile FROM events"); + db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT 'audit:' || id, 'audit', action, path, message, profile FROM audit"); + } + db.commit(); + return true; + } catch (const SqlError&) { + return false; + } +} + +} // namespace detz::store diff --git a/taicpp/src/store/schema.h b/taicpp/src/store/schema.h new file mode 100644 index 0000000..09c20ea --- /dev/null +++ b/taicpp/src/store/schema.h @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include "sqldb.h" +namespace detz::store { + +// The vault schema of tai.py: SCHEMA, SCHEMA_INDEXES, ensure_schema, +// migrate_profiles, rebuild_with_profile, setup_fts, ensure_column. +class Schema { + public: + struct Table { + std::string name; + std::vector columns; // "name TYPE [constraint]" + std::vector pk; // composite primary key, empty when the column carries it + }; + struct Index { std::string name, target; }; + + static const std::vector
& tables(); + static const std::vector& indexes(); + // creates missing tables, adds missing columns, creates indexes; returns "added table.column" notes + static std::vector ensure_schema(SqlDb& db); + // adds the profile column everywhere and rebuilds the composite-key tables + static void migrate_profiles(SqlDb& db); + // fts_docs table plus the sync triggers; false when sqlite lacks FTS5 + static bool setup_fts(SqlDb& db); + static void ensure_column(SqlDb& db, const std::string& table, const std::string& column, const std::string& ddl); + static std::vector table_pk_columns(SqlDb& db, const std::string& table); + static bool rebuild_with_profile(SqlDb& db, const std::string& table, const std::vector& columns, const std::vector& pk); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/seal_port.h b/taicpp/src/store/seal_port.h new file mode 100644 index 0000000..d9f34d4 --- /dev/null +++ b/taicpp/src/store/seal_port.h @@ -0,0 +1,31 @@ +#pragma once +#include +#include "errors.h" +namespace detz::store { + +// The surface of tai.py class Seal that Store needs. The vault crypto itself +// (PBKDF2 + HMAC-SHA256 stream, PREFIX "tai1$") is not owned by the +// foundation area: it plugs in by implementing this interface. NullSeal is +// the python "seal disabled" path (TAI_PASSPHRASE set to the empty string). +class SealPort { + public: + static constexpr const char* kPrefix = "tai1$"; + virtual ~SealPort() = default; + virtual bool enabled() const = 0; + virtual bool default_key() const { return false; } + virtual std::string lock(const std::string& text) const = 0; + virtual std::string unlock(const std::string& text) const = 0; // throws SealError + std::string safe_unlock(const std::string& text) const { + try { return unlock(text); } catch (const SealError&) { return "[sealed: decrypt failed]"; } + } + static bool is_sealed(const std::string& text) { return text.rfind(kPrefix, 0) == 0; } +}; + +class NullSeal : public SealPort { + public: + bool enabled() const override { return false; } + std::string lock(const std::string& text) const override { return text; } + std::string unlock(const std::string&) const override { throw SealError("not sealed"); } +}; + +} // namespace detz::store diff --git a/taicpp/src/store/sha256.cpp b/taicpp/src/store/sha256.cpp new file mode 100644 index 0000000..3db90fe --- /dev/null +++ b/taicpp/src/store/sha256.cpp @@ -0,0 +1,55 @@ +#include "sha256.h" +#include +#include +namespace detz::store { + +static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static inline uint32_t rotr(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } + +std::string Sha256::hex(const std::string& data) { + uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + std::string msg = data; + uint64_t bits = static_cast(data.size()) * 8; + msg += static_cast(0x80); + while (msg.size() % 64 != 56) msg += '\0'; + for (int k = 7; k >= 0; k--) msg += static_cast((bits >> (k * 8)) & 0xff); + for (size_t off = 0; off < msg.size(); off += 64) { + uint32_t w[64]; + for (int t = 0; t < 16; t++) { + size_t b = off + static_cast(t) * 4; + w[t] = (static_cast(static_cast(msg[b])) << 24) | (static_cast(static_cast(msg[b + 1])) << 16) | + (static_cast(static_cast(msg[b + 2])) << 8) | static_cast(static_cast(msg[b + 3])); + } + for (int t = 16; t < 64; t++) { + uint32_t s0 = rotr(w[t - 15], 7) ^ rotr(w[t - 15], 18) ^ (w[t - 15] >> 3); + uint32_t s1 = rotr(w[t - 2], 17) ^ rotr(w[t - 2], 19) ^ (w[t - 2] >> 10); + w[t] = w[t - 16] + s0 + w[t - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int t = 0; t < 64; t++) { + uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + S1 + ch + K[t] + w[t]; + uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t mj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + mj; + hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh; + } + std::string out; + char buf[16]; + for (int i = 0; i < 8; i++) { std::snprintf(buf, sizeof buf, "%08x", h[i]); out += buf; } + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/sha256.h b/taicpp/src/store/sha256.h new file mode 100644 index 0000000..fe367c3 --- /dev/null +++ b/taicpp/src/store/sha256.h @@ -0,0 +1,12 @@ +#pragma once +#include +namespace detz::store { + +// Self-contained SHA-256 (FIPS 180-4) so the store needs no crypto library. +class Sha256 { + public: + // lowercase hex digest of data (python hashlib.sha256(data).hexdigest()) + static std::string hex(const std::string& data); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/spill.cpp b/taicpp/src/store/spill.cpp new file mode 100644 index 0000000..cacd759 --- /dev/null +++ b/taicpp/src/store/spill.cpp @@ -0,0 +1,82 @@ +#include "spill.h" +#include +#include "store.h" +#include "utf8.h" +namespace detz::store { + +std::string Spill::truncate(const std::string& text, size_t head, size_t tail) { + size_t len = Utf8::length(text); + if (len <= head + tail + 50) return text; + size_t skipped = len - head - tail; + return Utf8::prefix(text, head) + "\n...[" + std::to_string(skipped) + " chars truncated]...\n" + Utf8::suffix(text, tail); +} + +std::vector Spill::splitlines(const std::string& text) { + std::vector out; + std::string cur; + size_t i = 0, n = text.size(); + auto starts = [&](const char* seq, size_t len) { return i + len <= n && text.compare(i, len, seq) == 0; }; + bool pending = false; // a line is open (has content or was started) + while (i < n) { + unsigned char c = static_cast(text[i]); + size_t brk = 0; + if (c == '\n' || c == 0x0b || c == 0x0c || c == 0x1c || c == 0x1d || c == 0x1e) brk = 1; + else if (c == '\r') brk = (i + 1 < n && text[i + 1] == '\n') ? 2 : 1; + else if (starts("\xc2\x85", 2)) brk = 2; + else if (starts("\xe2\x80\xa8", 3) || starts("\xe2\x80\xa9", 3)) brk = 3; + if (brk) { + out.push_back(cur); + cur.clear(); + pending = false; + i += brk; + } else { + cur += text[i++]; + pending = true; + } + } + if (pending) out.push_back(cur); + return out; +} + +std::vector Spill::chunk_lines(const std::string& text, int max_lines, int overlap_lines, int max_chars) { + auto lines = splitlines(text); + int n = static_cast(lines.size()); + std::vector chunks; + if (n == 0) return chunks; + int start = 0; + while (start < n) { + int end = start, count = 0; + long size = 0; + while (end < n && count < max_lines && size < max_chars) { + size += static_cast(Utf8::length(lines[static_cast(end)])) + 1; + count++; + end++; + } + std::string piece; + for (int k = start; k < end; k++) { if (k > start) piece += "\n"; piece += lines[static_cast(k)]; } + piece = Utf8::strip(piece); + if (!piece.empty()) chunks.push_back(piece); + if (end >= n) break; + start = std::max(start + 1, end - overlap_lines); + } + return chunks; +} + +std::string Spill::spill_output(Store& store, const std::string& kind, const std::string& title, const std::string& text, const std::vector& tags, const std::string& profile) { + std::string prof = profile.empty() ? store.profile : profile; + std::string record_id = store.add_record(kind, title, text, tags, prof); + store.add_chunks("record", record_id, chunk_lines(text), prof); + return record_id; +} + +std::string Spill::spilled_view(const std::string& text, const std::string& record_id, size_t size, size_t head, size_t tail) { + return Utf8::prefix(text, head) + "\n...[" + std::to_string(size) + " chars spilled to " + record_id + ", record_read pages the rest]...\n" + Utf8::suffix(text, tail); +} + +std::string Spill::spill_or_truncate(Store* store, const std::string& kind, const std::string& title, const std::string& text, const std::vector& tags, const std::string& profile) { + if (!store || Utf8::length(text) <= static_cast(kLimit)) return truncate(text); + std::string record_id = spill_output(*store, kind, title, text, tags, profile); + return spilled_view(text, record_id, Utf8::length(text)); +} + +} // namespace detz::store diff --git a/taicpp/src/store/spill.h b/taicpp/src/store/spill.h new file mode 100644 index 0000000..0ce57a9 --- /dev/null +++ b/taicpp/src/store/spill.h @@ -0,0 +1,28 @@ +#pragma once +#include +#include +namespace detz::store { + +class Store; + +// Large tool output goes into the vault as a record and the agent sees a +// head/tail view (tai.py spill_output, spilled_view, spill_or_truncate, +// truncate, chunk_lines). Lengths and slices count code points like python. +class Spill { + public: + static constexpr int kLimit = 6000; // SPILL_LIMIT + static constexpr int kChunkMaxLines = 60; // CHUNK_MAX_LINES + static constexpr int kChunkOverlap = 3; // CHUNK_OVERLAP_LINES + static constexpr int kChunkMaxChars = 4000;// CHUNK_MAX_CHARS + + static std::string truncate(const std::string& text, size_t head = 1500, size_t tail = 500); + // python str.splitlines() + static std::vector splitlines(const std::string& text); + static std::vector chunk_lines(const std::string& text, int max_lines = kChunkMaxLines, int overlap_lines = kChunkOverlap, int max_chars = kChunkMaxChars); + static std::string spill_output(Store& store, const std::string& kind, const std::string& title, const std::string& text, const std::vector& tags = {}, const std::string& profile = ""); + static std::string spilled_view(const std::string& text, const std::string& record_id, size_t size, size_t head = 1500, size_t tail = 500); + // store == nullptr behaves as python store=None + static std::string spill_or_truncate(Store* store, const std::string& kind, const std::string& title, const std::string& text, const std::vector& tags = {}, const std::string& profile = ""); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/sqldb.cpp b/taicpp/src/store/sqldb.cpp new file mode 100644 index 0000000..157253b --- /dev/null +++ b/taicpp/src/store/sqldb.cpp @@ -0,0 +1,149 @@ +#include "sqldb.h" +#include +#include +#include +#include +namespace detz::store { + +std::string Cell::text() const { + switch (type) { + case Type::Text: return s; + case Type::Int: return std::to_string(i); + case Type::Real: { char b[64]; std::snprintf(b, sizeof b, "%.17g", d); return b; } + default: return ""; + } +} +int64_t Cell::as_int() const { + if (type == Type::Int) return i; + if (type == Type::Real) return static_cast(d); + if (type == Type::Text) return std::strtoll(s.c_str(), nullptr, 10); + return 0; +} +double Cell::as_real() const { + if (type == Type::Real) return d; + if (type == Type::Int) return static_cast(i); + if (type == Type::Text) return std::strtod(s.c_str(), nullptr); + return 0; +} + +SqlDb::SqlDb(const std::string& path, double timeout_seconds) { + if (sqlite3_open_v2(path.c_str(), &db_, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, nullptr) != SQLITE_OK) { + std::string msg = db_ ? sqlite3_errmsg(db_) : "cannot open database"; + if (db_) sqlite3_close(db_); + db_ = nullptr; + throw SqlError(msg); + } + sqlite3_busy_timeout(db_, static_cast(timeout_seconds * 1000)); +} +SqlDb::~SqlDb() { close(); } + +void SqlDb::close() { + if (!db_) return; + sqlite3_close(db_); + db_ = nullptr; + for (auto* f : fns_) delete f; + fns_.clear(); +} + +bool SqlDb::in_transaction() const { return db_ && !sqlite3_get_autocommit(db_); } + +void SqlDb::begin_if_dml(const std::string& sql) { + size_t p = 0; + while (p < sql.size() && std::isspace(static_cast(sql[p]))) p++; + std::string w; + while (p < sql.size() && std::isalpha(static_cast(sql[p])) && w.size() < 8) w += static_cast(std::toupper(static_cast(sql[p++]))); + if ((w == "INSERT" || w == "UPDATE" || w == "DELETE" || w == "REPLACE") && !in_transaction()) { + char* err = nullptr; + if (sqlite3_exec(db_, "BEGIN", nullptr, nullptr, &err) != SQLITE_OK) { + std::string m = err ? err : "BEGIN failed"; + sqlite3_free(err); + throw SqlError(m); + } + } +} + +static void bind_all(sqlite3_stmt* st, const Params& params, sqlite3* db) { + for (size_t k = 0; k < params.size(); k++) { + const Cell& c = params[k]; + int idx = static_cast(k) + 1; + int rc = SQLITE_OK; + switch (c.type) { + case Cell::Type::Null: rc = sqlite3_bind_null(st, idx); break; + case Cell::Type::Int: rc = sqlite3_bind_int64(st, idx, c.i); break; + case Cell::Type::Real: rc = sqlite3_bind_double(st, idx, c.d); break; + case Cell::Type::Text: rc = sqlite3_bind_text(st, idx, c.s.data(), static_cast(c.s.size()), SQLITE_TRANSIENT); break; + } + if (rc != SQLITE_OK) throw SqlError(sqlite3_errmsg(db)); + } +} + +Cursor SqlDb::execute(const std::string& sql, const Params& params) { + if (!db_) throw SqlError("Cannot operate on a closed database."); + begin_if_dml(sql); + sqlite3_stmt* st = nullptr; + if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &st, nullptr) != SQLITE_OK) throw SqlError(sqlite3_errmsg(db_)); + Cursor cur; + try { + bind_all(st, params, db_); + int cols = sqlite3_column_count(st); + int rc; + while ((rc = sqlite3_step(st)) == SQLITE_ROW) { + Row row; + row.reserve(static_cast(cols)); + for (int c = 0; c < cols; c++) { + switch (sqlite3_column_type(st, c)) { + case SQLITE_INTEGER: row.emplace_back(static_cast(sqlite3_column_int64(st, c))); break; + case SQLITE_FLOAT: row.emplace_back(sqlite3_column_double(st, c)); break; + case SQLITE_NULL: row.emplace_back(); break; + default: { + const unsigned char* t = sqlite3_column_text(st, c); + int n = sqlite3_column_bytes(st, c); + row.emplace_back(std::string(reinterpret_cast(t ? t : reinterpret_cast("")), static_cast(n))); + } + } + } + cur.rows.push_back(std::move(row)); + } + if (rc != SQLITE_DONE) throw SqlError(sqlite3_errmsg(db_)); + cur.rowcount = sqlite3_changes(db_); + cur.lastrowid = sqlite3_last_insert_rowid(db_); + } catch (...) { + sqlite3_finalize(st); + throw; + } + sqlite3_finalize(st); + return cur; +} + +void SqlDb::executemany(const std::string& sql, const std::vector& batch) { + for (auto& p : batch) execute(sql, p); +} + +void SqlDb::commit() { + if (!db_) throw SqlError("Cannot operate on a closed database."); + if (in_transaction()) execute("COMMIT"); +} +void SqlDb::rollback() { + if (!db_) throw SqlError("Cannot operate on a closed database."); + if (in_transaction()) execute("ROLLBACK"); +} + +void SqlDb::create_function(const std::string& name, TextFn fn) { + auto* holder = new TextFn(std::move(fn)); + fns_.push_back(holder); + auto tramp = [](sqlite3_context* ctx, int argc, sqlite3_value** argv) { + auto* f = static_cast(sqlite3_user_data(ctx)); + if (argc < 1 || sqlite3_value_type(argv[0]) == SQLITE_NULL) { sqlite3_result_null(ctx); return; } + const unsigned char* t = sqlite3_value_text(argv[0]); + std::string in(reinterpret_cast(t ? t : reinterpret_cast("")), static_cast(sqlite3_value_bytes(argv[0]))); + try { + std::string out = (*f)(in); + sqlite3_result_text(ctx, out.data(), static_cast(out.size()), SQLITE_TRANSIENT); + } catch (const std::exception& e) { + sqlite3_result_error(ctx, e.what(), -1); + } + }; + if (sqlite3_create_function(db_, name.c_str(), 1, SQLITE_UTF8, holder, tramp, nullptr, nullptr) != SQLITE_OK) throw SqlError(sqlite3_errmsg(db_)); +} + +} // namespace detz::store diff --git a/taicpp/src/store/sqldb.h b/taicpp/src/store/sqldb.h new file mode 100644 index 0000000..b7b4fbc --- /dev/null +++ b/taicpp/src/store/sqldb.h @@ -0,0 +1,71 @@ +#pragma once +#include +#include +#include +#include +#include "errors.h" +struct sqlite3; +namespace detz::store { + +// One sqlite value as python's sqlite3 module sees it. +struct Cell { + enum class Type { Null, Int, Real, Text }; + Type type{Type::Null}; + int64_t i{0}; + double d{0}; + std::string s; + Cell() = default; + Cell(std::nullptr_t) {} + Cell(int v) : type(Type::Int), i(v) {} + Cell(long v) : type(Type::Int), i(v) {} + Cell(long long v) : type(Type::Int), i(v) {} + Cell(unsigned long v) : type(Type::Int), i(static_cast(v)) {} + Cell(bool v) : type(Type::Int), i(v ? 1 : 0) {} + Cell(double v) : type(Type::Real), d(v) {} + Cell(const char* v) : type(Type::Text), s(v) {} + Cell(std::string v) : type(Type::Text), s(std::move(v)) {} + bool is_null() const { return type == Type::Null; } + // text view of the value, "" for NULL (callers test is_null first where it matters) + std::string text() const; + int64_t as_int() const; + double as_real() const; +}; +using Row = std::vector; +using Params = std::vector; + +struct Cursor { + std::vector rows; + int64_t rowcount{0}; + int64_t lastrowid{0}; + const Row* first() const { return rows.empty() ? nullptr : &rows.front(); } +}; + +// Thin owner of one sqlite3 connection with python sqlite3 semantics: +// DML opens an implicit transaction (BEGIN) that commit()/rollback() ends, +// DDL does not; sqlite errors surface as SqlError (sqlite3.Error). +class SqlDb { + public: + using TextFn = std::function; + explicit SqlDb(const std::string& path, double timeout_seconds = 5.0); + ~SqlDb(); + SqlDb(const SqlDb&) = delete; + SqlDb& operator=(const SqlDb&) = delete; + + Cursor execute(const std::string& sql, const Params& params = {}); + void executemany(const std::string& sql, const std::vector& batch); + void commit(); + void rollback(); + void close(); + bool closed() const { return db_ == nullptr; } + bool in_transaction() const; + // scalar text function (python create_function(name, 1, fn)); exceptions become NULL-free sqlite errors + void create_function(const std::string& name, TextFn fn); + sqlite3* raw() { return db_; } + + private: + void begin_if_dml(const std::string& sql); + sqlite3* db_{nullptr}; + std::vector fns_; +}; + +} // namespace detz::store diff --git a/taicpp/src/store/store.cpp b/taicpp/src/store/store.cpp index 876b464..8732b6b 100644 --- a/taicpp/src/store/store.cpp +++ b/taicpp/src/store/store.cpp @@ -1,138 +1,392 @@ #include "store.h" -#include -#include +#include +#include +#include +#include +#include "clock.h" +#include "fts.h" +#include "schema.h" +#include "tag_rules.h" +#include "utf8.h" +#include "vault_files.h" namespace detz::store { -static void chk(int rc, sqlite3* d, const char* what) { - if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW) - throw std::runtime_error(std::string(what) + ": " + (d ? sqlite3_errmsg(d) : "?")); -} -Store::Store(const std::string& path, const std::string& profile) : profile_(profile) { - if (sqlite3_open(path.c_str(), &db_) != SQLITE_OK) { db_ = nullptr; throw std::runtime_error("open db"); } - exec("PRAGMA journal_mode=WAL;"); exec("PRAGMA synchronous=NORMAL;"); exec("PRAGMA busy_timeout=30000;"); - migrate(); setup_fts(); -} -Store::~Store() { if (db_) sqlite3_close(db_); } -void Store::exec(const std::string& sql) { - char* err = nullptr; - if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err) != SQLITE_OK) { - std::string e = err ? err : "?"; sqlite3_free(err); throw std::runtime_error(e); + +namespace fs = std::filesystem; + +Store::Store(const Config& config, const SealPort& seal) : config_(config), seal_(seal) { + db_ = std::make_unique(config_.dbPath, 30.0); + db_->execute("PRAGMA journal_mode=WAL"); + db_->execute("PRAGMA synchronous=NORMAL"); + db_->execute("PRAGMA busy_timeout=30000"); + schema_notes = Schema::ensure_schema(*db_); + Schema::migrate_profiles(*db_); + db_->commit(); + profile = config_.profile; + fts_ok = Schema::setup_fts(*db_); + if (seal_.enabled()) { + const SealPort* sp = &seal_; + db_->create_function("tai_enc", [sp](const std::string& t) { return sp->lock(t); }); + db_->create_function("tai_dec", [sp](const std::string& t) { return sp->safe_unlock(t); }); + db_->execute("UPDATE events SET text = tai_enc(text) WHERE text NOT LIKE 'tai1$%'"); + db_->execute("UPDATE secrets SET value = tai_enc(value) WHERE value NOT LIKE 'tai1$%'"); + db_->execute("UPDATE secrets SET meta = tai_enc(COALESCE(meta, '{}')) WHERE meta IS NULL OR meta NOT LIKE 'tai1$%'"); + db_->execute("UPDATE schedules SET prompt = tai_enc(prompt) WHERE prompt NOT LIKE 'tai1$%'"); + db_->execute("UPDATE input_history SET text = tai_enc(text) WHERE text NOT LIKE 'tai1$%'"); + db_->execute("UPDATE logs SET message = tai_enc(message) WHERE message NOT LIKE 'tai1$%'"); + db_->execute("UPDATE logs SET meta_json = tai_enc(meta_json) WHERE meta_json NOT LIKE 'tai1$%'"); + std::vector sealed_keys; + for (auto& [key, spec] : SettingsCatalog()) if (spec.encrypted) sealed_keys.push_back(key); + if (!sealed_keys.empty()) { + std::string marks; + Params params; + for (auto& k : sealed_keys) { if (!marks.empty()) marks += ", "; marks += "?"; params.emplace_back(k); } + db_->execute("UPDATE settings SET value = tai_enc(value) WHERE key IN (" + marks + ") AND value IS NOT NULL AND value NOT LIKE 'tai1$%'", params); } -} -void Store::migrate() { - exec("CREATE TABLE IF NOT EXISTS schema_version(v INTEGER);"); - exec("CREATE TABLE IF NOT EXISTS records(id INTEGER PRIMARY KEY, kind TEXT, title TEXT, content TEXT, profile TEXT DEFAULT 'default', created TEXT DEFAULT (datetime('now')));"); - exec("CREATE TABLE IF NOT EXISTS chunks(id INTEGER PRIMARY KEY, item TEXT, ref_id INTEGER, body TEXT, profile TEXT DEFAULT 'default');"); - exec("CREATE TABLE IF NOT EXISTS logs(id INTEGER PRIMARY KEY, level TEXT, message TEXT, ts TEXT DEFAULT (datetime('now')));"); - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "SELECT COUNT(*) FROM schema_version;", -1, &s, nullptr); - int n = 0; - if (sqlite3_step(s) == SQLITE_ROW) n = sqlite3_column_int(s, 0); - sqlite3_finalize(s); - if (n == 0) exec("INSERT INTO schema_version(v) VALUES(1);"); -} -void Store::setup_fts() { - exec("CREATE VIRTUAL TABLE IF NOT EXISTS fts_docs USING fts5(item, kind, sub, title, body, profile);"); - exec("CREATE TRIGGER IF NOT EXISTS trg_records_ai AFTER INSERT ON records BEGIN INSERT INTO fts_docs(item,kind,sub,title,body,profile) VALUES(new.id,'record',new.kind,new.title,new.content,new.profile); END;"); - exec("CREATE TRIGGER IF NOT EXISTS trg_records_ad AFTER DELETE ON records BEGIN DELETE FROM fts_docs WHERE item=old.id AND kind='record'; END;"); - exec("CREATE TRIGGER IF NOT EXISTS trg_records_au AFTER UPDATE ON records WHEN old.title!=new.title OR old.content!=new.content BEGIN DELETE FROM fts_docs WHERE item=old.id AND kind='record'; INSERT INTO fts_docs(item,kind,sub,title,body,profile) VALUES(new.id,'record',new.kind,new.title,new.content,new.profile); END;"); -} -int Store::schema_version() const { - sqlite3_stmt* s = nullptr; int v = 0; - sqlite3_prepare_v2(db_, "SELECT v FROM schema_version LIMIT 1;", -1, &s, nullptr); - if (sqlite3_step(s) == SQLITE_ROW) v = sqlite3_column_int(s, 0); - sqlite3_finalize(s); return v; -} -int64_t Store::add_record(const std::string& kind, const std::string& title, const std::string& content) { - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "INSERT INTO records(kind,title,content,profile) VALUES(?,?,?,?);", -1, &s, nullptr); - sqlite3_bind_text(s, 1, kind.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(s, 2, title.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(s, 3, content.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(s, 4, profile_.c_str(), -1, SQLITE_TRANSIENT); - chk(sqlite3_step(s), db_, "insert"); sqlite3_finalize(s); - return sqlite3_last_insert_rowid(db_); -} -bool Store::update_record(int64_t id, const std::string& title, const std::string& content) { - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "UPDATE records SET title=?, content=? WHERE id=?;", -1, &s, nullptr); - sqlite3_bind_text(s, 1, title.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(s, 2, content.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(s, 3, id); - sqlite3_step(s); sqlite3_finalize(s); return sqlite3_changes(db_) > 0; -} -bool Store::delete_record(int64_t id) { - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "DELETE FROM records WHERE id=?;", -1, &s, nullptr); - sqlite3_bind_int64(s, 1, id); sqlite3_step(s); sqlite3_finalize(s); - return sqlite3_changes(db_) > 0; -} -std::optional Store::get_record(int64_t id) { - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "SELECT id,kind,title,content,profile,created FROM records WHERE id=?;", -1, &s, nullptr); - sqlite3_bind_int64(s, 1, id); - std::optional r; - if (sqlite3_step(s) == SQLITE_ROW) { - Record x; x.id = sqlite3_column_int64(s, 0); - x.kind = (const char*)sqlite3_column_text(s, 1); - x.title = (const char*)sqlite3_column_text(s, 2); - const unsigned char* c = sqlite3_column_text(s, 3); - x.content = c ? (const char*)c : ""; - x.profile = (const char*)sqlite3_column_text(s, 4); - const unsigned char* t = sqlite3_column_text(s, 5); - x.created = t ? (const char*)t : ""; - r = x; + db_->commit(); + seal_files(); + } else { + ensure_sealed_or_throw(); + } + if (seal_.enabled() && fts_ok) { + try { + memdb_ = std::make_unique(":memory:"); + try { + memdb_->execute("CREATE VIRTUAL TABLE mem_events USING fts5(item, kind, sub, title, body, profile, tokenize='porter unicode61 remove_diacritics 2')"); + } catch (const SqlError&) { + memdb_->execute("CREATE VIRTUAL TABLE mem_events USING fts5(item, kind, sub, title, body, profile)"); + } + int64_t floor_id = db_->execute("SELECT COALESCE(MAX(id), 0) FROM events").rows[0][0].as_int() - kMemEventsCap; + mem_events_max_ = std::max(0, floor_id); + sync_mem_events(); + } catch (const SqlError&) { + memdb_.reset(); } - sqlite3_finalize(s); return r; + } } -std::vector Store::search(const std::string& q, int limit) { - std::vector out; - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "SELECT item,kind,sub,title,snippet(fts_docs,4,'[',']','...',12) FROM fts_docs WHERE fts_docs MATCH ? LIMIT ?;", -1, &s, nullptr); - sqlite3_bind_text(s, 1, q.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_int(s, 2, limit < 1 ? 1 : (limit > 100 ? 100 : limit)); - while (sqlite3_step(s) == SQLITE_ROW) { - SearchHit h; - h.item = (const char*)sqlite3_column_text(s, 0); - h.kind = (const char*)sqlite3_column_text(s, 1); - h.sub = (const char*)sqlite3_column_text(s, 2); - h.title = (const char*)sqlite3_column_text(s, 3); - const unsigned char* sn = sqlite3_column_text(s, 4); - h.snippet = sn ? (const char*)sn : ""; - out.push_back(h); + +Store::~Store() { close(); } + +std::unique_ptr Store::Open(const std::string& path, const std::string& prof) { + Config c = Config::FromEnv(); + fs::path p((path.empty() || path == ":memory:") ? (fs::temp_directory_path() / ("detz-vault-" + Clock::uuid_hex().substr(0, 8) + "/memory.db")).string() : path); + fs::path home = p.has_parent_path() ? p.parent_path() : fs::path("."); + std::error_code ec; + fs::create_directories(home / "profiles", ec); + c.home = home.string(); + c.profilesDir = (home / "profiles").string(); + c.audioDir = (home / "audio").string(); + c.dbPath = p.string(); + c.historyPath = (home / "history").string(); + c.profile = prof; + static const NullSeal null_seal; + return std::make_unique(c, null_seal); +} + +void Store::ensure_sealed_or_throw() { + struct Check { const char* sql; const char* msg; }; + static const Check checks[] = { + {"SELECT COUNT(*) FROM events WHERE text LIKE 'tai1$%'", "memory is sealed, set TAI_PASSPHRASE"}, + {"SELECT COUNT(*) FROM secrets WHERE value LIKE 'tai1$%' OR meta LIKE 'tai1$%'", "secrets are sealed, set TAI_PASSPHRASE"}, + {"SELECT COUNT(*) FROM schedules WHERE prompt LIKE 'tai1$%'", "schedules are sealed, set TAI_PASSPHRASE"}, + {"SELECT COUNT(*) FROM input_history WHERE text LIKE 'tai1$%'", "input history is sealed, set TAI_PASSPHRASE"}, + {"SELECT COUNT(*) FROM settings WHERE value LIKE 'tai1$%'", "settings are sealed, set TAI_PASSPHRASE"}, + {"SELECT COUNT(*) FROM mcp_servers WHERE headers_json LIKE 'tai1$%'", "mcp headers are sealed, set TAI_PASSPHRASE"}, + }; + for (auto& c : checks) if (db_->execute(c.sql).rows[0][0].as_int()) throw SealError(c.msg); +} + +void Store::close() { + if (db_ && !db_->closed()) { + try { db_->execute("PRAGMA wal_checkpoint(TRUNCATE)"); } catch (const SqlError&) {} + try { db_->close(); } catch (const SqlError&) {} + } + if (memdb_) { + try { memdb_->close(); } catch (const SqlError&) {} + memdb_.reset(); + } +} + +std::string Store::dec_col(const char* col) const { + return seal_.enabled() ? std::string("tai_dec(") + col + ")" : std::string(col); +} + +void Store::sync_mem_events() { + if (!memdb_) return; + std::vector rows; + try { + rows = db_->execute("SELECT id, profile, ts, role, kind, tai_dec(text) FROM events WHERE id > ? ORDER BY id ASC", {mem_events_max_}).rows; + } catch (const SqlError&) { + return; + } + if (rows.empty()) return; + try { + for (auto& r : rows) { + memdb_->execute("INSERT INTO mem_events(item, kind, sub, title, body, profile) VALUES (?, 'event', ? || '/' || ?, ? || ' ' || ? || '/' || ?, ?, ?)", + {r[0], r[3], r[4], r[2], r[3], r[4], r[5], r[1]}); } - sqlite3_finalize(s); return out; + memdb_->commit(); + } catch (const SqlError&) { + return; + } + mem_events_max_ = rows.back()[0].as_int(); + try { + memdb_->execute("DELETE FROM mem_events WHERE CAST(item AS INTEGER) <= ?", {mem_events_max_ - kMemEventsCap}); + memdb_->commit(); + } catch (const SqlError&) {} } -void Store::add_chunks(const std::string& item, int64_t id, const std::string& text) { - sqlite3_stmt* s = nullptr; - sqlite3_prepare_v2(db_, "INSERT INTO chunks(item,ref_id,body,profile) VALUES(?,?,?,?);", -1, &s, nullptr); - sqlite3_bind_text(s, 1, item.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(s, 2, id); - sqlite3_bind_text(s, 3, text.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(s, 4, profile_.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_step(s); sqlite3_finalize(s); + +void Store::seal_files() { + for (auto& ent : fs::directory_iterator(config_.profilesDir)) { + std::string entry = ent.path().filename().string(); + auto ends = [&](const char* suf) { size_t n = std::char_traits::length(suf); return entry.size() >= n && entry.compare(entry.size() - n, n, suf) == 0; }; + if (!(ends(".sys.md") || ends(".session.json") || ends(".bots.json"))) continue; + std::string path = ent.path().string(); + std::string raw; + try { raw = VaultFiles::read_text(path); } catch (const std::runtime_error&) { continue; } + if (SealPort::is_sealed(raw)) continue; + try { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) continue; + out << seal_.lock(raw); + } catch (const std::exception&) {} + } } -std::string truncate_text(const std::string& s, size_t limit) { - if (s.size() <= limit) return s; - return s.substr(0, limit) + "...[" + std::to_string(s.size()) + " chars truncated]..."; + +// ---------------------------------------------------------------- history + +std::optional Store::history_last(const std::string& prof) { + auto cur = db_->execute("SELECT " + dec_col("text") + " FROM input_history WHERE profile = ? ORDER BY id DESC LIMIT 1", {prof}); + if (cur.rows.empty()) return std::nullopt; + return cur.rows[0][0].text(); } -std::string spill_output(Store& st, const std::string& kind, const std::string& title, const std::string& text) { - int64_t id = st.add_record(kind, title, text); - st.add_chunks("record", id, text); - return std::to_string(id); + +bool Store::history_add(const std::string& prof, const std::string& bot, const std::string& text) { + std::string value = Utf8::prefix(Utf8::strip(text), kInputHistoryMaxChars); + if (value.empty()) return false; + try { + auto last = history_last(prof); + if (last && *last == value) return false; + if (seal_.enabled()) db_->execute("INSERT INTO input_history (profile, bot, ts, text) VALUES (?, ?, ?, tai_enc(?))", {prof, bot, Clock::now_iso(), value}); + else db_->execute("INSERT INTO input_history (profile, bot, ts, text) VALUES (?, ?, ?, ?)", {prof, bot, Clock::now_iso(), value}); + db_->execute("DELETE FROM input_history WHERE profile = ? AND id <= COALESCE((SELECT id FROM input_history WHERE profile = ? ORDER BY id DESC LIMIT 1 OFFSET ?), 0)", {prof, prof, kInputHistoryCap}); + db_->commit(); + return true; + } catch (const SqlError&) { + return false; + } } -std::string spilled_view(const std::string& text, int64_t rid, size_t head, size_t tail) { - if (text.size() <= head + tail) return text; - return text.substr(0, head) + "\n...[" + std::to_string(text.size()) + " chars spilled to " + std::to_string(rid) + "]...\n" + text.substr(text.size() - tail); + +std::vector Store::history_load(const std::string& prof, int limit) { + limit = std::max(1, std::min(limit, kInputHistoryCap)); + try { + int64_t count = db_->execute("SELECT COUNT(*) FROM input_history WHERE profile = ?", {prof}).rows[0][0].as_int(); + if (count == 0) history_import_file(prof, config_.historyPath); + auto rows = db_->execute("SELECT " + dec_col("text") + " FROM input_history WHERE profile = ? ORDER BY id DESC LIMIT ?", {prof, limit}).rows; + std::vector out; + for (auto it = rows.rbegin(); it != rows.rend(); ++it) out.push_back((*it)[0].text()); + return out; + } catch (const SqlError&) { + return {}; + } } -std::string spill_or_truncate(Store* st, const std::string& kind, const std::string& title, const std::string& text) { - const size_t LIM = 4000; - if (!st || text.size() <= LIM) return truncate_text(text, LIM); - int64_t id = st->add_record(kind, title, text); - st->add_chunks("record", id, text); - return spilled_view(text, id); + +int Store::history_import_file(const std::string& prof, const std::string& path) { + std::vector lines; + try { + std::ifstream in(path, std::ios::binary); + if (!in) return 0; + std::stringstream ss; + ss << in.rdbuf(); + std::string data = Utf8::sanitize(ss.str()); + // universal newlines, then keep non-blank lines with the trailing "\n" removed + std::string cur; + auto push = [&]() { + if (!Utf8::strip(cur).empty()) lines.push_back(cur); + cur.clear(); + }; + for (size_t i = 0; i < data.size(); i++) { + if (data[i] == '\r') { if (i + 1 < data.size() && data[i + 1] == '\n') i++; push(); } + else if (data[i] == '\n') push(); + else cur += data[i]; + } + if (!cur.empty()) push(); + } catch (const std::exception&) { + return 0; + } + if (lines.size() > static_cast(kInputHistoryCap)) lines.erase(lines.begin(), lines.end() - kInputHistoryCap); + int written = 0; + for (auto& l : lines) written += history_add(prof, "main", l) ? 1 : 0; + return written; } -void DebugLog::push(const std::string& l, const std::string& m) { lines_.push_back("[" + l + "] " + m); } -void DebugLog::info(const std::string& m) { push("INFO", m); } -void DebugLog::warn(const std::string& m) { push("WARN", m); } -void DebugLog::error(const std::string& m) { push("ERROR", m); } -} // namespace detz::store + +// ---------------------------------------------------------------- events + +void Store::log_event(const std::string& prof, const std::string& role, const std::string& kind, const std::string& text, const std::vector& tags) { + std::string value = Utf8::prefix(text, 2000); + std::string stamped = " " + role + " " + kind; + for (auto& t : TagRules::normalize_tags(tags)) stamped += " " + t; + stamped += " "; + try { + if (seal_.enabled()) db_->execute("INSERT INTO events (profile, ts, role, kind, text, tags) VALUES (?, ?, ?, ?, tai_enc(?), ?)", {prof, Clock::now_iso(), role, kind, value, stamped}); + else db_->execute("INSERT INTO events (profile, ts, role, kind, text, tags) VALUES (?, ?, ?, ?, ?, ?)", {prof, Clock::now_iso(), role, kind, value, stamped}); + db_->commit(); + sync_mem_events(); + } catch (const SqlError&) {} +} + +std::vector Store::log_query(const LogFilter& f) { + std::string query = "SELECT id, profile, bot, chat, ts, run, level, component, event, " + dec_col("message") + ", " + dec_col("meta_json") + " FROM logs"; + std::vector clauses; + Params params; + auto add = [&](const char* col, const std::string& v) { if (!v.empty()) { clauses.push_back(std::string(col) + " = ?"); params.emplace_back(v); } }; + add("level", f.level); add("component", f.component); add("event", f.event); add("profile", f.profile); + add("bot", f.bot); add("chat", f.chat); add("run", f.run); + if (!clauses.empty()) { + query += " WHERE "; + for (size_t k = 0; k < clauses.size(); k++) { if (k) query += " AND "; query += clauses[k]; } + } + query += " ORDER BY id DESC LIMIT ?"; + int lim = f.limit ? f.limit : 100; + params.emplace_back(std::max(1, std::min(lim, 1000))); + std::vector rows; + try { rows = db_->execute(query, params).rows; } catch (const SqlError&) { return {}; } + std::vector found; + for (auto& r : rows) { + Json meta = Json::object(); + try { + std::string raw = r[10].is_null() || r[10].text().empty() ? "{}" : r[10].text(); + Json m = Json::loads(raw); + if (m.is_object()) meta = m; + } catch (const JsonError&) {} + Json o = Json::object(); + o.set("id", static_cast(r[0].as_int())); + o.set("profile", r[1].text()); o.set("bot", r[2].text()); o.set("chat", r[3].text()); + o.set("ts", r[4].text()); o.set("run", r[5].text()); o.set("level", r[6].text()); + o.set("component", r[7].text()); o.set("event", r[8].text()); o.set("message", r[9].text()); + o.set("meta", meta); + found.push_back(o); + } + return found; +} + +// ---------------------------------------------------------------- search + +static FtsHit hit_from(const Row& r) { + FtsHit h; + h.item = r[0].text(); h.item_numeric = r[0].type == Cell::Type::Int; h.kind = r[1].text(); h.sub = r[2].text(); h.title = r[3].text(); h.snippet = r[4].text(); + h.rank = r[5].as_real(); + return h; +} + +std::vector Store::fts_search(const std::string& query, const std::vector& kinds, const std::string& prof_in, int limit) { + std::string prof = scope_(prof_in); + if (!fts_ok) return {}; + auto terms = Fts::escape(query); + if (!terms) return {}; + int capped = std::max(1, std::min(limit, 100)); + std::string rank = Fts::rank("fts_docs"); + std::string sql = "SELECT item, kind, sub, title, snippet(fts_docs, 4, '[', ']', '...', 12), " + rank + " FROM fts_docs WHERE fts_docs MATCH ? AND profile = ?"; + Params tail; + if (!kinds.empty()) { + std::string marks; + for (size_t k = 0; k < kinds.size(); k++) { if (k) marks += ", "; marks += "?"; tail.emplace_back(kinds[k]); } + sql += " AND kind IN (" + marks + ")"; + } + if (seal_.enabled()) sql += " AND kind != 'event' AND kind != 'log'"; + sql += " ORDER BY " + rank + " LIMIT ?"; + std::string mem_rank = Fts::rank("mem_events"); + std::string mem_sql = "SELECT item, kind, sub, title, snippet(mem_events, 4, '[', ']', '...', 12), " + mem_rank + " FROM mem_events WHERE mem_events MATCH ? AND profile = ? ORDER BY " + mem_rank + " LIMIT ?"; + for (auto& match : Fts::match_variants(*terms)) { + std::vector rows; + try { + Params p{match, prof}; + p.insert(p.end(), tail.begin(), tail.end()); + p.emplace_back(capped); + rows = db_->execute(sql, p).rows; + } catch (const SqlError&) { rows.clear(); } + std::vector hits; + for (auto& r : rows) hits.push_back(hit_from(r)); + bool has_event = kinds.empty() || std::find(kinds.begin(), kinds.end(), "event") != kinds.end(); + if (seal_.enabled() && memdb_ && has_event) { + sync_mem_events(); + std::vector mem_rows; + try { mem_rows = memdb_->execute(mem_sql, {match, prof, capped}).rows; } catch (const SqlError&) { mem_rows.clear(); } + for (auto& r : mem_rows) hits.push_back(hit_from(r)); + std::stable_sort(hits.begin(), hits.end(), [](const FtsHit& a, const FtsHit& b) { return a.rank < b.rank; }); + if (hits.size() > static_cast(capped)) hits.resize(static_cast(capped)); + } + if (!hits.empty()) return hits; + } + return {}; +} + +std::vector Store::audit_search(const std::string& query, const std::string& prof_in, int limit) { + std::string prof = scope_(prof_in); + auto hits = fts_search(query, {"audit"}, prof, limit); + std::vector found; + for (auto& hit : hits) { + size_t colon = hit.item.find(':'); + if (colon == std::string::npos) continue; + std::string tail = hit.item.substr(colon + 1); + if (tail.empty() || tail.find_first_not_of("0123456789") != std::string::npos) continue; + auto row = audit_get(std::stoll(tail), prof); + if (row) { row->set("snippet", hit.snippet); found.push_back(*row); } + } + return found; +} + +std::vector Store::search_events(const std::string& prof, const std::string& query, int limit, const std::vector& tags) { + auto clean = TagRules::normalize_tags(tags); + if (fts_ok) { + auto hits = fts_search(query, {"event"}, prof, std::max(limit * 5, 20)); + if (!hits.empty()) { + std::vector ids; + std::string marks; + Params params; + for (auto& h : hits) { ids.push_back(std::stoll(h.item)); if (!marks.empty()) marks += ", "; marks += "?"; params.emplace_back(static_cast(ids.back())); } + auto rows = db_->execute("SELECT id, ts, role, kind, " + dec_col("text") + ", tags FROM events WHERE id IN (" + marks + ")", params).rows; + std::map by_id; + for (auto& r : rows) by_id[r[0].as_int()] = r; + std::vector found; + for (auto& h : hits) { + auto it = by_id.find(std::stoll(h.item)); + if (it == by_id.end()) continue; + const Row& r = it->second; + if (!clean.empty()) { + std::string stamped = " " + (r[5].is_null() ? std::string() : r[5].text()) + " "; + bool all = true; + for (auto& t : clean) if (stamped.find(" " + t + " ") == std::string::npos) { all = false; break; } + if (!all) continue; + } + found.push_back({r[1].text(), r[2].text(), r[3].text(), r[4].text()}); + if (static_cast(found.size()) >= limit) break; + } + if (!found.empty()) return found; + } + } + return events_like(prof, query, limit, clean); +} + +std::vector Store::events_like(const std::string& prof, const std::string& query, int limit, const std::vector& tags) { + auto clean = TagRules::normalize_tags(tags); + std::string extra; + Params wild; + for (auto& t : clean) { extra += " AND tags LIKE ?"; wild.emplace_back("% " + t + " %"); } + std::vector rows; + if (seal_.enabled()) { + Params p{prof, "%" + query + "%"}; + p.insert(p.end(), wild.begin(), wild.end()); + p.emplace_back(limit); + rows = db_->execute("SELECT ts, role, kind, text FROM (SELECT id, ts, role, kind, tags, tai_dec(text) AS text FROM events WHERE profile = ? ORDER BY id DESC LIMIT 5000) WHERE text LIKE ?" + extra + " ORDER BY id DESC LIMIT ?", p).rows; + } else { + Params p{prof, "%" + query + "%"}; + p.insert(p.end(), wild.begin(), wild.end()); + p.emplace_back(limit); + rows = db_->execute("SELECT ts, role, kind, text FROM events WHERE profile = ? AND text LIKE ?" + extra + " ORDER BY id DESC LIMIT ?", p).rows; + } + std::vector out; + for (auto& r : rows) out.push_back({r[0].text(), r[1].text(), r[2].text(), r[3].text()}); + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/store.h b/taicpp/src/store/store.h index 24e2d06..494103d 100644 --- a/taicpp/src/store/store.h +++ b/taicpp/src/store/store.h @@ -1,41 +1,211 @@ #pragma once #include #include +#include +#include #include +#include #include +#include #include -struct sqlite3; +#include "config/config.h" +#include "errors.h" +#include "json.h" +#include "seal_port.h" +#include "sqldb.h" namespace detz::store { -struct Record { int64_t id{0}; std::string kind, title, content, profile, created; }; -struct SearchHit { std::string item, kind, sub, title, snippet; }; + +// One merged settings row (settings.md "Row shape"). +struct SettingRow { + std::string key, scope, type; + SettingValue def; + bool required{false}, nullable{false}, encrypted{false}; + std::vector options; + bool has_options{false}; + std::string env, desc; + std::optional stored; // decrypted canonical string, nullopt = null + SettingValue value; // effective typed value + std::string source; // env | db | default + std::optional created, updated; +}; + +// item is the fts_docs "item" column: an integer for events, text for everything else (item_numeric) +struct FtsHit { std::string item, kind, sub, title, snippet; double rank{0}; bool item_numeric{false}; }; +struct EventRow { std::string ts, role, kind, text; }; + +// Filters of Store::log_query; empty string = no filter. +struct LogFilter { + std::string level, component, event, profile, bot, chat, run; + int limit{100}; +}; + +// tai.py class Store: the sqlite vault (WAL, FTS5, per-profile slices) plus the +// profile / session / turn / bots files under /profiles. The vault crypto +// is injected as a SealPort. Methods keep the python names and argument order; +// an empty `profile` argument means "the store's current profile" (python +// profile=None). Defined across store*.cpp, one translation unit per concern. class Store { public: - explicit Store(const std::string& path = ":memory:", const std::string& profile = "default"); + // (stored value or null, created, updated) of one settings row + using RowTriple = std::optional, std::string, std::string>>; + static constexpr int kInputHistoryCap = 1000; + static constexpr size_t kInputHistoryMaxChars = 4000; + static constexpr int kMemEventsCap = 10000; + static constexpr int kAuditMaxChars = 20000; // AUDIT_MAX_CHARS + static constexpr int kFileRecordMax = 50000; // FILE_RECORD_MAX + static constexpr int kDirDefaultsCap = 512; // DIR_DEFAULTS_CAP + static constexpr int kMcpMaxServers = 32; // MCP_MAX_SERVERS + static constexpr int kFilesCap = 20; // ShellSession.FILES_CAP + + Store(const Config& config, const SealPort& seal); ~Store(); - void exec(const std::string& sql); - void migrate(); - void setup_fts(); - bool ok() const { return true; } - int schema_version() const; - int64_t add_record(const std::string& kind, const std::string& title, const std::string& content); - bool update_record(int64_t id, const std::string& title, const std::string& content); - bool delete_record(int64_t id); - std::optional get_record(int64_t id); - std::vector search(const std::string& q, int limit = 10); - void add_chunks(const std::string& item, int64_t id, const std::string& text); + Store(const Store&) = delete; + Store& operator=(const Store&) = delete; + + // Convenience for tools and tests: a vault at `path` ("" = a fresh private home) with no seal. + static std::unique_ptr Open(const std::string& path, const std::string& profile = "default"); + + const Config& config() const { return config_; } + const SealPort& seal() const { return seal_; } + SqlDb& db() { return *db_; } + std::string profile; + std::vector schema_notes; + bool fts_ok{false}; + void close(); + + // ---- input history (readline habit) ---- + std::optional history_last(const std::string& profile); + bool history_add(const std::string& profile, const std::string& bot, const std::string& text); + std::vector history_load(const std::string& profile, int limit = kInputHistoryCap); + int history_import_file(const std::string& profile, const std::string& path); + + // ---- events, logs, search ---- + void log_event(const std::string& profile, const std::string& role, const std::string& kind, const std::string& text, const std::vector& tags = {}); + std::vector log_query(const LogFilter& filter = {}); + std::vector fts_search(const std::string& query, const std::vector& kinds = {}, const std::string& profile = "", int limit = 10); + std::vector audit_search(const std::string& query, const std::string& profile = "", int limit = 20); + std::vector search_events(const std::string& profile, const std::string& query, int limit = 8, const std::vector& tags = {}); + std::vector events_like(const std::string& profile, const std::string& query, int limit = 8, const std::vector& tags = {}); + + // ---- secrets ---- + void save_secret(const std::string& name, const std::string& value, const Json& meta = Json(), const std::optional>& tags = std::nullopt, const std::string& profile = ""); + Json secret_meta(const std::string& name, const std::string& profile = ""); + std::vector list_secret_infos(const std::string& profile = ""); + std::optional load_secret(const std::string& name, const std::string& profile = ""); + bool delete_secret(const std::string& name, const std::string& profile = ""); + const std::map& secret_values(const std::string& profile = ""); + std::string redact(const std::string& text, const std::string& profile = ""); + + // ---- tags ---- + std::set known_tags(const std::string& profile = ""); + std::string canonical_tag(const std::string& tag, const std::set* vocab = nullptr, const std::string& profile = ""); + std::vector canonical_tags(const std::vector& tags, const std::set* vocab = nullptr, const std::string& profile = ""); + std::vector> tag_counts(const std::string& prefix = "", int limit = 50, const std::string& profile = ""); + std::vector auto_tags_for(const std::string& text, const std::vector& explicit_tags, const std::set* vocab = nullptr); + std::vector> tag_neighbors(const std::string& record_id, int limit = 0, const std::string& profile = ""); + void tag_item(const std::string& item, const std::vector& tags, const std::string& profile = ""); + void set_item_tags(const std::string& item, const std::vector& tags, const std::string& profile = ""); + std::vector item_tags(const std::string& item, const std::string& profile = ""); + // nullopt when `tags` normalises to nothing + std::optional> tagged_items(const std::vector& tags, const std::string& profile = ""); + + // ---- graph ---- + bool has_node(const std::string& kind, const std::string& key, const std::string& profile = ""); + std::string node_title(const std::string& node, const std::string& profile = ""); + std::string add_edge(const std::string& src, const std::string& dst, const std::string& relation = "linked", const std::string& profile = ""); + std::vector edges_for(const std::string& node, const std::string& profile = ""); + int64_t remove_edges(const std::string& node, const std::string& profile = ""); + std::vector traverse(const std::string& start, int depth = 2, int limit = 50, const std::string& profile = ""); + + // ---- records and chunks ---- + std::string add_record(const std::string& kind, const std::string& title, const std::string& content, const std::vector& tags = {}, const std::string& profile = ""); + void add_chunks(const std::string& parent_kind, const std::string& parent_id, const std::vector& texts, const std::string& profile = ""); + std::map get_chunks_by_ids(const std::vector& ids, const std::string& profile = ""); + std::optional get_record(const std::string& record_id, const std::string& profile = ""); + std::optional read_record(const std::string& record_id, int64_t offset = 0, int64_t limit = 4000, const std::string& profile = ""); + std::vector search_records(const std::string& query = "", const std::string& kind = "", const std::vector& tags = {}, int limit = 8, const std::string& profile = ""); + std::vector search_records_fts(const std::vector& terms, const std::string& kind, const std::vector& clean, int limit, const std::string& profile); + bool delete_record(const std::string& record_id, const std::string& profile = ""); + std::optional file_record_id(const std::string& path, const std::string& profile = ""); + // true_size < 0 = unknown (python None) + std::string upsert_file_record(const std::string& path, const std::string& content, int64_t true_size = -1, const std::string& profile = ""); + bool delete_file_record(const std::string& path, const std::string& profile = ""); + + // ---- audit trail ---- + // old/new nullopt = python None; sizes < 0 = None + int64_t audit_event(const std::string& actor, const std::string& action, const std::string& path, const std::string& message = "", const std::optional& old_text = std::nullopt, const std::optional& new_text = std::nullopt, const std::vector& tags = {}, int64_t old_size = -1, int64_t new_size = -1, const std::string& profile = ""); + std::vector audit_history(const std::string& path = "", const std::string& tag = "", int limit = 20, const std::string& profile = ""); + std::optional audit_get(int64_t row_id, const std::string& profile = ""); + + // ---- schedules ---- + int64_t add_schedule(const std::string& name, const std::string& prompt, const std::string& profile, int64_t every_sec, const std::string& next_run, int64_t timeout, const std::optional>& tags = std::nullopt); + bool remove_schedule(int64_t row_id, const std::string& profile = ""); + std::vector list_schedules(const std::string& profile = ""); + + // ---- profile / session / turn / bots files ---- + std::string profile_path(const std::string& name) const; + std::string session_path(const std::string& name, const std::string& ctx = "") const; + std::vector list_profiles() const; + std::string unseal_file(const std::string& raw) const; + std::optional load_system(const std::string& name); + void save_system(const std::string& name, const std::string& text); + // python load_session / save_session: only user/assistant/tool messages come back, system messages are never saved + std::vector load_session(const std::string& name, const std::string& bot = "main", const std::string& ctx = ""); + void save_session(const std::string& name, const std::vector& messages, const std::string& bot = "main", const std::string& ctx = ""); + void prune_ctx_files(const std::string& prefix, const std::string& legacy, int cap = 0); + std::string turn_state_path(const std::string& profile, const std::string& bot = "main", const std::string& ctx = "") const; + Json load_turn_state(const std::string& profile, const std::string& bot = "main", const std::string& ctx = ""); + void save_turn_state(const std::string& profile, const std::string& bot, const Json& state, const std::string& ctx = ""); + std::string bot_system_path(const std::string& profile, const std::string& bot) const; + std::string bot_session_path(const std::string& profile, const std::string& bot, const std::string& ctx = "") const; + std::string bots_registry_path(const std::string& profile) const; + Json load_bots(const std::string& profile); + void save_bots(const std::string& profile, const Json& bots); + std::vector list_bots(const std::string& profile); + std::optional resolve_bot(const std::string& profile, const std::string& mention); + std::optional load_bot_system(const std::string& profile, const std::string& bot); + void save_bot_system(const std::string& profile, const std::string& bot, const std::string& text); + + // ---- settings (settings.md) ---- + SettingRow settings_row(const std::string& key, const std::string& scope, const RowTriple& own, const RowTriple& fallback); + std::vector get_or_create_settings(const std::optional>& keys = std::nullopt, const std::string& scope = "global"); + std::vector get_all_settings(const std::string& scope = "global"); + std::vector upsert_settings(const std::map& items, const std::string& scope = "global"); + + // ---- MCP server registry ---- + void mcp_save_server(const Json& entry); + std::optional mcp_get_server(const std::string& name); + std::vector mcp_list_servers(); + void mcp_remove_server(const std::string& name); + void mcp_set_enabled(const std::string& server, const std::string& profile, bool enabled); + bool mcp_server_enabled(const std::string& server, const std::string& profile); + std::vector mcp_enabled_servers(const std::string& profile); + void mcp_store_tools(const std::string& server, const std::vector& tools); + std::vector mcp_cached_tools(const std::string& server); + std::map mcp_tool_pins(const std::string& server); + + // ---- per-directory defaults ---- + std::map load_dir_defaults(const std::string& directory); + // empty strings = python None; clear lists keys to drop + void save_dir_defaults(const std::string& directory, const std::string& provider = "", const std::string& model = "", const std::string& bot = "", const std::vector& clear = {}); + private: - sqlite3* db_{nullptr}; std::string profile_; -}; -std::string truncate_text(const std::string& s, std::size_t limit); -std::string spill_output(Store& st, const std::string& kind, const std::string& title, const std::string& text); -std::string spilled_view(const std::string& text, int64_t id, std::size_t head = 2000, std::size_t tail = 2000); -std::string spill_or_truncate(Store* st, const std::string& kind, const std::string& title, const std::string& text); -class DebugLog { - public: - void push(const std::string& level, const std::string& msg); - void info(const std::string& m); void warn(const std::string& m); void error(const std::string& m); - const std::vector& lines() const { return lines_; } - private: - std::vector lines_; + std::string scope_(const std::string& p) const { return p.empty() ? profile : p; } + std::string dec_col(const char* col) const; // "tai_dec(col)" when sealed else "col" + void sync_mem_events(); + void seal_files(); + void ensure_sealed_or_throw(); + std::map read_setting_rows(const std::string& scope); + std::optional read_unsealed_json_file(const std::string& path); + static Json mcp_parse(const std::string& raw, Json::Kind fallback_kind); + Json mcp_row(const Row& found); + + Config config_; + const SealPort& seal_; + std::unique_ptr db_; + std::unique_ptr memdb_; + int64_t mem_events_max_{0}; + std::map> secret_cache_; }; + } // namespace detz::store diff --git a/taicpp/src/store/store_files.cpp b/taicpp/src/store/store_files.cpp new file mode 100644 index 0000000..ec1770e --- /dev/null +++ b/taicpp/src/store/store_files.cpp @@ -0,0 +1,271 @@ +// Store: profile / session / turn-state / bots files under /profiles, plus dir_defaults. +#include +#include +#include +#include "clock.h" +#include "session/shell_context.h" +#include "store.h" +#include "utf8.h" +#include "vault_files.h" +namespace detz::store { + +namespace fs = std::filesystem; +using session::ShellContext; + +static std::string sid_of(const std::string& ctx) { return ctx.empty() ? "" : ShellContext::sanitize(ctx); } +static std::string join_path(const std::string& dir, const std::string& name) { return (fs::path(dir) / name).string(); } + +std::string Store::profile_path(const std::string& name) const { return join_path(config_.profilesDir, name + ".sys.md"); } + +std::string Store::session_path(const std::string& name, const std::string& ctx) const { + std::string sid = sid_of(ctx); + if (!sid.empty()) return join_path(config_.profilesDir, name + ".session." + sid + ".json"); + return join_path(config_.profilesDir, name + ".session.json"); +} + +std::vector Store::list_profiles() const { + std::vector entries; + for (auto& e : fs::directory_iterator(config_.profilesDir)) entries.push_back(e.path().filename().string()); + std::sort(entries.begin(), entries.end()); + std::vector names; + for (auto& entry : entries) { + if (entry.size() >= 7 && entry.compare(entry.size() - 7, 7, ".sys.md") == 0 && entry.find(".bot.") == std::string::npos) names.push_back(entry.substr(0, entry.size() - 7)); + } + return names; +} + +std::string Store::unseal_file(const std::string& raw) const { + if (SealPort::is_sealed(raw)) { + if (!seal_.enabled()) throw SealError("profile is sealed, set TAI_PASSPHRASE"); + return seal_.unlock(raw); + } + return raw; +} + +// Shared body of the load_* readers: read, unseal, hand the text to `parse`. +// OSError/ValueError (any runtime_error that is not a SealError) quarantines +// the file; a SealError quarantines it too unless the vault is unsealed, in +// which case it propagates (python "profile is sealed" refusal). +template +static T load_guarded(const Store& st, const std::string& path, T empty, Parse parse) { + try { + return parse(st.unseal_file(VaultFiles::read_text(path))); + } catch (const SealError&) { + if (!st.seal().enabled()) throw; + VaultFiles::quarantine(path); + return empty; + } catch (const std::runtime_error&) { + VaultFiles::quarantine(path); + return empty; + } +} + +std::optional Store::load_system(const std::string& name) { + std::string path = profile_path(name); + if (!VaultFiles::exists(path)) return std::nullopt; + return load_guarded>(*this, path, std::nullopt, [](const std::string& t) { return std::optional(t); }); +} + +void Store::save_system(const std::string& name, const std::string& text) { + try { VaultFiles::sealed_write(profile_path(name), seal_.lock(text)); } catch (const std::runtime_error&) {} +} + +std::vector Store::load_session(const std::string& name, const std::string& bot, const std::string& ctx) { + std::string path = bot == "main" ? session_path(name, ctx) : bot_session_path(name, bot, ctx); + if (!VaultFiles::exists(path)) return {}; + return load_guarded>(*this, path, {}, [](const std::string& t) { + Json data = Json::loads(t); + std::vector out; + if (!data.is_array()) return out; + for (auto& item : data.items()) { + if (!item.is_object()) continue; + const Json* role = item.find("role"); + if (role && role->is_string() && (role->as_string() == "user" || role->as_string() == "assistant" || role->as_string() == "tool")) out.push_back(item); + } + return out; + }); +} + +void Store::save_session(const std::string& name, const std::vector& messages, const std::string& bot, const std::string& ctx) { + std::string path = bot == "main" ? session_path(name, ctx) : bot_session_path(name, bot, ctx); + Json arr = Json::array(); + for (auto& m : messages) { + const Json* role = m.is_object() ? m.find("role") : nullptr; + if (role && role->is_string() && role->as_string() == "system") continue; + arr.push(m); + } + try { VaultFiles::sealed_write(path, seal_.lock(arr.dumps())); } catch (const std::runtime_error&) {} + if (!sid_of(ctx).empty()) { + std::string stem = bot == "main" ? name : name + ".bot." + bot; + prune_ctx_files(stem + ".session.", stem + ".session.json"); + } +} + +void Store::prune_ctx_files(const std::string& prefix, const std::string& legacy, int cap) { + std::vector paths; + try { + for (auto& e : fs::directory_iterator(config_.profilesDir)) { + std::string entry = e.path().filename().string(); + if (entry.rfind(prefix, 0) == 0 && entry.size() >= 5 && entry.compare(entry.size() - 5, 5, ".json") == 0 && entry != legacy) paths.push_back(e.path().string()); + } + } catch (const std::exception&) { + return; + } + size_t keep = static_cast(cap ? cap : kFilesCap); + if (paths.size() <= keep) return; + std::vector> by_mtime; + for (auto& p : paths) { + struct stat st; + if (::stat(p.c_str(), &st) != 0) return; + by_mtime.emplace_back(static_cast(st.st_mtim.tv_sec) + static_cast(st.st_mtim.tv_nsec) / 1e9, p); + } + std::stable_sort(by_mtime.begin(), by_mtime.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (size_t k = 0; k < by_mtime.size() - keep; k++) { + std::error_code ec; + fs::remove(by_mtime[k].second, ec); + } +} + +std::string Store::turn_state_path(const std::string& prof, const std::string& bot, const std::string& ctx) const { + std::string sid = sid_of(ctx); + std::string stem = bot == "main" ? prof : prof + ".bot." + bot; + if (!sid.empty()) return join_path(config_.profilesDir, stem + ".turn." + sid + ".json"); + return join_path(config_.profilesDir, stem + ".turn.json"); +} + +Json Store::load_turn_state(const std::string& prof, const std::string& bot, const std::string& ctx) { + std::string path = turn_state_path(prof, bot, ctx); + if (!VaultFiles::exists(path)) return Json::object(); + return load_guarded(*this, path, Json::object(), [](const std::string& t) { + Json d = Json::loads(t); + return d.is_object() ? d : Json::object(); + }); +} + +void Store::save_turn_state(const std::string& prof, const std::string& bot, const Json& state, const std::string& ctx) { + try { VaultFiles::sealed_write(turn_state_path(prof, bot, ctx), seal_.lock(state.dumps())); } catch (const std::runtime_error&) {} + if (!sid_of(ctx).empty()) { + std::string stem = bot == "main" ? prof : prof + ".bot." + bot; + prune_ctx_files(stem + ".turn.", stem + ".turn.json"); + } +} + +std::string Store::bot_system_path(const std::string& prof, const std::string& bot) const { return join_path(config_.profilesDir, prof + ".bot." + bot + ".sys.md"); } + +std::string Store::bot_session_path(const std::string& prof, const std::string& bot, const std::string& ctx) const { + std::string sid = sid_of(ctx); + if (!sid.empty()) return join_path(config_.profilesDir, prof + ".bot." + bot + ".session." + sid + ".json"); + return join_path(config_.profilesDir, prof + ".bot." + bot + ".session.json"); +} + +std::string Store::bots_registry_path(const std::string& prof) const { return join_path(config_.profilesDir, prof + ".bots.json"); } + +Json Store::load_bots(const std::string& prof) { + std::string path = bots_registry_path(prof); + if (!VaultFiles::exists(path)) return Json::object(); + return load_guarded(*this, path, Json::object(), [](const std::string& t) { + Json d = Json::loads(t); + const Json* bots = d.is_object() ? d.find("bots") : nullptr; + return (bots && bots->is_object()) ? *bots : Json::object(); + }); +} + +void Store::save_bots(const std::string& prof, const Json& bots) { + Json wrap = Json::object(); + wrap.set("bots", bots); + try { VaultFiles::sealed_write(bots_registry_path(prof), seal_.lock(wrap.dumps())); } catch (const std::runtime_error&) {} +} + +std::vector Store::list_bots(const std::string& prof) { + Json bots = load_bots(prof); + std::vector found; + Json main = Json::object(); + main.set("name", "main"); + main.set("nicknames", Json::array()); + found.push_back(main); + std::vector names; + for (auto& m : bots.members()) names.push_back(m.first); + std::sort(names.begin(), names.end()); + for (auto& name : names) { + const Json* entry = bots.find(name); + Json nick = Json::array(); + if (entry && entry->is_object()) { + const Json* n = entry->find("nicknames"); + if (n && n->is_array()) nick = *n; + } + Json o = Json::object(); + o.set("name", name); + o.set("nicknames", nick); + found.push_back(o); + } + return found; +} + +std::optional Store::resolve_bot(const std::string& prof, const std::string& mention) { + std::string want = Utf8::strip(mention); + for (auto& c : want) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + if (want.empty() || want == "main") return std::string("main"); + Json bots = load_bots(prof); + if (bots.has(want)) return want; + std::vector names; + for (auto& m : bots.members()) names.push_back(m.first); + std::sort(names.begin(), names.end()); + for (auto& name : names) { + const Json* entry = bots.find(name); + const Json* nick = (entry && entry->is_object()) ? entry->find("nicknames") : nullptr; + if (nick && nick->is_array()) { + for (auto& n : nick->items()) if (n.is_string() && n.as_string() == want) return name; + } + } + return std::nullopt; +} + +std::optional Store::load_bot_system(const std::string& prof, const std::string& bot) { + if (bot == "main") return load_system(prof); + std::string path = bot_system_path(prof, bot); + if (!VaultFiles::exists(path)) return std::nullopt; + return load_guarded>(*this, path, std::nullopt, [](const std::string& t) { return std::optional(t); }); +} + +void Store::save_bot_system(const std::string& prof, const std::string& bot, const std::string& text) { + try { VaultFiles::sealed_write(bot_system_path(prof, bot), seal_.lock(text)); } catch (const std::runtime_error&) {} +} + +// ---------------------------------------------------------------- dir defaults + +std::map Store::load_dir_defaults(const std::string& directory) { + std::map out; + try { + auto cur = db_->execute("SELECT provider, model, bot FROM dir_defaults WHERE dir = ?", {directory}); + if (cur.rows.empty()) return out; + const char* keys[] = {"provider", "model", "bot"}; + for (int k = 0; k < 3; k++) { + const Cell& c = cur.rows[0][static_cast(k)]; + if (!c.is_null() && !c.text().empty()) out[keys[k]] = c.text(); + } + } catch (const SqlError&) { + return {}; + } + return out; +} + +void Store::save_dir_defaults(const std::string& directory, const std::string& provider, const std::string& model, const std::string& bot, const std::vector& clear) { + if (directory.empty()) return; + auto current = load_dir_defaults(directory); + if (!provider.empty()) current["provider"] = provider; + if (!model.empty()) current["model"] = model; + if (!bot.empty()) current["bot"] = bot; + for (auto& k : clear) current.erase(k); + try { + if (current.empty()) { + db_->execute("DELETE FROM dir_defaults WHERE dir = ?", {directory}); + } else { + auto get = [&](const char* k) { auto it = current.find(k); return it == current.end() ? std::string() : it->second; }; + db_->execute("INSERT OR REPLACE INTO dir_defaults (dir, provider, model, bot, updated) VALUES (?, ?, ?, ?, ?)", {directory, get("provider"), get("model"), get("bot"), Clock::now_iso()}); + db_->execute("DELETE FROM dir_defaults WHERE dir NOT IN (SELECT dir FROM dir_defaults ORDER BY updated DESC LIMIT ?)", {kDirDefaultsCap}); + } + db_->commit(); + } catch (const SqlError&) {} +} + +} // namespace detz::store diff --git a/taicpp/src/store/store_mcp.cpp b/taicpp/src/store/store_mcp.cpp new file mode 100644 index 0000000..989b5fd --- /dev/null +++ b/taicpp/src/store/store_mcp.cpp @@ -0,0 +1,151 @@ +// Store: MCP server registry tables (mcp.md). +#include +#include "clock.h" +#include "patterns.h" +#include "sha256.h" +#include "store.h" +namespace detz::store { + +namespace { +Json text_or_null(const Cell& c) { return c.is_null() ? Json() : Json(c.text()); } +bool truthy(const Json* v) { + if (!v || v->is_null()) return false; + if (v->is_bool()) return v->as_bool(); + if (v->is_int()) return v->as_int() != 0; + if (v->is_float()) return v->as_double() != 0; + if (v->is_string()) return !v->as_string().empty(); + return v->size() > 0; +} +Json or_empty(const Json* v, Json::Kind kind) { + if (truthy(v)) return *v; + return kind == Json::Kind::Array ? Json::array() : Json::object(); +} +} // namespace + +Json Store::mcp_parse(const std::string& raw, Json::Kind fallback_kind) { + Json fallback = fallback_kind == Json::Kind::Array ? Json::array() : Json::object(); + try { + Json data = Json::loads(raw); + return data.kind() == fallback_kind ? data : fallback; + } catch (const JsonError&) { + return fallback; + } +} + +Json Store::mcp_row(const Row& f) { + std::string headers_json = f[6].is_null() ? "" : f[6].text(); + if (SealPort::is_sealed(headers_json)) headers_json = seal_.safe_unlock(headers_json); + auto s = [&](size_t i) { return f[i].is_null() ? std::string() : f[i].text(); }; + Json o = Json::object(); + o.set("name", f[0].text()); + o.set("transport", text_or_null(f[1])); + o.set("command", s(2)); + o.set("args", mcp_parse(s(3), Json::Kind::Array)); + o.set("env", mcp_parse(s(4), Json::Kind::Object)); + o.set("url", s(5)); + o.set("headers", mcp_parse(headers_json, Json::Kind::Object)); + o.set("cwd", s(7)); + o.set("allow", mcp_parse(s(8), Json::Kind::Array)); + o.set("deny", mcp_parse(s(9), Json::Kind::Array)); + o.set("source", s(10)); + o.set("created", text_or_null(f[11])); + o.set("updated", text_or_null(f[12])); + return o; +} + +static const char* kMcpCols = "name, transport, command, args_json, env_json, url, headers_json, cwd, allow_json, deny_json, source, created, updated"; + +void Store::mcp_save_server(const Json& entry) { + std::string name = entry.get_string("name", ""); + if (!Patterns::mcp_server(name)) throw ValueError("bad MCP server name '" + name + "'"); + auto known = db_->execute("SELECT created FROM mcp_servers WHERE name = ?", {name}); + if (known.rows.empty()) { + int64_t count = db_->execute("SELECT COUNT(*) FROM mcp_servers").rows[0][0].as_int(); + if (count >= kMcpMaxServers) throw ValueError("too many MCP servers (cap " + std::to_string(kMcpMaxServers) + ")"); + } + Json headers = or_empty(entry.find("headers"), Json::Kind::Object); + std::string sealed = truthy(entry.find("headers")) ? seal_.lock(headers.dumps(true)) : ""; + std::string now = Clock::now_iso(); + std::string created = known.rows.empty() ? now : known.rows[0][0].text(); + const Json* tr = entry.find("transport"); + Cell transport = (tr && tr->is_string()) ? Cell(tr->as_string()) : Cell(); + auto str_of = [&](const char* k) { const Json* v = entry.find(k); return (truthy(v) && v->is_string()) ? v->as_string() : std::string(); }; + db_->execute(std::string("INSERT OR REPLACE INTO mcp_servers (") + kMcpCols + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + {name, transport, str_of("command"), or_empty(entry.find("args"), Json::Kind::Array).dumps(), or_empty(entry.find("env"), Json::Kind::Object).dumps(), + str_of("url"), sealed, str_of("cwd"), or_empty(entry.find("allow"), Json::Kind::Array).dumps(), or_empty(entry.find("deny"), Json::Kind::Array).dumps(), + str_of("source"), created, now}); + db_->commit(); +} + +std::optional Store::mcp_get_server(const std::string& name) { + auto cur = db_->execute(std::string("SELECT ") + kMcpCols + " FROM mcp_servers WHERE name = ?", {name}); + if (cur.rows.empty()) return std::nullopt; + return mcp_row(cur.rows[0]); +} + +std::vector Store::mcp_list_servers() { + std::vector out; + for (auto& r : db_->execute(std::string("SELECT ") + kMcpCols + " FROM mcp_servers ORDER BY name").rows) out.push_back(mcp_row(r)); + return out; +} + +void Store::mcp_remove_server(const std::string& name) { + db_->execute("DELETE FROM mcp_server_tools WHERE server = ?", {name}); + db_->execute("DELETE FROM mcp_tool_pins WHERE server = ?", {name}); + db_->execute("DELETE FROM mcp_server_profiles WHERE server = ?", {name}); + db_->execute("DELETE FROM mcp_servers WHERE name = ?", {name}); + db_->commit(); +} + +void Store::mcp_set_enabled(const std::string& server, const std::string& prof, bool enabled) { + db_->execute("INSERT OR REPLACE INTO mcp_server_profiles (server, profile, enabled) VALUES (?, ?, ?)", {server, prof, enabled ? 1 : 0}); + db_->commit(); +} + +bool Store::mcp_server_enabled(const std::string& server, const std::string& prof) { + auto cur = db_->execute("SELECT enabled FROM mcp_server_profiles WHERE server = ? AND profile = ?", {server, prof}); + return !cur.rows.empty() && cur.rows[0][0].as_int() != 0; +} + +std::vector Store::mcp_enabled_servers(const std::string& prof) { + std::vector out; + for (auto& row : mcp_list_servers()) if (mcp_server_enabled(row.get_string("name"), prof)) out.push_back(row); + return out; +} + +void Store::mcp_store_tools(const std::string& server, const std::vector& tools) { + std::string now = Clock::now_iso(); + db_->execute("DELETE FROM mcp_server_tools WHERE server = ?", {server}); + db_->execute("DELETE FROM mcp_tool_pins WHERE server = ?", {server}); + for (auto& item : tools) { + if (!item.is_object() || !truthy(item.find("name"))) continue; + std::string tool = item.get_string("name"); + const Json* d = item.find("description"); + std::string desc = (truthy(d) && d->is_string()) ? d->as_string() : ""; + Json schema = or_empty(item.find("inputSchema"), Json::Kind::Object); + db_->execute("INSERT INTO mcp_server_tools (server, tool, description, schema_json, updated) VALUES (?, ?, ?, ?, ?)", {server, tool, desc, schema.dumps(), now}); + std::string blob = tool + "\n" + desc + "\n" + schema.dumps(true); + db_->execute("INSERT INTO mcp_tool_pins (server, tool, hash, updated) VALUES (?, ?, ?, ?)", {server, tool, Sha256::hex(blob).substr(0, 16), now}); + } + db_->commit(); +} + +std::vector Store::mcp_cached_tools(const std::string& server) { + std::vector out; + for (auto& r : db_->execute("SELECT tool, description, schema_json FROM mcp_server_tools WHERE server = ? ORDER BY tool", {server}).rows) { + Json o = Json::object(); + o.set("tool", r[0].text()); + o.set("description", r[1].is_null() ? std::string() : r[1].text()); + o.set("schema", mcp_parse(r[2].is_null() ? "" : r[2].text(), Json::Kind::Object)); + out.push_back(o); + } + return out; +} + +std::map Store::mcp_tool_pins(const std::string& server) { + std::map out; + for (auto& r : db_->execute("SELECT tool, hash FROM mcp_tool_pins WHERE server = ?", {server}).rows) out[r[0].text()] = r[1].text(); + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/store_settings.cpp b/taicpp/src/store/store_settings.cpp new file mode 100644 index 0000000..cf451d8 --- /dev/null +++ b/taicpp/src/store/store_settings.cpp @@ -0,0 +1,141 @@ +// Store: vault-backed settings (settings.md). +#include +#include +#include "clock.h" +#include "config/settings_rules.h" +#include "store.h" +namespace detz::store { + +namespace { +std::string env_get(const std::string& name) { + const char* v = std::getenv(name.c_str()); + return v ? v : ""; +} +} // namespace + +SettingRow Store::settings_row(const std::string& key, const std::string& scope, const RowTriple& own, const RowTriple& fallback) { + const SettingSpec& spec = SettingsCatalog().at(key); + std::optional stored; + std::optional created, updated; + if (own && std::get<0>(*own)) { + stored = std::get<0>(*own); created = std::get<1>(*own); updated = std::get<2>(*own); + } else if (fallback && std::get<0>(*fallback)) { + stored = std::get<0>(*fallback); created = std::get<1>(*fallback); updated = std::get<2>(*fallback); + } else if (own) { + created = std::get<1>(*own); updated = std::get<2>(*own); + } + if (spec.encrypted && stored && SealPort::is_sealed(*stored)) stored = seal_.safe_unlock(*stored); + std::string env_raw = env_get(spec.env); + std::string env_stripped = SettingsRules::PyStrip(env_raw); + SettingRow row; + if (!env_stripped.empty()) { + row.value = SettingsRules::ParseEnv(spec, env_stripped); + row.source = "env"; + } else if (stored) { + row.value = SettingsRules::ParseStored(spec, *stored); + row.source = "db"; + } else { + row.value = spec.def; + row.source = "default"; + } + if (spec.required && (row.value.IsNull() || (row.value.kind == SettingValue::Kind::String && row.value.s.empty()))) { + throw ValueError("setting '" + key + "' is required but unset"); + } + row.key = key; row.scope = scope; row.type = spec.type; row.def = spec.def; + row.required = spec.required; row.nullable = spec.nullable; row.options = spec.options; row.has_options = spec.hasOptions; + row.encrypted = spec.encrypted; row.env = spec.env; row.desc = spec.help; + row.stored = stored; row.created = created; row.updated = updated; + return row; +} + +std::map Store::read_setting_rows(const std::string& scope) { + std::map out; + try { + for (auto& r : db_->execute("SELECT key, value, created, updated FROM settings WHERE scope = ?", {scope}).rows) { + std::optional v; + if (!r[1].is_null()) v = r[1].text(); + out[r[0].text()] = std::make_tuple(v, r[2].text(), r[3].text()); + } + } catch (const SqlError&) { + return {}; + } + return out; +} + +static const Store::RowTriple* pick(const std::map& m, const std::string& k) { + auto it = m.find(k); + return it == m.end() ? nullptr : &it->second; +} + +std::vector Store::get_or_create_settings(const std::optional>& keys, const std::string& scope) { + SettingsRules::ValidateScope(scope); + std::vector names; + if (!keys) { for (auto& [k, _] : SettingsCatalog()) names.push_back(k); } + else names = *keys; + std::sort(names.begin(), names.end()); + for (auto& n : names) if (!SettingsCatalog().count(n)) throw ValueError("unknown setting '" + n + "'"); + auto own = read_setting_rows(scope); + std::map global_rows = scope != SettingsRules::kScopeGlobal ? read_setting_rows(SettingsRules::kScopeGlobal) : own; + std::string now = Clock::now_iso(); + bool created_any = false; + for (auto& n : names) { + if (!own.count(n)) { + db_->execute("INSERT INTO settings (key, scope, value, created, updated) VALUES (?, ?, NULL, ?, ?)", {n, scope, now, now}); + created_any = true; + } + } + if (created_any) { + db_->commit(); + own = read_setting_rows(scope); + } + std::vector rows; + for (auto& n : names) { + const RowTriple* o = pick(own, n); + const RowTriple* g = scope == SettingsRules::kScopeGlobal ? nullptr : pick(global_rows, n); + rows.push_back(settings_row(n, scope, o ? *o : RowTriple(), g ? *g : RowTriple())); + } + return rows; +} + +std::vector Store::get_all_settings(const std::string& scope) { + SettingsRules::ValidateScope(scope); + auto own = read_setting_rows(scope); + std::map global_rows = scope != SettingsRules::kScopeGlobal ? read_setting_rows(SettingsRules::kScopeGlobal) : own; + std::vector rows; + for (auto& [n, _] : SettingsCatalog()) { + const RowTriple* o = pick(own, n); + const RowTriple* g = scope == SettingsRules::kScopeGlobal ? nullptr : pick(global_rows, n); + rows.push_back(settings_row(n, scope, o ? *o : RowTriple(), g ? *g : RowTriple())); + } + return rows; +} + +std::vector Store::upsert_settings(const std::map& items, const std::string& scope) { + SettingsRules::ValidateScope(scope); + if (items.empty()) throw ValueError("upsert needs a non-empty {key: value} mapping"); + std::map> canonical; + for (auto& [key, value] : items) { + auto it = SettingsCatalog().find(key); + if (it == SettingsCatalog().end()) throw ValueError("unknown setting '" + key + "'"); + canonical[key] = SettingsRules::Normalize(key, it->second, value); + } + std::string now = Clock::now_iso(); + try { + for (auto& [key, val] : canonical) { + const SettingSpec& spec = SettingsCatalog().at(key); + Cell value = val ? Cell(spec.encrypted ? seal_.lock(*val) : *val) : Cell(); + auto exists = db_->execute("SELECT 1 FROM settings WHERE key = ? AND scope = ?", {key, scope}); + if (!exists.rows.empty()) db_->execute("UPDATE settings SET value = ?, updated = ? WHERE key = ? AND scope = ?", {value, now, key, scope}); + else db_->execute("INSERT INTO settings (key, scope, value, created, updated) VALUES (?, ?, ?, ?, ?)", {key, scope, value, now, now}); + } + db_->commit(); + } catch (const SqlError& e) { + db_->rollback(); + throw ValueError(std::string("settings upsert failed: ") + e.what()); + } + std::vector keys; + for (auto& [k, _] : canonical) keys.push_back(k); + return get_or_create_settings(keys, scope); +} + +} // namespace detz::store diff --git a/taicpp/src/store/store_vault.cpp b/taicpp/src/store/store_vault.cpp new file mode 100644 index 0000000..3d10966 --- /dev/null +++ b/taicpp/src/store/store_vault.cpp @@ -0,0 +1,698 @@ +// Store: secrets, tags, graph, records, chunks, audit trail, schedules. +#include +#include "clock.h" +#include "debuglog.h" +#include "fts.h" +#include "store.h" +#include "tag_rules.h" +#include "utf8.h" +namespace detz::store { + +namespace { +std::string marker(int64_t true_bytes) { return "\n...[truncated, full size " + std::to_string(true_bytes) + " bytes]..."; } +std::string marks_for(size_t n) { + std::string m; + for (size_t k = 0; k < n; k++) { if (k) m += ", "; m += "?"; } + return m; +} +Json text_or_null(const Cell& c) { return c.is_null() ? Json() : Json(c.text()); } +Json int_or_null(const Cell& c) { return c.is_null() ? Json() : Json(static_cast(c.as_int())); } +Json str_array(const std::vector& v) { + Json a = Json::array(); + for (auto& s : v) a.push(Json(s)); + return a; +} +} // namespace + +// ---------------------------------------------------------------- secrets + +void Store::save_secret(const std::string& name, const std::string& value, const Json& meta, const std::optional>& tags, const std::string& prof_in) { + std::string prof = scope_(prof_in); + secret_cache_.erase(prof); + std::string stored = seal_.enabled() ? seal_.lock(value) : value; + std::string blob = (meta.is_object() && meta.size() > 0) ? meta.dumps() : "{}"; + std::string locked = seal_.enabled() ? seal_.lock(blob) : blob; + db_->execute("INSERT OR REPLACE INTO secrets (profile, name, value, meta, updated) VALUES (?, ?, ?, ?, ?)", {prof, name, stored, locked, Clock::now_iso()}); + if (!tags) tag_item("secret:" + name, {"secret"}, prof); + else { + std::vector all{"secret"}; + all.insert(all.end(), tags->begin(), tags->end()); + set_item_tags("secret:" + name, all, prof); + } + db_->commit(); + Json m = Json::object(); + m.set("name", name); + DebugLog::Global().event("DEBUG", "vault", "secret_saved", "secret " + name + " saved", prof, "", "", m); +} + +static Json parse_meta_blob(const SealPort& seal, const std::string& stored) { + std::string raw = SealPort::is_sealed(stored) ? seal.safe_unlock(stored) : stored; + try { + Json d = Json::loads(raw); + return d.is_object() ? d : Json::object(); + } catch (const JsonError&) { + return Json::object(); + } +} + +Json Store::secret_meta(const std::string& name, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT meta FROM secrets WHERE profile = ? AND name = ?", {prof, name}); + if (cur.rows.empty() || cur.rows[0][0].is_null() || cur.rows[0][0].text().empty()) return Json::object(); + return parse_meta_blob(seal_, cur.rows[0][0].text()); +} + +std::vector Store::list_secret_infos(const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto rows = db_->execute("SELECT name, meta, updated FROM secrets WHERE profile = ? ORDER BY name", {prof}).rows; + std::vector items; + for (auto& r : rows) { + Json data = Json::object(); + if (!r[1].is_null() && !r[1].text().empty()) data = parse_meta_blob(seal_, r[1].text()); + Json o = Json::object(); + o.set("name", r[0].text()); + o.set("meta", data); + o.set("tags", str_array(item_tags("secret:" + r[0].text(), prof))); + o.set("updated", text_or_null(r[2])); + items.push_back(o); + } + return items; +} + +std::optional Store::load_secret(const std::string& name, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT value FROM secrets WHERE profile = ? AND name = ?", {prof, name}); + if (cur.rows.empty()) return std::nullopt; + std::string v = cur.rows[0][0].text(); + if (SealPort::is_sealed(v)) { + if (!seal_.enabled()) return std::nullopt; + return seal_.unlock(v); + } + return v; +} + +bool Store::delete_secret(const std::string& name, const std::string& prof_in) { + std::string prof = scope_(prof_in); + secret_cache_.erase(prof); + auto done = db_->execute("DELETE FROM secrets WHERE profile = ? AND name = ?", {prof, name}); + if (done.rowcount > 0) { + std::string item = "secret:" + name; + db_->execute("DELETE FROM tags WHERE item = ? AND profile = ?", {item, prof}); + db_->execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {prof, item, item}); + db_->commit(); + Json m = Json::object(); + m.set("name", name); + DebugLog::Global().event("DEBUG", "vault", "secret_deleted", "secret " + name + " deleted", prof, "", "", m); + return true; + } + db_->commit(); + return false; +} + +const std::map& Store::secret_values(const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto it = secret_cache_.find(prof); + if (it == secret_cache_.end()) { + std::map found; + for (auto& r : db_->execute("SELECT name, value FROM secrets WHERE profile = ?", {prof}).rows) { + std::string value = r[1].text(); + if (SealPort::is_sealed(value)) { + if (!seal_.enabled()) continue; + value = seal_.unlock(value); + } + if (Utf8::length(value) >= 4) found[r[0].text()] = value; + } + it = secret_cache_.emplace(prof, std::move(found)).first; + } + return it->second; +} + +std::string Store::redact(const std::string& in, const std::string& prof_in) { + std::string text = in; + std::vector> items(secret_values(prof_in).begin(), secret_values(prof_in).end()); + std::stable_sort(items.begin(), items.end(), [](const auto& a, const auto& b) { return Utf8::length(a.second) > Utf8::length(b.second); }); + for (auto& [name, value] : items) { + size_t pos = text.find(value); + if (pos == std::string::npos) continue; + std::string rep = "[redacted:" + name + "]"; + std::string out; + size_t from = 0; + while (pos != std::string::npos) { + out.append(text, from, pos - from); + out += rep; + from = pos + value.size(); + pos = text.find(value, from); + } + out.append(text, from, std::string::npos); + text = out; + } + return text; +} + +// ---------------------------------------------------------------- tags + +std::set Store::known_tags(const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::set s; + for (auto& r : db_->execute("SELECT DISTINCT tag FROM tags WHERE profile = ?", {prof}).rows) s.insert(r[0].text()); + return s; +} + +std::string Store::canonical_tag(const std::string& tag, const std::set* vocab, const std::string& prof) { + std::set own; + if (!vocab) { own = known_tags(prof); vocab = &own; } + if (vocab->count(tag)) return tag; + std::string single = TagRules::singular_noun(tag); + if (single != tag && vocab->count(single)) return single; + return tag; +} + +std::vector Store::canonical_tags(const std::vector& tags, const std::set* vocab, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::set own; + if (!vocab) { own = known_tags(prof); vocab = &own; } + std::vector seen; + for (auto& tag : TagRules::normalize_tags(tags)) { + std::string clean = canonical_tag(tag, vocab); + if (std::find(seen.begin(), seen.end(), clean) == seen.end()) seen.push_back(clean); + } + return seen; +} + +std::vector> Store::tag_counts(const std::string& prefix, int limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::string clean = TagRules::normalize_tag(prefix); + std::string sql = "SELECT tag, COUNT(*) FROM tags WHERE profile = ?"; + Params params{prof}; + if (!clean.empty()) { + sql += " AND tag LIKE ? ESCAPE '\\'"; + std::string esc; + for (char c : clean) { if (c == '\\' || c == '%' || c == '_') esc += '\\'; esc += c; } + params.emplace_back(esc + "%"); + } + sql += " GROUP BY tag ORDER BY COUNT(*) DESC, tag LIMIT ?"; + params.emplace_back(std::max(1, std::min(limit, 200))); + std::vector> out; + for (auto& r : db_->execute(sql, params).rows) out.emplace_back(r[0].text(), r[1].as_int()); + return out; +} + +std::vector Store::auto_tags_for(const std::string& text, const std::vector& explicit_tags, const std::set* vocab) { + std::set own; + if (!vocab) { own = known_tags(); vocab = &own; } + std::set skip(explicit_tags.begin(), explicit_tags.end()); + skip.insert("record"); + skip.insert("file"); + auto words = TagRules::tokens(text); + std::vector found; + auto has = [&](const std::string& s) { return std::find(found.begin(), found.end(), s) != found.end(); }; + for (size_t pos = 0; pos < words.size(); pos++) { + if (static_cast(found.size()) >= TagRules::kAutoMax) break; + const std::string& word = words[pos]; + if (pos + 1 < words.size()) { + std::string pair = word + "-" + words[pos + 1]; + if (vocab->count(pair) && !skip.count(pair) && !has(pair)) { found.push_back(pair); continue; } + } + if (vocab->count(word) && !skip.count(word) && !has(word)) { found.push_back(word); continue; } + std::string single = TagRules::singular_noun(word); + if (single != word && vocab->count(single) && !skip.count(single) && !has(single)) found.push_back(single); + } + return found; +} + +std::vector> Store::tag_neighbors(const std::string& record_id, int limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto rows = db_->execute("SELECT r.id, MIN(t.tag) FROM records r JOIN tags t ON t.item = r.id WHERE t.tag IN (SELECT tag FROM tags WHERE item = ? AND profile = ? AND tag NOT IN ('record', 'file')) AND r.id != ? AND r.profile = ? AND t.profile = ? GROUP BY r.id ORDER BY r.updated DESC LIMIT ?", + {record_id, prof, record_id, prof, prof, std::max(1, limit ? limit : TagRules::kLinkMax)}).rows; + std::vector> out; + for (auto& r : rows) out.emplace_back(r[0].text(), r[1].text()); + return out; +} + +void Store::tag_item(const std::string& item, const std::vector& tags, const std::string& prof_in) { + std::string prof = scope_(prof_in); + for (auto& tag : canonical_tags(tags, nullptr, prof)) db_->execute("INSERT OR IGNORE INTO tags (item, tag, profile) VALUES (?, ?, ?)", {item, tag, prof}); + db_->commit(); +} + +void Store::set_item_tags(const std::string& item, const std::vector& tags, const std::string& prof_in) { + std::string prof = scope_(prof_in); + db_->execute("DELETE FROM tags WHERE item = ? AND profile = ?", {item, prof}); + for (auto& tag : canonical_tags(tags, nullptr, prof)) db_->execute("INSERT OR IGNORE INTO tags (item, tag, profile) VALUES (?, ?, ?)", {item, tag, prof}); + db_->commit(); +} + +std::vector Store::item_tags(const std::string& item, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::vector out; + for (auto& r : db_->execute("SELECT tag FROM tags WHERE item = ? AND profile = ? ORDER BY tag", {item, prof}).rows) out.push_back(r[0].text()); + return out; +} + +std::optional> Store::tagged_items(const std::vector& tags, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto clean = TagRules::normalize_tags(tags); + if (clean.empty()) return std::nullopt; + Params params{prof}; + for (auto& t : clean) params.emplace_back(t); + params.emplace_back(static_cast(clean.size())); + std::vector out; + for (auto& r : db_->execute("SELECT item FROM tags WHERE profile = ? AND tag IN (" + marks_for(clean.size()) + ") GROUP BY item HAVING COUNT(DISTINCT tag) = ?", params).rows) out.push_back(r[0].text()); + return out; +} + +// ---------------------------------------------------------------- graph + +bool Store::has_node(const std::string& kind, const std::string& key, const std::string& prof_in) { + std::string prof = scope_(prof_in); + Cursor cur; + if (kind == "mem") cur = db_->execute("SELECT 1 FROM records WHERE profile = ? AND id = ?", {prof, "mem:" + key}); + else if (kind == "secret") cur = db_->execute("SELECT 1 FROM secrets WHERE profile = ? AND name = ?", {prof, key}); + else cur = db_->execute("SELECT 1 FROM schedules WHERE profile = ? AND id = ?", {prof, static_cast(std::stoll(key))}); + return !cur.rows.empty(); +} + +std::string Store::node_title(const std::string& node, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto parsed = TagRules::parse_node_id(node); + if (!parsed) return node; + const auto& [kind, key] = *parsed; + if (kind == "mem") { + auto cur = db_->execute("SELECT title FROM records WHERE profile = ? AND id = ?", {prof, node}); + if (cur.rows.empty()) return node; + const Cell& t = cur.rows[0][0]; + return (t.is_null() || t.text().empty()) ? node : t.text(); + } + if (kind == "secret") return "secret:" + key; + auto cur = db_->execute("SELECT name, prompt FROM schedules WHERE profile = ? AND id = ?", {prof, static_cast(std::stoll(key))}); + if (cur.rows.empty()) return node; + const Cell& n = cur.rows[0][0]; + return (n.is_null() || n.text().empty()) ? "schedule #" + key : n.text(); +} + +std::string Store::add_edge(const std::string& src, const std::string& dst, const std::string& relation, const std::string& prof_in) { + std::string prof = scope_(prof_in); + for (auto* node : {&src, &dst}) { + auto parsed = TagRules::parse_node_id(*node); + if (!parsed || !has_node(parsed->first, parsed->second, prof)) throw ValueError("unknown node " + *node); + } + std::string clean = TagRules::normalize_tag(relation); + if (clean.empty()) clean = "linked"; + db_->execute("INSERT OR IGNORE INTO edges(src, dst, relation, profile, created) VALUES(?, ?, ?, ?, ?)", {src, dst, clean, prof, Clock::now_iso_micro()}); + db_->commit(); + return clean; +} + +std::vector Store::edges_for(const std::string& node, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::vector out; + for (auto& r : db_->execute("SELECT src, dst, relation FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {prof, node, node}).rows) { + Json o = Json::object(); + o.set("relation", r[2].text()); + o.set("other", r[0].text() == node ? r[1].text() : r[0].text()); + o.set("direction", r[0].text() == node ? "out" : "in"); + out.push_back(o); + } + return out; +} + +int64_t Store::remove_edges(const std::string& node, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto done = db_->execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {prof, node, node}); + db_->commit(); + return done.rowcount; +} + +std::vector Store::traverse(const std::string& start, int depth, int limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::set seen{start}; + std::vector found; + { + Json o = Json::object(); + o.set("node", start); o.set("title", node_title(start, prof)); o.set("depth", 0); o.set("via", ""); + found.push_back(o); + } + struct Item { std::string node; int level; }; + std::vector frontier{{start, 0}}; + size_t head = 0; + int capped = std::max(1, std::min(depth, 4)); + size_t cap = static_cast(std::max(1, std::min(limit, 200))); + while (head < frontier.size() && found.size() < cap) { + Item it = frontier[head++]; + if (it.level >= capped) continue; + for (auto& edge : edges_for(it.node, prof)) { + std::string other = edge.get_string("other"); + if (seen.count(other)) continue; + seen.insert(other); + std::string rel = edge.get_string("relation"); + std::string hop = edge.get_string("direction") == "out" ? it.node + " -[" + rel + "]-> " + other : it.node + " <-[" + rel + "]- " + other; + Json o = Json::object(); + o.set("node", other); o.set("title", node_title(other, prof)); o.set("depth", it.level + 1); o.set("via", hop); + found.push_back(o); + frontier.push_back({other, it.level + 1}); + if (found.size() >= cap) break; + } + } + return found; +} + +// ---------------------------------------------------------------- records + +std::string Store::add_record(const std::string& kind, const std::string& title, const std::string& content, const std::vector& tags, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::string record_id = "mem:" + Clock::uuid_hex().substr(0, 16); + while (!db_->execute("SELECT 1 FROM records WHERE id = ?", {record_id}).rows.empty()) record_id = "mem:" + Clock::uuid_hex().substr(0, 16); + std::string stamp = Clock::now_iso_micro(); + long long size = static_cast(content.size()); + db_->execute("INSERT INTO records(id, kind, profile, title, content, size, reads, created, updated) VALUES(?, ?, ?, ?, ?, ?, 0, ?, ?)", {record_id, kind, prof, title, content, size, stamp, stamp}); + auto vocab = known_tags(prof); + auto explicit_tags = canonical_tags(tags, &vocab, prof); + size_t cut = content.find("\n...[truncated"); + std::string scan = title + "\n" + (cut == std::string::npos ? content : content.substr(0, cut)); + auto autos = auto_tags_for(scan, explicit_tags, &vocab); + std::vector all{"record"}; + all.insert(all.end(), explicit_tags.begin(), explicit_tags.end()); + all.insert(all.end(), autos.begin(), autos.end()); + set_item_tags(record_id, all, prof); + for (auto& [other, shared] : tag_neighbors(record_id, 0, prof)) { + db_->execute("INSERT OR IGNORE INTO edges(src, dst, relation, profile, created) VALUES(?, ?, ?, ?, ?)", {record_id, other, "shares-" + shared, prof, stamp}); + } + db_->commit(); + Json m = Json::object(); + m.set("id", record_id); m.set("kind", kind); m.set("size", size); + DebugLog::Global().event("DEBUG", "vault", "record_saved", "record " + record_id + " saved (" + kind + ", " + std::to_string(size) + " bytes)", prof, "", "", m); + return record_id; +} + +void Store::add_chunks(const std::string& parent_kind, const std::string& parent_id, const std::vector& texts, const std::string& prof_in) { + std::string prof = scope_(prof_in); + db_->execute("DELETE FROM chunks WHERE parent_kind = ? AND parent_id = ? AND profile = ?", {parent_kind, parent_id, prof}); + std::string stamp = Clock::now_iso_micro(); + for (size_t idx = 0; idx < texts.size(); idx++) { + db_->execute("INSERT INTO chunks(parent_kind, parent_id, idx, text, profile, created) VALUES (?, ?, ?, ?, ?, ?)", {parent_kind, parent_id, static_cast(idx), texts[idx], prof, stamp}); + } + db_->commit(); +} + +std::map Store::get_chunks_by_ids(const std::vector& ids, const std::string& prof_in) { + std::map out; + if (ids.empty()) return out; + std::string prof = scope_(prof_in); + Params params{prof}; + for (auto id : ids) params.emplace_back(static_cast(id)); + for (auto& r : db_->execute("SELECT id, parent_kind, parent_id, idx, text FROM chunks WHERE profile = ? AND id IN (" + marks_for(ids.size()) + ")", params).rows) { + Json o = Json::object(); + o.set("parent_kind", text_or_null(r[1])); o.set("parent_id", text_or_null(r[2])); + o.set("idx", int_or_null(r[3])); o.set("text", text_or_null(r[4])); + out[r[0].as_int()] = o; + } + return out; +} + +std::optional Store::get_record(const std::string& record_id, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT id, kind, title, content, size, reads, created, updated FROM records WHERE profile = ? AND id = ?", {prof, record_id}); + if (cur.rows.empty()) return std::nullopt; + const Row& r = cur.rows[0]; + db_->execute("UPDATE records SET reads = reads + 1 WHERE id = ?", {record_id}); + db_->commit(); + Json o = Json::object(); + o.set("id", r[0].text()); o.set("kind", text_or_null(r[1])); o.set("title", text_or_null(r[2])); o.set("content", text_or_null(r[3])); + o.set("size", static_cast(r[4].as_int())); o.set("reads", static_cast(r[5].as_int() + 1)); + o.set("created", text_or_null(r[6])); o.set("updated", text_or_null(r[7])); + o.set("tags", str_array(item_tags(r[0].text(), prof))); + return o; +} + +std::optional Store::read_record(const std::string& record_id, int64_t offset, int64_t limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT content FROM records WHERE profile = ? AND id = ?", {prof, record_id}); + if (cur.rows.empty()) return std::nullopt; + std::string content = cur.rows[0][0].text(); + int64_t total = static_cast(Utf8::length(content)); + int64_t start = std::max(0, std::min(offset, total)); + int64_t end = std::min(total, start + std::max(1, limit)); + db_->execute("UPDATE records SET reads = reads + 1 WHERE id = ?", {record_id}); + db_->commit(); + Json o = Json::object(); + o.set("id", record_id); + o.set("slice", Utf8::slice(content, static_cast(start), static_cast(end))); + o.set("start", static_cast(start)); o.set("end", static_cast(end)); o.set("total", static_cast(total)); + o.set("tags", str_array(item_tags(record_id, prof))); + return o; +} + +static Json record_summary(Store& st, const Row& r, const std::string& prof) { + Json o = Json::object(); + o.set("id", r[0].text()); o.set("kind", text_or_null(r[1])); o.set("title", text_or_null(r[2])); + o.set("size", int_or_null(r[3])); o.set("reads", int_or_null(r[4])); o.set("updated", text_or_null(r[5])); + o.set("tags", str_array(st.item_tags(r[0].text(), prof))); + return o; +} + +std::vector Store::search_records(const std::string& query, const std::string& kind, const std::vector& tags, int limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto clean = TagRules::normalize_tags(tags); + std::optional> terms; + if (!query.empty()) terms = Fts::escape(query); + if (terms && fts_ok) { + auto ranked = search_records_fts(*terms, kind, clean, limit, prof); + if (!ranked.empty()) return ranked; + } + std::string sql = "SELECT id, kind, title, size, reads, updated FROM records"; + std::vector clauses{"profile = ?"}; + Params params{prof}; + if (!kind.empty()) { clauses.push_back("kind = ?"); params.emplace_back(kind); } + if (!query.empty()) { + clauses.push_back("(title LIKE ? OR content LIKE ?)"); + params.emplace_back("%" + query + "%"); + params.emplace_back("%" + query + "%"); + } + for (auto& tag : clean) { + auto options = TagRules::tag_variants(tag); + clauses.push_back("id IN (SELECT item FROM tags WHERE profile = ? AND tag IN (" + marks_for(options.size()) + "))"); + params.emplace_back(prof); + for (auto& o : options) params.emplace_back(o); + } + sql += " WHERE "; + for (size_t k = 0; k < clauses.size(); k++) { if (k) sql += " AND "; sql += clauses[k]; } + sql += " ORDER BY updated DESC LIMIT ?"; + params.emplace_back(std::max(1, std::min(limit, 50))); + std::vector out; + for (auto& r : db_->execute(sql, params).rows) out.push_back(record_summary(*this, r, prof)); + return out; +} + +std::vector Store::search_records_fts(const std::vector& terms, const std::string& kind, const std::vector& clean, int limit, const std::string& prof) { + std::string rank = Fts::rank("f"); + std::string sql = "SELECT r.id, r.kind, r.title, r.size, r.reads, r.updated FROM records r JOIN fts_docs f ON f.item = r.id AND f.kind = 'record' WHERE r.profile = ? AND f MATCH ?"; + Params tail; + if (!kind.empty()) { sql += " AND r.kind = ?"; tail.emplace_back(kind); } + for (auto& tag : clean) { + auto options = TagRules::tag_variants(tag); + sql += " AND r.id IN (SELECT item FROM tags WHERE profile = ? AND tag IN (" + marks_for(options.size()) + "))"; + tail.emplace_back(prof); + for (auto& o : options) tail.emplace_back(o); + } + sql += " ORDER BY " + rank + " LIMIT ?"; + int capped = std::max(1, std::min(limit, 50)); + std::vector rows; + for (auto& match : Fts::match_variants(terms)) { + try { + Params p{prof, match}; + p.insert(p.end(), tail.begin(), tail.end()); + p.emplace_back(capped); + rows = db_->execute(sql, p).rows; + } catch (const SqlError&) { rows.clear(); } + if (!rows.empty()) break; + } + std::vector out; + for (auto& r : rows) out.push_back(record_summary(*this, r, prof)); + return out; +} + +bool Store::delete_record(const std::string& record_id, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto done = db_->execute("DELETE FROM records WHERE profile = ? AND id = ?", {prof, record_id}); + if (done.rowcount > 0) { + db_->execute("DELETE FROM tags WHERE item = ? AND profile = ?", {record_id, prof}); + db_->execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {prof, record_id, record_id}); + db_->execute("DELETE FROM chunks WHERE parent_kind = 'record' AND parent_id = ? AND profile = ?", {record_id, prof}); + db_->commit(); + Json m = Json::object(); + m.set("id", record_id); + DebugLog::Global().event("DEBUG", "vault", "record_deleted", "record " + record_id + " deleted", prof, "", "", m); + return true; + } + db_->commit(); + return false; +} + +std::optional Store::file_record_id(const std::string& path, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT id FROM records WHERE profile = ? AND kind = 'file' AND title = ?", {prof, path}); + if (cur.rows.empty()) return std::nullopt; + return cur.rows[0][0].text(); +} + +std::string Store::upsert_file_record(const std::string& path, const std::string& content, int64_t true_size, const std::string& prof_in) { + std::string prof = scope_(prof_in); + int64_t full_bytes = static_cast(content.size()); + std::string text; + if (true_size >= 0 && true_size > full_bytes) text = content + marker(true_size); + else if (Utf8::length(content) <= static_cast(kFileRecordMax)) text = content; + else text = Utf8::prefix(content, kFileRecordMax) + marker(full_bytes); + // os.path.splitext(path)[1].lstrip(".").lower() + size_t slash = path.rfind('/'); + std::string base = slash == std::string::npos ? path : path.substr(slash + 1); + std::string ext; + size_t lead = 0; + while (lead < base.size() && base[lead] == '.') lead++; + size_t dot = base.rfind('.'); + if (dot != std::string::npos && dot >= lead) ext = base.substr(dot); + size_t nz = 0; + while (nz < ext.size() && ext[nz] == '.') nz++; + ext = ext.substr(nz); + for (auto& c : ext) if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + bool alnum = !ext.empty(); + for (unsigned char c : ext) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || c >= 0x80)) alnum = false; + std::vector explicit_tags{"file"}; + if (alnum && Utf8::length(ext) <= 5) explicit_tags.push_back(ext); + auto vocab = known_tags(prof); + auto tags = explicit_tags; + for (auto& t : auto_tags_for(path + "\n" + content, explicit_tags, &vocab)) tags.push_back(t); + auto record_id = file_record_id(path, prof); + std::string stamp = Clock::now_iso_micro(); + if (!record_id) return add_record("file", path, text, tags, prof); + db_->execute("UPDATE records SET content = ?, size = ?, updated = ? WHERE id = ?", {text, static_cast(text.size()), stamp, *record_id}); + set_item_tags(*record_id, tags, prof); + db_->commit(); + return *record_id; +} + +bool Store::delete_file_record(const std::string& path, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto id = file_record_id(path, prof); + if (!id) return false; + return delete_record(*id, prof); +} + +// ---------------------------------------------------------------- audit + +int64_t Store::audit_event(const std::string& actor, const std::string& action, const std::string& path, const std::string& message, const std::optional& old_in, const std::optional& new_in, const std::vector& tags, int64_t old_size, int64_t new_size, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cap = [](const std::optional& t, std::optional& out, std::optional& sz) { + if (!t) { out.reset(); sz.reset(); return; } + int64_t bytes = static_cast(t->size()); + if (Utf8::length(*t) <= static_cast(kAuditMaxChars)) out = *t; + else out = Utf8::prefix(*t, kAuditMaxChars) + marker(bytes); + sz = bytes; + }; + std::optional old_text, new_text; + std::optional computed_old, computed_new; + cap(old_in, old_text, computed_old); + cap(new_in, new_text, computed_new); + if (old_in && old_size >= 0 && old_size > static_cast(old_in->size())) { old_text = Utf8::prefix(*old_in, kAuditMaxChars) + marker(old_size); computed_old = old_size; } + if (new_in && new_size >= 0 && new_size > static_cast(new_in->size())) { new_text = Utf8::prefix(*new_in, kAuditMaxChars) + marker(new_size); computed_new = new_size; } + std::string joined; + for (auto& t : TagRules::normalize_tags(tags)) { if (!joined.empty()) joined += " "; joined += t; } + auto cell_s = [](const std::optional& v) { return v ? Cell(*v) : Cell(); }; + auto cell_i = [](const std::optional& v) { return v ? Cell(static_cast(*v)) : Cell(); }; + auto cur = db_->execute("INSERT INTO audit(profile, ts, actor, action, path, message, old_size, new_size, old, new, tags) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + {prof, Clock::now_iso_micro(), actor, action, path, Utf8::prefix(message, 2000), cell_i(computed_old), cell_i(computed_new), cell_s(old_text), cell_s(new_text), joined}); + db_->commit(); + return cur.lastrowid; +} + +std::vector Store::audit_history(const std::string& path, const std::string& tag, int limit, const std::string& prof_in) { + std::string prof = scope_(prof_in); + std::string sql = "SELECT id, ts, actor, action, path, message, old_size, new_size, tags FROM audit"; + std::vector clauses{"profile = ?"}; + Params params{prof}; + if (!path.empty()) { clauses.push_back("path = ?"); params.emplace_back(path); } + if (!tag.empty()) { clauses.push_back("tags LIKE ?"); params.emplace_back("%" + tag + "%"); } + sql += " WHERE "; + for (size_t k = 0; k < clauses.size(); k++) { if (k) sql += " AND "; sql += clauses[k]; } + sql += " ORDER BY id DESC LIMIT ?"; + params.emplace_back(std::max(1, std::min(limit, 100))); + std::vector out; + for (auto& r : db_->execute(sql, params).rows) { + Json o = Json::object(); + o.set("id", static_cast(r[0].as_int())); o.set("ts", text_or_null(r[1])); o.set("actor", text_or_null(r[2])); + o.set("action", text_or_null(r[3])); o.set("path", text_or_null(r[4])); o.set("message", text_or_null(r[5])); + o.set("old_size", int_or_null(r[6])); o.set("new_size", int_or_null(r[7])); o.set("tags", text_or_null(r[8])); + out.push_back(o); + } + return out; +} + +std::optional Store::audit_get(int64_t row_id, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto cur = db_->execute("SELECT id, ts, actor, action, path, message, old_size, new_size, old, new, tags FROM audit WHERE profile = ? AND id = ?", {prof, static_cast(row_id)}); + if (cur.rows.empty()) return std::nullopt; + const Row& r = cur.rows[0]; + Json o = Json::object(); + o.set("id", static_cast(r[0].as_int())); o.set("ts", text_or_null(r[1])); o.set("actor", text_or_null(r[2])); + o.set("action", text_or_null(r[3])); o.set("path", text_or_null(r[4])); o.set("message", text_or_null(r[5])); + o.set("old_size", int_or_null(r[6])); o.set("new_size", int_or_null(r[7])); + o.set("old", text_or_null(r[8])); o.set("new", text_or_null(r[9])); o.set("tags", text_or_null(r[10])); + return o; +} + +// ---------------------------------------------------------------- schedules + +int64_t Store::add_schedule(const std::string& name, const std::string& prompt, const std::string& profile_in, int64_t every_sec, const std::string& next_run, int64_t timeout, const std::optional>& tags) { + std::string stored = seal_.enabled() ? seal_.lock(prompt) : prompt; + std::string now = Clock::now_iso(); + auto done = db_->execute("INSERT INTO schedules (name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created, updated) VALUES (?, ?, ?, ?, ?, ?, 'pending', '', '', ?, ?)", + {name, stored, profile_in, static_cast(every_sec), next_run, static_cast(timeout), now, Clock::now_iso()}); + std::string item = "sched:" + std::to_string(done.lastrowid); + if (!tags) tag_item(item, {"schedule"}, profile_in); + else { + std::vector all{"schedule"}; + all.insert(all.end(), tags->begin(), tags->end()); + set_item_tags(item, all, profile_in); + } + db_->commit(); + Json m = Json::object(); + m.set("id", static_cast(done.lastrowid)); m.set("name", name); + DebugLog::Global().event("DEBUG", "vault", "schedule_saved", "schedule #" + std::to_string(done.lastrowid) + " saved (" + name + ")", profile_in, "", "", m); + return done.lastrowid; +} + +bool Store::remove_schedule(int64_t row_id, const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto done = db_->execute("DELETE FROM schedules WHERE id = ? AND profile = ?", {static_cast(row_id), prof}); + if (done.rowcount > 0) { + std::string item = "sched:" + std::to_string(row_id); + db_->execute("DELETE FROM tags WHERE item = ? AND profile = ?", {item, prof}); + db_->execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", {prof, item, item}); + db_->commit(); + Json m = Json::object(); + m.set("id", static_cast(row_id)); + DebugLog::Global().event("DEBUG", "vault", "schedule_removed", "schedule #" + std::to_string(row_id) + " removed", prof, "", "", m); + return true; + } + db_->commit(); + return false; +} + +std::vector Store::list_schedules(const std::string& prof_in) { + std::string prof = scope_(prof_in); + auto rows = db_->execute("SELECT id, name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created FROM schedules WHERE profile = ? ORDER BY next_run", {prof}).rows; + std::vector items; + for (auto& r : rows) { + std::string prompt = r[2].is_null() ? "" : r[2].text(); + if (SealPort::is_sealed(prompt)) prompt = seal_.safe_unlock(prompt); + Json o = Json::object(); + o.set("id", static_cast(r[0].as_int())); o.set("name", text_or_null(r[1])); o.set("prompt", prompt); + o.set("profile", text_or_null(r[3])); o.set("every", int_or_null(r[4])); o.set("next_run", text_or_null(r[5])); + o.set("timeout", int_or_null(r[6])); o.set("status", text_or_null(r[7])); o.set("last", text_or_null(r[8])); + o.set("result", text_or_null(r[9])); o.set("created", text_or_null(r[10])); + o.set("tags", str_array(item_tags("sched:" + std::to_string(r[0].as_int()), r[3].text()))); + items.push_back(o); + } + return items; +} + +} // namespace detz::store diff --git a/taicpp/src/store/tag_rules.cpp b/taicpp/src/store/tag_rules.cpp new file mode 100644 index 0000000..820cb74 --- /dev/null +++ b/taicpp/src/store/tag_rules.cpp @@ -0,0 +1,108 @@ +#include "tag_rules.h" +#include +#include +#include "patterns.h" +#include "utf8.h" +namespace detz::store { + + +std::vector TagRules::tokens(const std::string& raw) { + // lower() then the runs of [a-z0-9]; every other code point separates. + std::vector out; + std::string cur; + auto flush = [&]() { if (!cur.empty()) { out.push_back(cur); cur.clear(); } }; + size_t i = 0; + while (i < raw.size()) { + unsigned char c = static_cast(raw[i]); + if (c < 0x80) { + if (c >= 'A' && c <= 'Z') c = static_cast(c + 32); + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) cur += static_cast(c); + else flush(); + i++; + continue; + } + // the only non-ascii code points whose lower() carries ascii letters + if (c == 0xE2 && i + 2 < raw.size() && static_cast(raw[i + 1]) == 0x84 && static_cast(raw[i + 2]) == 0xAA) { + cur += 'k'; // KELVIN SIGN + i += 3; + continue; + } + if (c == 0xC4 && i + 1 < raw.size() && static_cast(raw[i + 1]) == 0xB0) { + cur += 'i'; // LATIN CAPITAL I WITH DOT ABOVE lowers to i + U+0307 + flush(); + i += 2; + continue; + } + flush(); + i++; + while (i < raw.size() && (static_cast(raw[i]) & 0xC0) == 0x80) i++; + } + flush(); + return out; +} + +std::string TagRules::normalize_tag(const std::string& raw) { + std::string out; + for (auto& t : tokens(raw)) { if (!out.empty()) out += '-'; out += t; } + if (out.size() > 32) out.resize(32); // strip happens before the cut, so a trailing "-" may remain + return out; +} + +std::vector TagRules::normalize_tags(const std::vector& items) { + std::vector seen; + for (auto& item : items) { + std::string c = normalize_tag(item); + if (!c.empty() && std::find(seen.begin(), seen.end(), c) == seen.end()) seen.push_back(c); + } + if (seen.size() > 20) seen.resize(20); + return seen; +} + +static bool ends_with(const std::string& s, const char* suf) { + size_t n = std::char_traits::length(suf); + return s.size() >= n && s.compare(s.size() - n, n, suf) == 0; +} + +std::string TagRules::singular_noun(const std::string& word) { + static const char* keep[] = {"news", "means", "series", "species", "physics"}; + for (auto* k : keep) if (word == k) return word; + if (word.size() <= 3) return word; + if (ends_with(word, "ies")) return word.substr(0, word.size() - 3) + "y"; + if (ends_with(word, "ses") || ends_with(word, "xes") || ends_with(word, "zes") || ends_with(word, "ches") || ends_with(word, "shes")) return word.substr(0, word.size() - 2); + if (ends_with(word, "s") && !(ends_with(word, "ss") || ends_with(word, "us") || ends_with(word, "is") || ends_with(word, "os"))) return word.substr(0, word.size() - 1); + return word; +} + +std::vector TagRules::tag_variants(const std::string& tag) { + std::vector found{tag, singular_noun(tag)}; + if (!ends_with(tag, "s")) found.push_back(tag + "s"); + std::sort(found.begin(), found.end()); + found.erase(std::unique(found.begin(), found.end()), found.end()); + return found; +} + +std::optional> TagRules::parse_node_id(const std::string& raw) { + // str(raw).strip() then fullmatch (mem|secret|sched):(.+) + std::string s = Utf8::strip(raw); + std::string kind; + for (const char* k : {"mem", "secret", "sched"}) { + std::string pre = std::string(k) + ":"; + if (s.rfind(pre, 0) == 0) { kind = k; break; } + } + if (kind.empty()) return std::nullopt; + std::string key = s.substr(kind.size() + 1); + if (key.empty() || key.find('\n') != std::string::npos) return std::nullopt; + if (kind == "mem") { + if (key.size() != 16) return std::nullopt; + for (char c : key) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return std::nullopt; + return std::make_pair(kind, key); + } + if (kind == "secret") { + if (!Patterns::secret(key)) return std::nullopt; + return std::make_pair(kind, key); + } + for (char c : key) if (c < '0' || c > '9') return std::nullopt; + return std::make_pair(kind, key); +} + +} // namespace detz::store diff --git a/taicpp/src/store/tag_rules.h b/taicpp/src/store/tag_rules.h new file mode 100644 index 0000000..ef329bc --- /dev/null +++ b/taicpp/src/store/tag_rules.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include +#include +namespace detz::store { + +// Tag vocabulary rules (tai.py normalize_tag, normalize_tags, singular_noun, +// tag_variants, parse_node_id and the TAG_* constants). +class TagRules { + public: + static constexpr int kAutoMax = 5; + static constexpr int kLinkMax = 3; + // lowercased [a-z0-9]+ runs (python re.findall(r"[a-z0-9]+", raw.lower())) + static std::vector tokens(const std::string& raw); + static std::string normalize_tag(const std::string& raw); + static std::vector normalize_tags(const std::vector& items); + static std::string singular_noun(const std::string& word); + static std::vector tag_variants(const std::string& tag); // sorted, unique + // (kind, key) for mem:<16 hex>, secret:, sched:; nullopt otherwise + static std::optional> parse_node_id(const std::string& raw); + static bool auto_skip(const std::string& tag) { return tag == "record" || tag == "file"; } +}; + +} // namespace detz::store diff --git a/taicpp/src/store/utf8.cpp b/taicpp/src/store/utf8.cpp new file mode 100644 index 0000000..21a0473 --- /dev/null +++ b/taicpp/src/store/utf8.cpp @@ -0,0 +1,138 @@ +#include "utf8.h" +#include +namespace detz::store { + +size_t Utf8::length(const std::string& s) { + size_t n = 0; + for (unsigned char c : s) if ((c & 0xC0) != 0x80) n++; + return n; +} + +size_t Utf8::offset(const std::string& s, size_t n) { + size_t seen = 0, i = 0; + while (i < s.size()) { + if ((static_cast(s[i]) & 0xC0) != 0x80) { + if (seen == n) return i; + seen++; + } + i++; + } + return s.size(); +} + +std::string Utf8::slice(const std::string& s, size_t start, size_t end) { + if (end <= start) return ""; + size_t a = offset(s, start); + size_t b = offset(s, end); + return s.substr(a, b - a); +} + +std::string Utf8::suffix(const std::string& s, size_t n) { + size_t len = length(s); + if (n >= len) return s; + return s.substr(offset(s, len - n)); +} + +// Decodes the code point at s[i]; returns its byte length (0 when invalid) and the value. +static int decode(const std::string& s, size_t i, uint32_t& cp) { + unsigned char c = static_cast(s[i]); + int len; + if (c < 0x80) { cp = c; return 1; } + if (c >= 0xC2 && c <= 0xDF) { len = 2; cp = c & 0x1Fu; } + else if (c >= 0xE0 && c <= 0xEF) { len = 3; cp = c & 0x0Fu; } + else if (c >= 0xF0 && c <= 0xF4) { len = 4; cp = c & 0x07u; } + else return 0; + if (i + static_cast(len) > s.size()) return 0; + for (int k = 1; k < len; k++) { + unsigned char d = static_cast(s[i + static_cast(k)]); + if ((d & 0xC0) != 0x80) return 0; + cp = (cp << 6) | (d & 0x3Fu); + } + if (len == 3 && (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF))) return 0; + if (len == 4 && (cp < 0x10000 || cp > 0x10FFFF)) return 0; + return len; +} + +static bool is_space(uint32_t cp) { + return (cp >= 9 && cp <= 13) || (cp >= 0x1c && cp <= 0x20) || cp == 0x85 || cp == 0xA0 || cp == 0x1680 || + (cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; +} + +std::string Utf8::strip(const std::string& s) { + size_t a = 0; + while (a < s.size()) { + uint32_t cp; + int n = decode(s, a, cp); + if (n == 0 || !is_space(cp)) break; + a += static_cast(n); + } + size_t b = s.size(); + while (b > a) { + size_t k = b - 1; + while (k > a && (static_cast(s[k]) & 0xC0) == 0x80) k--; + uint32_t cp; + int n = decode(s, k, cp); + if (n == 0 || k + static_cast(n) != b || !is_space(cp)) break; + b = k; + } + return s.substr(a, b - a); +} + +bool Utf8::has_space(const std::string& s) { + size_t i = 0; + while (i < s.size()) { + uint32_t cp; + int n = decode(s, i, cp); + if (n == 0) { i++; continue; } + if (is_space(cp)) return true; + i += static_cast(n); + } + return false; +} + +std::vector Utf8::split_ws(const std::string& s) { + std::vector out; + std::string cur; + size_t i = 0; + while (i < s.size()) { + uint32_t cp; + int n = decode(s, i, cp); + if (n == 0) { cur += s[i]; i++; continue; } + if (is_space(cp)) { if (!cur.empty()) { out.push_back(cur); cur.clear(); } } + else cur.append(s, i, static_cast(n)); + i += static_cast(n); + } + if (!cur.empty()) out.push_back(cur); + return out; +} + +std::pair Utf8::split_first(const std::string& s) { + size_t i = 0; + uint32_t cp; + int n; + while (i < s.size() && (n = decode(s, i, cp)) != 0 && is_space(cp)) i += static_cast(n); + size_t start = i; + while (i < s.size()) { + n = decode(s, i, cp); + if (n == 0) { i++; continue; } + if (is_space(cp)) break; + i += static_cast(n); + } + std::string word = s.substr(start, i - start); + while (i < s.size() && (n = decode(s, i, cp)) != 0 && is_space(cp)) i += static_cast(n); + return {word, s.substr(i)}; +} + +std::string Utf8::sanitize(const std::string& s) { + std::string out; + size_t i = 0; + while (i < s.size()) { + uint32_t cp; + int n = decode(s, i, cp); + if (n == 0) { out += "\xef\xbf\xbd"; i++; } + else { out.append(s, i, static_cast(n)); i += static_cast(n); } + } + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/utf8.h b/taicpp/src/store/utf8.h new file mode 100644 index 0000000..08b4944 --- /dev/null +++ b/taicpp/src/store/utf8.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +#include +#include +namespace detz::store { + +// python str indexing on utf-8 data: lengths and slices count code points, +// not bytes (len(text), text[a:b], text[:n], text[-n:]). +class Utf8 { + public: + static size_t length(const std::string& s); + // text[start:end] with python clamping (start/end >= 0) + static std::string slice(const std::string& s, size_t start, size_t end); + static std::string prefix(const std::string& s, size_t n) { return slice(s, 0, n); } + // text[-n:] + static std::string suffix(const std::string& s, size_t n); + // python str.strip() (unicode whitespace) + static std::string strip(const std::string& s); + // true when the (non-empty) string holds any unicode whitespace + static bool has_space(const std::string& s); + // python str.split(): runs of unicode whitespace separate the tokens + static std::vector split_ws(const std::string& s); + // s.split(None, 1): first whitespace-delimited token and the rest with leading whitespace removed ("" when none) + static std::pair split_first(const std::string& s); + // replace invalid utf-8 bytes with U+FFFD (python errors="replace") + static std::string sanitize(const std::string& s); + // byte offset of the n-th code point (clamped to size) + static size_t offset(const std::string& s, size_t n); +}; + +} // namespace detz::store diff --git a/taicpp/src/store/vault_files.cpp b/taicpp/src/store/vault_files.cpp new file mode 100644 index 0000000..630f077 --- /dev/null +++ b/taicpp/src/store/vault_files.cpp @@ -0,0 +1,91 @@ +#include "vault_files.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "errors.h" +namespace detz::store { + +void VaultFiles::atomic_write(const std::string& path, const std::string& content) { + std::string tmp = path + ".tmp-" + std::to_string(static_cast(::getpid())); + int fd = ::open(tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) throw std::runtime_error(std::string("open ") + tmp + ": " + std::strerror(errno)); + size_t done = 0; + while (done < content.size()) { + ssize_t n = ::write(fd, content.data() + done, content.size() - done); + if (n < 0) { if (errno == EINTR) continue; int e = errno; ::close(fd); throw std::runtime_error(std::string("write: ") + std::strerror(e)); } + done += static_cast(n); + } + ::fsync(fd); + ::close(fd); + if (::rename(tmp.c_str(), path.c_str()) != 0) { + int e = errno; + ::unlink(tmp.c_str()); + throw std::runtime_error(std::string("rename: ") + std::strerror(e)); + } +} + +void VaultFiles::sealed_write(const std::string& path, const std::string& content) { + atomic_write(path, content); + ::chmod(path.c_str(), 0600); +} + +void VaultFiles::quarantine(const std::string& path) { + if (!exists(path)) return; + std::string dest = path + ".corrupt-" + std::to_string(static_cast(std::time(nullptr))); + ::rename(path.c_str(), dest.c_str()); +} + +bool VaultFiles::exists(const std::string& path) { + struct stat st; + return ::stat(path.c_str(), &st) == 0; +} + +bool VaultFiles::valid_utf8(const std::string& s) { + size_t i = 0, n = s.size(); + while (i < n) { + unsigned char c = static_cast(s[i]); + if (c < 0x80) { i++; continue; } + int len; + unsigned cp; + if (c >= 0xC2 && c <= 0xDF) { len = 2; cp = c & 0x1Fu; } + else if (c >= 0xE0 && c <= 0xEF) { len = 3; cp = c & 0x0Fu; } + else if (c >= 0xF0 && c <= 0xF4) { len = 4; cp = c & 0x07u; } + else return false; + if (i + static_cast(len) > n) return false; + for (int k = 1; k < len; k++) { + unsigned char d = static_cast(s[i + static_cast(k)]); + if ((d & 0xC0) != 0x80) return false; + cp = (cp << 6) | (d & 0x3Fu); + } + if (len == 3 && (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF))) return false; + if (len == 4 && (cp < 0x10000 || cp > 0x10FFFF)) return false; + i += static_cast(len); + } + return true; +} + +std::string VaultFiles::read_text(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("cannot read " + path); + std::stringstream ss; + ss << in.rdbuf(); + if (in.bad()) throw std::runtime_error("read failed " + path); + std::string raw = ss.str(); + if (!valid_utf8(raw)) throw ValueError("'utf-8' codec can't decode bytes"); + std::string out; + out.reserve(raw.size()); + for (size_t i = 0; i < raw.size(); i++) { + if (raw[i] == '\r') { out += '\n'; if (i + 1 < raw.size() && raw[i + 1] == '\n') i++; } + else out += raw[i]; + } + return out; +} + +} // namespace detz::store diff --git a/taicpp/src/store/vault_files.h b/taicpp/src/store/vault_files.h new file mode 100644 index 0000000..1a0cb0c --- /dev/null +++ b/taicpp/src/store/vault_files.h @@ -0,0 +1,22 @@ +#pragma once +#include +namespace detz::store { + +// File helpers behind the profile / session / turn / bots files (tai.py +// atomic_write, sealed_write, quarantine and python text-mode reads). +class VaultFiles { + public: + // tmp file ".tmp-", fsync, rename over path. Throws std::runtime_error (python OSError). + static void atomic_write(const std::string& path, const std::string& content); + // atomic_write then chmod 0600 (chmod failure ignored) + static void sealed_write(const std::string& path, const std::string& content); + // rename path to ".corrupt-" when it exists; never throws + static void quarantine(const std::string& path); + // open(path, "r", encoding="utf-8").read(): strict utf-8 (throws ValueError) and universal newlines; + // throws std::runtime_error when the file cannot be read (python OSError) + static std::string read_text(const std::string& path); + static bool exists(const std::string& path); + static bool valid_utf8(const std::string& s); +}; + +} // namespace detz::store diff --git a/taicpp/tests/foundation.mk b/taicpp/tests/foundation.mk new file mode 100644 index 0000000..3010f1c --- /dev/null +++ b/taicpp/tests/foundation.mk @@ -0,0 +1,35 @@ +# Foundation tests (store, config, session, plexer, repl). +# Run from taicpp/: make -f tests/foundation.mk test +# Outputs stay under build/. Independent of the main Makefile on purpose: the +# foundation area builds and tests itself with only sqlite3 as a link dependency. +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1 +INC = -Isrc -Isrc/store -Isrc/session -Isrc/config -Isrc/plexer -Isrc/repl +FB = build/foundation +FSRCS = $(wildcard src/store/*.cpp) $(wildcard src/session/*.cpp) $(wildcard src/config/*.cpp) $(wildcard src/plexer/*.cpp) $(wildcard src/repl/*.cpp) +FOBJS = $(patsubst src/%.cpp,$(FB)/%.o,$(FSRCS)) +LIBS = -lsqlite3 -pthread + +.PHONY: all test clean-foundation +all: build/foundation_probe build/repl_probe build/foundation_test + +$(FB)/%.o: src/%.cpp + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(INC) -c $< -o $@ + +build/foundation_probe: tests/foundation_probe.cpp $(FOBJS) + $(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@ + +build/repl_probe: tests/repl_probe.cpp $(FOBJS) + $(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@ + +build/foundation_test: tests/foundation_test.cpp $(FOBJS) + $(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@ + +test: all + build/foundation_test + python3 tests/foundation_parity.py + python3 tests/repl_persist.py + +clean-foundation: + rm -rf $(FB) build/foundation_probe build/repl_probe build/foundation_test diff --git a/taicpp/tests/foundation_cases.py b/taicpp/tests/foundation_cases.py new file mode 100644 index 0000000..c2eac4e --- /dev/null +++ b/taicpp/tests/foundation_cases.py @@ -0,0 +1,274 @@ +"""Input tables for tests/foundation_parity.py: pure-function cases, flow specs +and a fake /proc tree. Each case is [op, positional args]; both +implementations run the same list and the driver diffs the outputs.""" +import os + + +def pure_cases(): + cases = [] + add = cases.append + for name in ("DEFAULT_SYSTEM", "MERGE_SYSTEM", "COMPACT_SYSTEM", "AUTONOMOUS_NOTE", "HELP_TEXT", "LEGACY_PASSWORD_NOTE", "VAULT_MEMORY_NOTE"): + add(["prompt", [name]]) + add(["capability_block", []]) + add(["capability_block", [True, False]]) + add(["capability_block", [False, True]]) + add(["orientation_block", ["/tmp"]]) + add(["orientation_block", ["/tmp", True]]) + add(["orientation_block", [""]]) + add(["settings_catalog", []]) + nested = {"b": 1, "a": [1, 2.5, "ü😀", None, True, {"x": "\n\t\"\\"}], "z": {"y": {"q": []}}} + for value in (nested, 1e16, 1e-5, 0.1, 1.0, 123456789012345678, -0.0, "\x7f\x1f\x00", [], {}, 1e22, 12345678.9, [1e15, 1e-4]): + add(["json_dumps", [value]]) + for value in (0.1, 1e16, 1e-5, 123456.789, 1.5e300, 5e-324, 2.5, 100.0, 1e15, 1.2345678901234567e16, 0.0001, 0.00001234, 3.0, 1e100, 123456789.123456789): + add(["float_repr", [value]]) + for text in ('{"a":1}', '[1,2,3]', ' {"k" : "\\u00e9\\ud83d\\ude00"} ', '{"a":', '[1,]', 'NaN', '{"a":1} trailing', '"\\x"', '01', '1.', '-', + '{"a":1,"a":2}', '[1e2, -0, 2E-2, 1.5e+3]', '"\\ud83d"', 'tru', '{"a" 1}', '[[[[1]]]]'): + add(["json_roundtrip", [text]]) + for raw in ("Hello World!", " --x-- ", "ÅÄÖ test", "a" * 40, "İstanbul", "Kelvin K", "", "x" * 31 + "-yz", "a--b__c", "Ünï-cödé", "123 456", "---"): + add(["normalize_tag", [raw]]) + add(["normalize_tags", [["A b", "a-b", "", "C", "c", "d"] + ["t%d" % n for n in range(30)]]]) + for word in ("news", "cats", "boxes", "ladies", "class", "bus", "analysis", "photos", "is", "ab", "classes", "churches", "dishes", "buzzes", "series", "dogs", "ss", "gas", "physics", "mice"): + add(["singular_noun", [word]]) + add(["tag_variants", [word]]) + for raw in ("mem:0123456789abcdef", "mem:0123456789abcdeg", "mem:0123456789ABCDEF", "secret:api_key", "secret:bad key", "secret:", "sched:12", "sched:1a", + " mem:0123456789abcdef \n", "other:1", "secret:a\n", "secret:a.b-c_d", "sched:", "mem:abc", ""): + add(["parse_node_id", [raw]]) + for q in ("Hello World", "", "!!!", "x", "ÄÖ ünïcode 123", "a-b c_d", "KELVIN K", "İ"): + add(["fts_match_variants", [q]]) + add(["fts_rank", ["fts_docs"]]) + for ident in ("a", "work-1", "-bad", "a" * 32, "a" * 33, "ok\n", "", "ü", "A_b-c", "_a", "ok\n\n"): + add(["profile_re", [ident]]) + add(["bot_re", [ident]]) + add(["secret_re", [ident]]) + add(["mcp_server_re", [ident]]) + add(["split_models", ["a, b\nc,,d "]]) + add(["split_models", [""]]) + add(["split_models", [" , "]]) + add(["split_models", ["x;y, z"]]) + add(["parse_backends", ["lab|http://x/|key|m|opencode;l2|b|$HOME|;bad|x;\nl3|b|k|m|zzz;|b|k;l4||k; l5 | http://y// | k | m | OpenCode "]]) + add(["parse_backends", [""]]) + nsv = [("model", "x"), ("model", ""), ("model", None), ("model", 5), ("backends", None), ("backends", "a|b|c"), ("retry_seconds", 2.5), ("retry_seconds", "3"), + ("retry_seconds", "1_0"), ("retry_seconds", " 4.5 "), ("retry_seconds", "inf"), ("retry_seconds", "nan"), ("retry_seconds", "x"), ("retry_seconds", True), + ("retry_seconds", -0.5), ("retry_seconds", 0), ("retry_seconds", [1]), ("retry_seconds", 1e20), ("retry_seconds", ".5"), ("retry_seconds", "5."), + ("zen_enabled", True), ("zen_enabled", False), ("zen_enabled", 1), ("zen_enabled", 0), ("zen_enabled", 1.0), ("zen_enabled", 2), ("zen_enabled", "ON"), + ("zen_enabled", " no "), ("zen_enabled", "maybe"), ("zen_enabled", None), ("zen_enabled", []), ("theme", "dark"), ("theme", "neon"), ("theme", 3), + ("gemini_plan", "plan"), ("muse_bin", None), ("muse_bin", "/x"), ("muse_bin", ""), ("voice", ""), ("devplace_base", "http://x/")] + for key, value in nsv: + add(["normalize_setting_value", [key, value]]) + pse = [("model", "x"), ("theme", "nord"), ("theme", "bad"), ("zen_enabled", "On"), ("zen_enabled", "junk"), ("zen_enabled", " 1 "), ("gemini_plan", "plan"), + ("gemini_plan", "Plan"), ("gemini_plan", "1"), ("codex_sandbox", "read-only"), ("codex_sandbox", "no"), ("box_caps", "0"), ("box_caps", "false"), + ("box_caps", "1"), ("box_caps", "junk"), ("retry_seconds", "2.5"), ("retry_seconds", "abc"), ("retry_seconds", "12"), ("retry_seconds", "1e3"), + ("retry_seconds", "5."), ("retry_seconds", "007"), ("openrouter_enabled", "off"), ("opencode_enabled", "")] + for key, raw in pse: + add(["parse_setting_env", [key, raw]]) + for key, raw in (("retry_seconds", "2.5"), ("zen_enabled", "1"), ("zen_enabled", "0"), ("model", "m"), ("retry_seconds", "0.0")): + add(["parse_setting_stored", [key, raw]]) + lines = "".join("line %d é\n" % n for n in range(130)) + for text, extra in ((lines, []), (lines, [10, 2, 100]), ("", []), ("one", []), ("a\r\nb\rc\x0cd\x0be\x1cf
g\u0085h\n\n\ni", []), + ("x" * 5000 + "\n" + "y" * 10, [60, 3, 4000]), (" \n lead\n\n", [2, 1, 50]), ("a\nb\nc\nd\ne\nf\n", [2, 1, 4000])): + add(["chunk_lines", [text] + extra]) + for text, extra in (("short", []), ("x" * 2100, []), ("é" * 2100, []), ("y" * 2051, []), ("z" * 2050, []), ("abcdefghij" * 10, [10, 10])): + add(["truncate", [text] + extra]) + add(["sanitize_sid", ["a b/c!d.e-f_g" + "x" * 70]]) + add(["sanitize_sid", [""]]) + return cases + + +def flow_cases(): + cases = [] + add = cases.append + base = {"title": "T", "screens": [ + {"id": "s1", "title": "S1", "body": "hello", "fields": [ + {"id": "c", "type": "choice", "label": "Pick", "options": ["a", {"label": "B", "value": 2}], "default": 1}, + {"id": "t", "type": "text", "label": "Name", "initial": "x", "show_if": {"c": 2}}, + {"id": "i", "type": "info", "text": "note"}], + "actions": [{"id": "ok", "label": "OK"}, {"id": "no", "label": "No"}], "next": {"ok": "s2", "no": "$cancel"}}, + {"id": "s2", "body": "bye"}]} + bad = [ + None, [], {}, {"v": 2, "screens": []}, {"screens": []}, {"screens": [{}]}, {"screens": [{"id": "a b", "body": "x"}]}, {"screens": [{"id": "a"}]}, + {"screens": [{"id": "a", "body": "x"}, {"id": "a", "body": "y"}]}, {"timeout": 1, "screens": [{"id": "a", "body": "x"}]}, + {"timeout": "9", "screens": [{"id": "a", "body": "x"}]}, {"timeout": "z", "screens": [{"id": "a", "body": "x"}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "bogus"}]}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "choice", "options": []}]}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "choice", "options": ["a"], "default": 3}]}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "choice", "options": [{"label": " "}]}]}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "choice", "options": [{"label": "l", "value": [1]}]}]}]}, + {"screens": [{"id": "a", "body": "x", "show_if": {"nope": 1}}]}, + {"screens": [{"id": "a", "body": "x", "next": "zzz"}]}, + {"screens": [{"id": "a", "body": "x", "next": {"k": "a"}}]}, + {"screens": [{"id": "a", "body": "x", "actions": [{"id": "k", "label": "K"}], "next": {"z": "a"}}]}, + {"screens": [{"id": "a", "body": "x", "actions": [{"id": "k", "label": " "}]}]}, + {"screens": [{"id": "a", "body": "x", "actions": [{"id": "k", "label": "K"}, {"id": "k", "label": "K"}]}]}, + {"screens": [{"id": "a", "body": "x", "next": 5}]}, + {"screens": [{"id": "a", "body": "x"}], "start": "q"}, + {"title": 5, "screens": [{"id": "a", "body": "x"}]}, + {"screens": [{"id": "a", "title": 5, "body": "x"}]}, + {"screens": [{"id": "a", "body": "x" * 2001}]}, + {"screens": [{"id": "a", "body": "x", "fields": "z"}]}, + {"screens": [{"id": "a", "body": "x", "actions": 7}]}, + {"screens": [{"id": "a", "body": "x", "fields": [{"id": "f", "type": "text", "show_if": {"f": 1}}, {"id": "g", "type": "text", "show_if": {}}]}]}, + {"screens": ["str"]}, + {"screens": [{"id": "a", "body": "x", "fields": [5]}]}, + {"screens": [{"id": "a", "body": "x", "actions": [5]}]}, + {"screens": [{"id": 5, "body": "x"}]}, + {"screens": [{"id": "a" * 70, "body": "x"}]}, + ] + add(["validate_flow_spec", [base]]) + for spec in bad: + add(["validate_flow_spec", [spec]]) + ok = {"status": "ok"} + add(["run_flow", [base, [dict(ok, value=1), {"status": "ok", "value": "Zed"}, dict(ok, value=0), ok]]]) + add(["run_flow", [base, [dict(ok, value=0), dict(ok, value=1)]]]) + add(["run_flow", [base, [dict(ok, value=0), dict(ok, value=1), ok]]]) + add(["run_flow", [base, [{"status": "cancelled"}]]]) + add(["run_flow", [base, [dict(ok, value=0), {"status": "timeout"}]]]) + add(["run_flow", [{"screens": [{"id": "a", "body": "x", "next": "a"}]}, [ok] * 80]]) + add(["run_flow", [{"screens": [{"id": "a", "body": "x"}, {"id": "b", "body": "y", "show_if": {}}]}, [ok, ok]]]) + return cases + + +def proc_cases(proc): + """ShellSession.current against the fake /proc: nested shells, a shell-less chain, bad pins.""" + cases = [] + for env, pid in (({"TAI_SESSION": ""}, 200), ({"TAI_SESSION": "pinned"}, 200), ({"TAI_SESSION": "20260101-000000-p100-fa6db223"}, 200), + ({"TAI_SESSION": "20260101-000000-p999-fa6db223"}, 200), ({"TAI_SESSION": "bad id!/x"}, 200), ({}, 100), ({}, 300), + ({"TAI_SESSION": "x-p9-"}, 300), ({}, 400), ({}, 999999)): + cases.append(["shell_current", [env, pid, proc, 1761400000.5, 100]]) + for variant in (proc + "_noboot", proc + "_nobtime", proc + "_bare"): + for pid in (200, 100, 300): + cases.append(["shell_current", [{}, pid, variant, 1761400000.5, 100]]) + cases.append(["shell_current", [{}, 200, proc, 1761400000.5, 250]]) + cases.append(["shell_current", [{}, 200, proc, 1761400000.25, 0 + 100]]) + return cases + + +def build_fake_proc(root, boot_id=True, btime=True): + """A /proc with init(1) > zsh(50) > bash(100) > tai(200), plus a lone tai(300) under init and an oddly named process.""" + os.makedirs(os.path.join(root, "sys", "kernel", "random")) + if btime: + with open(os.path.join(root, "stat"), "w") as handle: + handle.write("cpu 1 2 3\nbtime 1700000000\nprocesses 5\n") + if boot_id: + with open(os.path.join(root, "sys", "kernel", "random", "boot_id"), "w") as handle: + handle.write("FA6DB223-1111-2222-3333-444444444444\n") + + def make(pid, comm, ppid, ticks, cmdline): + os.makedirs(os.path.join(root, str(pid))) + tail = "S %d 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 %d 1000 100" % (ppid, ticks) + with open(os.path.join(root, str(pid), "stat"), "w") as handle: + handle.write("%d (%s) %s\n" % (pid, comm, tail)) + with open(os.path.join(root, str(pid), "cmdline"), "wb") as handle: + handle.write(cmdline.replace(" ", "\0").encode() + b"\0") + make(1, "init", 0, 10, "/sbin/init") + make(50, "zsh", 1, 5000, "-zsh") + make(100, "bash", 50, 123456, "/usr/bin/bash --login") + make(200, "tai", 100, 123999, "/usr/bin/python3 tai.py") + make(300, "tai", 1, 700, "/opt/tai") + make(400, "my (odd) name", 100, 124000, "weird") + + +def debug_cases(): + """DebugLog: sentence and json sinks, redaction, flattening, quoting, meta caps.""" + long_text = "word " * 2000 + events = [ + ["INFO", "app", "boot", "tai booted", {"profile": "default", "meta": {"version": "1", "home": "/x y", "mode": "repl"}}], + ["DEBUG", "vault", "secret_use", "token is topsecret-9999 and more", {"profile": "default", "bot": "main", "chat": "c 1", "meta": {"name": "dbg_secret", "val": "topsecret-9999"}}], + ["WARNING", "net", "slow", "multi\nline\t message with spaces", {"meta": {"n": 3, "f": 1.5, "b": True, "none": None, "list": ["a", "b c", 1], "nested": {"k": {"k2": {"k3": {"k4": {"k5": "deep"}}}}}}}], + ["ERROR", "app", "big", "long meta", {"profile": "p", "meta": {"text": long_text, "other": "ü" * 5000, "q": 'say "hi" \\ there', "empty": ""}}], + ["INFO", "app", "huge", "x" * 3000, {"meta": {"a%d" % n: "y" * 400 for n in range(60)}}], + ["INFO", "app", "plain", "no meta", {}], + ] + return [ + ["save_secret", ["dbg_secret", "topsecret-9999"]], + ["debug_log", ["file", "sentence", events]], + ["debug_log", ["file", "json", events]], + ["log_query", [{}]], + ["log_query", [{"level": "ERROR"}]], + ["log_query", [{"component": "vault", "event": "secret_use"}]], + ["log_query", [{"limit": 2}]], + ] + + +def prune_cases(): + """More scoped session and turn files than FILES_CAP: the oldest are removed on save, the legacy file never.""" + cases = [["save_session", ["prune", [{"role": "user", "content": "legacy"}]]], ["save_turn_state", ["prune", "main", {"open": False}]]] + for n in range(24): + cases.append(["save_session", ["prune", [{"role": "user", "content": "m%d" % n}], "main", "ctx%02d" % n]]) + cases.append(["save_turn_state", ["prune", "main", {"open": False, "n": n}, "ctx%02d" % n]]) + cases.append(["save_session", ["prune", [{"role": "user", "content": "b%d" % n}], "bot1", "ctx%02d" % n]]) + cases.append(["sleep", [0.01]]) + return cases + + +def readback_cases(session): + """Read-only operations replayed by BOTH implementations on BOTH vaults (a vault is portable between them).""" + return [ + ["get_all_settings", []], ["get_all_settings", [], {"scope": "work"}], ["list_bots", ["default"]], ["load_session", ["default", "main", session]], + ["load_session", ["default", "helper", session]], ["load_system", ["default"]], ["load_system", ["work"]], ["load_bot_system", ["default", "helper"]], + ["load_turn_state", ["default", "main", session]], ["search_records", ["fox"]], ["search_records", ["fox", None, [], 8, "work"]], ["list_schedules", []], + ["history_load", ["default"]], ["list_profiles", []], ["mcp_list_servers", []], ["audit_history", []], ["list_secret_infos", []], ["known_tags", []], + ["tag_counts", []], ["log_query", [{}]], ["fts_search", ["fox"]], ["search_events", ["default", "fox"]], ["load_dir_defaults", ["/tmp/dd"]], + ["resolve_bot", ["default", "h"]], ["load_bots", ["default"]], ["file_record_id", ["/tmp/x.py"]], + ] + + +def style_cases(): + """Glow banner pieces: status bar layouts at several widths, ANSI helpers.""" + cases = [["glow_prompt", []], ["glow_title", ["1.2.3", "work"]], ["glow_title", ["v", "ünï"]]] + for width in (200, 80, 40, 30, 20, 12, 5, 1): + for yolo, agents in ((False, 0), (True, 0), (False, 3), (True, 2)): + cases.append(["statusbar", ["1.44.2", "default", "/tmp/some/long/directory/name/here", yolo, agents, width]]) + cases.append(["statusbar", ["1", "p", "@@HOME@@", False, 0, 80]]) + cases.append(["statusbar", ["1", "p", "@@HOME@@/sub/dir", False, 0, 80]]) + sample = "\033[1m\033[38;5;111mhello\033[0m \033[2mwörld\033[0m tail" + for text in (sample, "plain", "", "\033[31m\033[0m", "a\033[1mb\033[0mc\033[2m"): + cases.append(["strip_ansi", [text]]) + for width in (0, 1, 3, 5, 8, 100): + cases.append(["truncate_ansi", [text, width]]) + return cases + + +def plexer_cases(): + """NativePlexerController against the scripted backend ($SOCK) and TmuxPlexerController against a private tmux server.""" + choice = {"title": "t", "text": "pick", "kind": "choice", "options": ["a", "b"], "default": 0} + cases = [ + ["native_ctl", ["$SOCK", "7", "status", []]], + ["native_ctl", ["$SOCK", "", "status", []]], + ["native_ctl", ["$SOCK", "7", "send_keys", ["ls -l", True]]], + ["native_ctl", ["$SOCK", "7", "send_keys", ["éé", False]]], + ["native_ctl", ["$SOCK", "7", "read_pane", [50]]], + ["native_ctl", ["$SOCK", "7", "type_text", ["héy", True]]], + ["native_ctl", ["$SOCK", "7", "ask", [choice, 5]]], + ["native_ctl", ["$SOCK", "7", "ask", [{"title": "t2", "text": "name?", "kind": "text", "initial": ""}, 5]]], + ["native_ctl", ["$SOCK", "7", "ask", [dict(choice, title="cancel"), 5]]], + ["native_ctl", ["$SOCK", "7", "ask", [dict(choice, title="timeout"), 5]]], + ["native_ctl", ["$SOCK", "7", "ask", [dict(choice, title="weird"), 5]]], + ["native_ctl", ["$SOCK", "7", "ask", [dict(choice, title="never"), 1]]], + ["native_ctl", ["/nonexistent/sock", "7", "status", []]], + ["native_ctl", ["$SOCK", "7", "call", [{"cmd": "refuse"}]]], + ["native_ctl", ["$SOCK", "7", "call", [{"cmd": "boom"}]]], + ["native_ctl", ["$SOCK", "7", "call", [{"cmd": "garbage"}]]], + ["native_ctl", ["$SOCK", "7", "call", [{"cmd": "nope"}]]], + ["tmux_ctl", ["%0", "status", []]], + ["tmux_ctl", ["%0", "send_keys", ["hello world", True]]], + ["sleep", [0.5]], + ["tmux_ctl", ["%0", "read_pane", [5]]], + ["tmux_ctl", ["%0", "type_text", ["abc", True]]], + ["sleep", [0.5]], + ["tmux_ctl", ["%0", "read_pane", [8]]], + ["tmux_ctl", ["%99", "status", []]], + ["tmux_ctl", ["%99", "send_keys", ["x", False]]], + ["tmux_ctl", ["%99", "read_pane", [5]]], + ["tmux_ctl", ["%0", "ask", [choice, 2]]], + ["tmux_ctl", ["%0", "ask", [{"title": "p", "text": "q", "kind": "text", "initial": ""}, 2]]], + ["tmux_ctl", ["%0", "ask", [{"title": "p", "text": "q", "kind": "message"}, 2]]], + ["tmux_ctl", ["%0", "ask", [{"title": "p", "kind": "bogus"}, 2]]], + ] + return cases + + +def repl_cases(): + """REPL slash commands that the session layer owns, run on a shared store.""" + script = ["/profiles", "/bots", "/bot", "/bot nobody", "/bot helper", "/bots", "/bot H", "/bot main", "/profile", "/profile bad name", "/profile -x", + "/profile work", "/profiles", "/profile default", "/clear", "/ctx", "/help", "/zzz", "/", "/PROFILES", "/bots extra args", "/quit"] + return [["repl_script", [script]]] diff --git a/taicpp/tests/foundation_fixtures.py b/taicpp/tests/foundation_fixtures.py new file mode 100644 index 0000000..3fcfc42 --- /dev/null +++ b/taicpp/tests/foundation_fixtures.py @@ -0,0 +1,116 @@ +"""Process fixtures for tests/foundation_parity.py: a scripted native-plexer +backend on a unix socket and an isolated tmux server.""" +import json +import os +import shutil +import socket +import subprocess +import threading + + +class FakeBackend: + """Answers the NativePlexerBackend line protocol and logs every request. + + send/read/status answer fixed data; show/result run a two-poll flow whose + outcome follows the screen title (cancel, timeout, weird, error, else ok); + cmd "garbage" answers non-JSON and cmd "refuse" answers ok=false.""" + + def __init__(self, path): + self.path = path + self.log = [] + self.flows = {} + self.counter = 0 + self.server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.server.bind(path) + self.server.listen(8) + self.thread = threading.Thread(target=self.serve, daemon=True) + self.thread.start() + + def serve(self): + while True: + try: + conn, _ = self.server.accept() + except OSError: + return + with conn: + data = b"" + while not data.endswith(b"\n"): + chunk = conn.recv(65536) + if not chunk: + break + data += chunk + try: + request = json.loads(data.decode("utf-8")) + except ValueError: + continue + self.log.append(request) + conn.sendall(self.answer(request)) + + def answer(self, request): + cmd = request.get("cmd") + if cmd == "send": + return self.ok(True) + if cmd == "read": + return self.ok("screen of %s lines\nsecond line ü" % request.get("lines")) + if cmd == "status": + return self.ok({"backend": "native", "pane": request.get("pane")}) + if cmd == "garbage": + return b"this is not json\n" + if cmd == "refuse": + return (json.dumps({"ok": False}) + "\n").encode() + if cmd == "boom": + return (json.dumps({"ok": False, "error": "boom"}) + "\n").encode() + if cmd == "show": + self.counter += 1 + flow_id = "f%d" % self.counter + self.flows[flow_id] = {"screen": request.get("screen") or {}, "polls": 0} + return self.ok({"id": flow_id}) + if cmd == "result": + flow = self.flows.get(request.get("id")) + if flow is None: + return (json.dumps({"ok": False, "error": "unknown flow"}) + "\n").encode() + flow["polls"] += 1 + if flow["polls"] < 2: + return self.ok({"done": False}) + title = flow["screen"].get("title") + if title == "cancel": + return self.ok({"done": True, "status": "cancelled"}) + if title == "timeout": + return self.ok({"done": True, "status": "timeout"}) + if title == "weird": + return self.ok({"done": True, "status": "weird"}) + if title == "never": + return self.ok({"done": False}) + return self.ok({"done": True, "status": "ok", "value": 1 if flow["screen"].get("kind") == "choice" else "typed"}) + if cmd == "cancel": + return self.ok(True) + return (json.dumps({"ok": False, "error": "unknown cmd %s" % cmd}) + "\n").encode() + + @staticmethod + def ok(data): + return (json.dumps({"ok": True, "data": data}) + "\n").encode() + + def close(self): + try: + self.server.close() + finally: + if os.path.exists(self.path): + os.unlink(self.path) + + +class TmuxServer: + """A private tmux server (own TMUX_TMPDIR) with one session running `cat`, so typed text is echoed back.""" + + def __init__(self, root): + self.tmpdir = os.path.join(root, "tmux") + os.makedirs(self.tmpdir) + self.env = {key: value for key, value in os.environ.items() if not key.startswith("TMUX")} + self.env["TMUX_TMPDIR"] = self.tmpdir + self.available = shutil.which("tmux") is not None + if self.available: + subprocess.run(["tmux", "-f", "/dev/null", "new-session", "-d", "-s", "t", "-x", "80", "-y", "24", "cat"], env=self.env, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def close(self): + if self.available: + subprocess.run(["tmux", "kill-server"], env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/taicpp/tests/foundation_parity.py b/taicpp/tests/foundation_parity.py new file mode 100644 index 0000000..419a9cf --- /dev/null +++ b/taicpp/tests/foundation_parity.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""Vault parity: replay one scenario through tai.py and through the C++ Store +(build/foundation_probe) against two throwaway TAI_HOMEs, then diff the +operation results, every table, the FTS index, the schema and every file under +profiles/. Timestamps and random record ids are masked; everything else must +be identical. + +Usage: python3 tests/foundation_parity.py [--probe PATH] [--keep] +""" +import argparse +import json +import os +import re +import sqlite3 +import subprocess +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import foundation_cases as cases # noqa: E402 +import foundation_fixtures as fixtures # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SESSION = "pane-parity-ctx" +BIG = ("Zażółć gęślą jaźń 😀 line %d fox cats\n" * 300) % tuple(range(300)) +BIG = "".join("Zażółć gęślą jaźń 😀 line %d fox cats\n" % n for n in range(300)) + +SCENARIO = [ + ["add_record", ["note", "Alpha plan", "The quick brown fox jumps over the lazy dogs. cats and mice", ["Animals", "fox"]]], # 0 + ["add_record", ["note", "Second note", "foxes and cats share tags with animals", ["animal"]]], # 1 + ["add_record", ["file", "/tmp/x.py", "print('héllo wörld 😀')", ["python"]]], # 2 + ["add_record", ["output", "Big output", BIG, []]], # 3 + ["add_chunks", ["record", "$R3", ["chunk a fox", "chunk b"]]], # 4 + ["get_record", ["$R0"]], # 5 + ["read_record", ["$R3", 4, 25]], # 6 + ["search_records", ["fox"]], # 7 + ["search_records", ["fox animals"]], # 8 + ["search_records", ["", "note"]], # 9 + ["search_records", ["zzzz"]], # 10 + ["search_records", ["", None, ["animals"]]], # 11 + ["tag_counts", []], # 12 + ["add_edge", ["$R0", "$R1", "Relates To!"]], # 13 + ["add_edge", ["$R0", "mem:deadbeefdeadbeef"]], # 14 + ["edges_for", ["$R0"]], # 15 + ["traverse", ["$R0"]], # 16 + ["save_secret", ["api_key", "sekrit-value-123", {"k": 1, "z": "ü"}, ["svc"]]], # 17 + ["secret_meta", ["api_key"]], # 18 + ["list_secret_infos", []], # 19 + ["load_secret", ["api_key"]], # 20 + ["redact", ["my token sekrit-value-123 here"]], # 21 + ["add_edge", ["$R0", "secret:api_key", "uses"]], # 22 + ["add_schedule", ["daily", "do the thing", "default", 3600, "2026-01-01T00:00:00+00:00", 60]], # 23 + ["list_schedules", []], # 24 + ["add_edge", ["$R0", "sched:1"]], # 25 + ["traverse", ["$R0", 3]], # 26 + ["audit_event", ["main", "write", "/tmp/x.py", "msg about fox", "old text", "new text", ["Edit", "py"]]], # 27 + ["audit_history", ["/tmp/x.py"]], # 28 + ["audit_get", ["$R27"]], # 29 + ["audit_search", ["fox"]], # 30 + ["upsert_file_record", ["/tmp/y.md", "# Title\nfox content"]], # 31 + ["file_record_id", ["/tmp/y.md"]], # 32 + ["upsert_file_record", ["/tmp/y.md", "# Title 2\nupdated fox", None]], # 33 + ["delete_file_record", ["/tmp/y.md"]], # 34 + ["log_event", ["default", "user", "say", "hello fox world", ["a"]]], # 35 + ["log_event", ["default", "assistant", "say", "bye"]], # 36 + ["search_events", ["default", "fox"]], # 37 + ["events_like", ["default", "bye"]], # 38 + ["history_add", ["default", "main", "first input"]], # 39 + ["history_add", ["default", "main", "first input"]], # 40 + ["history_add", ["default", "main", " padded ünï "]], # 41 + ["history_load", ["default"]], # 42 + ["fts_search", ["fox"]], # 43 + ["fts_search", ["fox", ["record"]]], # 44 + ["save_system", ["default", "You are a test."]], # 45 + ["load_system", ["default"]], # 46 + ["save_session", ["default", [{"role": "system", "content": "S"}, {"role": "user", "content": "hi ü 😀"}, + {"role": "assistant", "content": "yo", "tool_calls": [{"id": "1", "function": {"name": "shell", "arguments": "{}"}}]}, + {"role": "tool", "content": "x", "tool_call_id": "1"}], "main", SESSION]], # 47 + ["load_session", ["default", "main", SESSION]], # 48 + ["load_session", ["default", "main", "other-ctx"]], # 49 + ["save_bots", ["default", {"helper": {"nicknames": ["h"]}, "zed": {}}]], # 50 + ["list_bots", ["default"]], # 51 + ["resolve_bot", ["default", "H"]], # 52 + ["resolve_bot", ["default", "main"]], # 53 + ["resolve_bot", ["default", "nope"]], # 54 + ["save_bot_system", ["default", "helper", "I am helper"]], # 55 + ["load_bot_system", ["default", "helper"]], # 56 + ["save_session", ["default", [{"role": "user", "content": "to helper"}], "helper", SESSION]], # 57 + ["load_session", ["default", "helper", SESSION]], # 58 + ["save_turn_state", ["default", "main", {"open": True, "goal": "ship it", "steps": 3, "tools": 2, "updated": "2026-01-01T00:00:00+00:00"}, SESSION]], # 59 + ["load_turn_state", ["default", "main", SESSION]], # 60 + ["get_or_create_settings", []], # 61 + ["upsert_settings", [{"model": "foo", "voice": "v", "retry_seconds": 2.5, "zen_enabled": "yes", "theme": "nord", "openrouter_key": "abc", "gemini_plan": True}]], # 62 + ["upsert_settings", [{"bogus": 1}]], # 63 + ["upsert_settings", [{"theme": "neon"}]], # 64 + ["upsert_settings", [{"model": None}]], # 65 + ["upsert_settings", [{"retry_seconds": -1}]], # 66 + ["upsert_settings", [{"zen_enabled": "maybe"}]], # 67 + ["upsert_settings", [{"muse_bin": None, "opencode_models": "a,b"}], {"scope": "work"}], # 68 + ["get_all_settings", [], {"scope": "work"}], # 69 + ["get_or_create_settings", [["model", "voice"]], {"scope": "bad scope!"}], # 70 + ["mcp_save_server", [{"name": "srv", "transport": "stdio", "command": "x", "args": ["a", "b"], "env": {"K": "v"}, "headers": {"Authorization": "Bearer t"}, "allow": ["t1"], "source": "test"}]], # 71 + ["mcp_save_server", [{"name": "bad name!", "transport": "stdio"}]], # 72 + ["mcp_get_server", ["srv"]], # 73 + ["mcp_store_tools", ["srv", [{"name": "t1", "description": "d1", "inputSchema": {"type": "object", "b": 1, "a": 2}}, {"name": "t2"}, {"nope": 1}]]], # 74 + ["mcp_cached_tools", ["srv"]], # 75 + ["mcp_tool_pins", ["srv"]], # 76 + ["mcp_set_enabled", ["srv", "default", True]], # 77 + ["mcp_enabled_servers", ["default"]], # 78 + ["mcp_list_servers", []], # 79 + ["save_dir_defaults", ["/tmp/dd", "prov", "mod", "bot1"]], # 80 + ["load_dir_defaults", ["/tmp/dd"]], # 81 + ["save_dir_defaults", ["/tmp/dd", None, None, None, ["model"]]], # 82 + ["load_dir_defaults", ["/tmp/dd"]], # 83 + ["spill_or_truncate", ["output", "spilled", BIG]], # 84 + ["spill_or_truncate", ["output", "small", "short text"]], # 85 + ["known_tags", []], # 86 + ["canonical_tags", [["Animals", "new tag", "cats"]]], # 87 + ["tagged_items", [["record", "fox"]]], # 88 + ["item_tags", ["$R0"]], # 89 + ["delete_record", ["$R1"]], # 90 + ["delete_secret", ["api_key"]], # 91 + ["remove_schedule", [1]], # 92 + ["search_records", ["fox"]], # 93 + ["add_record", ["note", "Work note", "work profile fox", ["w"], "work"]], # 94 + ["search_records", ["fox", None, [], 8, "work"]], # 95 + ["mcp_remove_server", ["srv"]], # 96 + ["list_profiles", []], # 97 + ["get_chunks_by_ids", [[1, 2, 999]]], # 98 +] + +UNORDERED = {"tagged_items"} +TS = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?\+00:00") +MEM = re.compile(r"mem:[0-9a-f]{16}") +EPHEMERAL = re.compile(r"(\d{8}-\d{6}-e\d+)-[0-9a-f]{4}") + + +PURE_OPS = {"prompt", "capability_block", "orientation_block", "settings_catalog", "json_dumps", "json_roundtrip", "float_repr", "normalize_tag", "normalize_tags", + "singular_noun", "tag_variants", "parse_node_id", "fts_match_variants", "fts_rank", "profile_re", "bot_re", "secret_re", "mcp_server_re", + "normalize_setting_value", "parse_setting_env", "parse_setting_stored", "parse_backends", "split_models", "chunk_lines", "truncate", "sanitize_sid", + "shell_current", "validate_flow_spec", "run_flow", "sleep", "native_ctl", "tmux_ctl", + "statusbar", "strip_ansi", "truncate_ansi", "glow_prompt", "glow_title"} + + +def special(tai, op, pos): + """Module-level functions of tai.py that have no Store: the python side of the pure cases.""" + if op == "prompt": + return getattr(tai, pos[0]) + if op == "capability_block": + return tai.capability_block(*pos) + if op == "orientation_block": + return tai.orientation_block(*pos) + if op == "settings_catalog": + return {key: {"key": spec["key"], "env": spec["env"], "type": spec["type"], "default": spec["default"], "desc": spec["desc"], "required": spec["required"], + "nullable": spec["nullable"], "encrypted": spec["encrypted"], "options": spec["options"], "preferred": spec["preferred"], + "true_words": spec["true_words"], "false_words": spec["false_words"]} for key, spec in tai.SETTINGS_CATALOG.items()} + if op == "json_dumps": + return [json.dumps(pos[0]), json.dumps(pos[0], sort_keys=True)] + if op == "json_roundtrip": + try: + return json.dumps(json.loads(pos[0]), sort_keys=True) + except ValueError: + return {"ValueError": "x"} + if op == "float_repr": + return repr(float(pos[0])) + if op == "normalize_tag": + return tai.normalize_tag(pos[0]) + if op == "normalize_tags": + return tai.normalize_tags(pos[0]) + if op == "singular_noun": + return tai.singular_noun(pos[0]) + if op == "tag_variants": + return tai.tag_variants(pos[0]) + if op == "parse_node_id": + return tai.parse_node_id(pos[0]) + if op == "fts_match_variants": + terms = tai.fts_escape(pos[0]) + return None if terms is None else tai.fts_match_variants(terms) + if op == "fts_rank": + return tai.fts_rank(pos[0]) + if op in ("profile_re", "bot_re", "secret_re", "mcp_server_re"): + regex = {"profile_re": tai.PROFILE_RE, "bot_re": tai.BOT_RE, "secret_re": tai.SECRET_RE, "mcp_server_re": tai.MCP_SERVER_RE}[op] + return regex.match(pos[0]) is not None + if op == "normalize_setting_value": + try: + return tai.normalize_setting_value(pos[0], tai.SETTINGS_CATALOG[pos[0]], pos[1]) + except ValueError as exc: + return {"ValueError": str(exc)} + if op == "parse_setting_env": + try: + return tai.parse_setting_env(tai.SETTINGS_CATALOG[pos[0]], pos[1].strip()) + except ValueError as exc: + return {"ValueError": str(exc)} + if op == "parse_setting_stored": + try: + return tai.parse_setting_stored(tai.SETTINGS_CATALOG[pos[0]], pos[1]) + except ValueError: + return {"ValueError": "could not convert"} + if op == "parse_backends": + return tai.parse_backends(pos[0]) + if op == "split_models": + return tai.split_models(pos[0]) + if op == "chunk_lines": + return tai.chunk_lines(*pos) + if op == "truncate": + return tai.truncate(*pos) + if op == "sanitize_sid": + return tai.ShellSession.sanitize(pos[0]) + if op == "shell_current": + env, pid, root, now, tick = pos + c = tai.ShellSession.current(env=env, pid=pid, proc_root=root, now=now, clk_tck=tick) + return [c.sid, c.shell, c.pid, c.started, c.parent, c.chain] + if op == "statusbar": + return tai.Glow.statusbar(pos[0], pos[1], pos[2], yolo=pos[3], agents=pos[4], width=pos[5]) + if op == "strip_ansi": + return tai.strip_ansi(pos[0]) + if op == "truncate_ansi": + return tai.truncate_ansi(pos[0], pos[1]) + if op == "glow_prompt": + return tai.Glow.prompt_text() + if op == "glow_title": + A, version, profile = tai.Ansi, pos[0], pos[1] + return "%s◆%s %stai%s %s%s%s %s· profile %s%s" % (A.BOLD, A.RESET, A.BOLD, A.RESET, tai.FG_FILE, version, A.RESET, A.DIM, profile, A.RESET) + if op == "sleep": + import time + time.sleep(pos[0]) + return None + if op in ("native_ctl", "tmux_ctl"): + if op == "native_ctl": + sock, pane, method, args = pos + ctl = tai.NativePlexerController(sock, pane or None) + else: + pane, method, args = pos + ctl = tai.TmuxPlexerController(pane) + try: + if method == "type_text": + return ctl.type_text(args[0], 0.0, args[1]) + if method == "call": + return ctl._call(args[0]) + return getattr(ctl, method)(*args) + except tai.PlexerError as exc: + return {"PlexerError": str(exc)} + if op == "validate_flow_spec": + app, error = tai.validate_flow_spec(pos[0]) + return [app if not error else None, error] + if op == "run_flow": + app, error = tai.validate_flow_spec(pos[0]) + if error: + return [None, error] + + class Fake(tai.PlexerController): + def __init__(self, replies): + self.replies = list(replies) + self.asked = [] + + def ask(self, screen, timeout): + self.asked.append(screen) + return self.replies.pop(0) + fake = Fake(pos[1]) + outcome = tai.PlexerFlow(app).run(fake) + return [outcome, fake.asked] + raise KeyError(op) + + +STORE_OPS = {"debug_log", "repl_script", "log_query"} + + +def special_store(tai, op, pos, store, config): + """tai.py behaviour that needs the live store: DebugLog sinks, REPL commands, log queries.""" + import contextlib + import io + if op == "log_query": + return store.log_query(**pos[0]) + if op == "debug_log": + sink, fmt, events = pos + tai.LOG.configure(sink, fmt, config.home, store=store) + for level, component, event, message, kwargs in events: + tai.LOG.event(level, component, event, message, **kwargs) + run = tai.LOG.run + tai.LOG.close() + tai.LOG.reset() + with open(os.path.join(config.home, "debug.log"), "r", encoding="utf-8") as handle: + return [run, handle.read()] + if op == "repl_script": + agent = tai.Agent(config, store, ctx=os.environ.get("TAI_SESSION")) + outputs = [] + for text in pos[0]: + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + try: + cont = tai.handle_command(agent, text) + except IndexError: + print("unknown command, try /help") # python crashes on a bare "/"; the C++ port reports it as unknown + cont = True + outputs.append([buf.getvalue(), cont]) + return [outputs, [agent.profile, agent.bot, agent.messages[1:]]] + raise KeyError(op) + + +def run_python(home, scenario_path): + """Executed in a child interpreter: replay through tai.py.""" + sys.path.insert(0, ROOT) + import tai + import argparse as ap + args = ap.Namespace(profile="default", yes=False, no_opencode=False) + config = tai.Config(args) + seal = tai.Seal(config.home, "") + store = tai.Store(config, seal) + with open(scenario_path, "r", encoding="utf-8") as handle: + scenario = json.load(handle) + results = [] + + def sub(value): + if isinstance(value, str) and value.startswith("$R") and value[2:].isdigit(): + return results[int(value[2:])] + if isinstance(value, list): + return [sub(item) for item in value] + if isinstance(value, dict): + return {key: sub(item) for key, item in value.items()} + return value + + def plain(value): + if isinstance(value, tuple): + return [plain(item) for item in value] + if isinstance(value, list): + return [plain(item) for item in value] + if isinstance(value, set): + return sorted(value) + if isinstance(value, dict): + return {str(key): plain(item) for key, item in value.items()} + return value + + for step in scenario: + op, pos = step[0], sub(step[1]) if len(step) > 1 else [] + kw = sub(step[2]) if len(step) > 2 else {} + try: + if op in STORE_OPS: + out = special_store(tai, op, pos, store, config) + elif op in PURE_OPS: + out = special(tai, op, pos) + elif op == "spill_or_truncate": + out = tai.spill_or_truncate(store, *pos, **kw) + elif op == "mcp_set_enabled": + out = store.mcp_set_enabled(*pos, **kw) + else: + out = getattr(store, op)(*pos, **kw) + except ValueError as exc: + out = {"ValueError": str(exc)} + results.append(plain(out)) + store.close() + print(json.dumps(results, sort_keys=True)) + + +def dump_vault(home): + out = {"tables": {}, "schema": [], "files": {}} + db = sqlite3.connect(os.path.join(home, "memory.db")) + for name, sql in db.execute("SELECT name, sql FROM sqlite_master WHERE name NOT LIKE 'fts_docs_%' AND name NOT LIKE 'sqlite_%' ORDER BY type, name").fetchall(): + out["schema"].append([name, " ".join((sql or "").split())]) + tables = [row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'fts_docs_%' AND name NOT LIKE 'sqlite_%' ORDER BY name").fetchall()] + for table in tables: + cols = [row[1] for row in db.execute("PRAGMA table_info(%s)" % table).fetchall()] + if table == "fts_docs": + rows = db.execute("SELECT item, kind, sub, title, body, profile FROM fts_docs").fetchall() + cols = ["item", "kind", "sub", "title", "body", "profile"] + else: + rows = db.execute("SELECT * FROM %s" % table).fetchall() + out["tables"][table] = {"columns": cols, "rows": [list(row) for row in rows]} + db.close() + prof = os.path.join(home, "profiles") + for entry in sorted(os.listdir(prof)): + with open(os.path.join(prof, entry), "r", encoding="utf-8") as handle: + out["files"][entry] = handle.read() + return out + + +def normalizer(results, scenario): + runs = [results[pos][0] for pos, step in enumerate(scenario) if step[0] == "debug_log"] + order = [] + for blob in (json.dumps(results, sort_keys=True),): + for match in MEM.finditer(blob): + if match.group(0) not in order: + order.append(match.group(0)) + names = {mem: "" % pos for pos, mem in enumerate(order)} + + def norm(value): + if isinstance(value, str): + for run in runs: + value = value.replace(run, "") + value = TS.sub("", value) + value = EPHEMERAL.sub(r"\1-", value) + return MEM.sub(lambda m: names.get(m.group(0), ""), value) + if isinstance(value, float): + return float("%.9g" % value) + if isinstance(value, list): + return [norm(item) for item in value] + if isinstance(value, dict): + return {norm(key): norm(item) for key, item in value.items()} + return value + return norm + + +def settle(dump, norm): + dump = norm(dump) + for table in dump["tables"].values(): + table["rows"].sort(key=lambda row: json.dumps(row, sort_keys=True, default=str)) + return dump + + +def first_diff(a, b, path="$"): + if type(a) is not type(b): + return "%s: type %s vs %s: %r vs %r" % (path, type(a).__name__, type(b).__name__, str(a)[:200], str(b)[:200]) + if isinstance(a, dict): + for key in sorted(set(a) | set(b)): + if key not in a or key not in b: + return "%s.%s: only in %s" % (path, key, "python" if key in a else "cpp") + found = first_diff(a[key], b[key], "%s.%s" % (path, key)) + if found: + return found + return None + if isinstance(a, list): + if len(a) != len(b): + return "%s: length %d vs %d" % (path, len(a), len(b)) + for pos, (x, y) in enumerate(zip(a, b)): + found = first_diff(x, y, "%s[%d]" % (path, pos)) + if found: + return found + return None + if a != b: + return "%s: %r vs %r" % (path, str(a)[:300], str(b)[:300]) + return None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--probe", default=os.path.join(os.path.dirname(HERE), "build", "foundation_probe")) + parser.add_argument("--keep", action="store_true") + parser.add_argument("--show", nargs="*", default=[], metavar="OP", help="print the C++ result of every step with this op") + parser.add_argument("--py-runner", nargs=2, metavar=("HOME", "SCENARIO")) + opts = parser.parse_args() + if opts.py_runner: + run_python(*opts.py_runner) + return 0 + work = tempfile.mkdtemp(prefix="detz-parity-") + proc = os.path.join(work, "proc") + cases.build_fake_proc(proc) + cases.build_fake_proc(proc + "_noboot", boot_id=False) + cases.build_fake_proc(proc + "_nobtime", btime=False) + cases.build_fake_proc(proc + "_bare", boot_id=False, btime=False) + scenario = SCENARIO + cases.debug_cases() + cases.repl_cases() + cases.pure_cases() + cases.flow_cases() + cases.proc_cases(proc) + cases.style_cases() + cases.prune_cases() + cases.plexer_cases() + runs = {} + for label in ("python", "cpp"): + home = os.path.join(work, label) + os.makedirs(home) + backend = fixtures.FakeBackend(os.path.join(work, label + ".sock")) + tmux_server = fixtures.TmuxServer(os.path.join(work, label + "-tmux-root")) + scenario_path = os.path.join(work, label + "-scenario.json") + with open(scenario_path, "w", encoding="utf-8") as handle: + json.dump(json.loads(json.dumps(scenario).replace("$SOCK", backend.path).replace("@@HOME@@", work)), handle) + env = {key: value for key, value in os.environ.items() if not key.startswith(("TAI_", "TMUX"))} + env.update({"TAI_HOME": home, "TAI_SESSION": SESSION, "TAI_PASSPHRASE": "", "HOME": work, "TMUX_TMPDIR": tmux_server.tmpdir}) + if label == "python": + cmd = [sys.executable, "-I", os.path.abspath(__file__), "--py-runner", home, scenario_path] + else: + cmd = [opts.probe, scenario_path] + try: + done = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=300) + finally: + backend.close() + tmux_server.close() + if done.returncode != 0: + print("%s run failed (%d):\n%s" % (label, done.returncode, done.stderr[-2000:])) + return 1 + runs[label] = {"results": json.loads(done.stdout), "dump": dump_vault(home), "requests": backend.log} + # vault portability: each implementation reads the vault the other one wrote + read_path = os.path.join(work, "readback.json") + with open(read_path, "w", encoding="utf-8") as handle: + json.dump(cases.readback_cases(SESSION), handle) + + def read_vault(impl, home): + env = {key: value for key, value in os.environ.items() if not key.startswith(("TAI_", "TMUX"))} + env.update({"TAI_HOME": home, "TAI_SESSION": SESSION, "TAI_PASSPHRASE": "", "HOME": work}) + cmd = [sys.executable, "-I", os.path.abspath(__file__), "--py-runner", home, read_path] if impl == "python" else [opts.probe, read_path] + done = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=120) + if done.returncode != 0: + raise RuntimeError("%s readback failed:\n%s" % (impl, done.stderr[-1500:])) + return json.loads(done.stdout) + + portability = 0 + for home_label in ("python", "cpp"): + home = os.path.join(work, home_label) + by_python, by_cpp = read_vault("python", home), read_vault("cpp", home) + for pos, (a, b) in enumerate(zip(by_python, by_cpp)): + diff = first_diff(a, b) + if diff: + portability += 1 + print("PORTABILITY %s vault, step %d (%s): %s" % (home_label, pos, cases.readback_cases(SESSION)[pos][0], diff)) + py_norm, cpp_norm = normalizer(runs["python"]["results"], scenario), normalizer(runs["cpp"]["results"], scenario) + py_res, cpp_res = py_norm(runs["python"]["results"]), cpp_norm(runs["cpp"]["results"]) + failures = portability + for pos, step in enumerate(scenario): + if step[0] in opts.show: + print("%d %s -> %s" % (pos, step[0], json.dumps(cpp_res[pos], ensure_ascii=False)[:300])) + for pos, step in enumerate(scenario): + if step[0] in UNORDERED: # SQL group order follows the random record ids + py_res[pos] = sorted(py_res[pos], key=str) + cpp_res[pos] = sorted(cpp_res[pos], key=str) + for pos, (a, b) in enumerate(zip(py_res, cpp_res)): + diff = first_diff(a, b) + if diff: + failures += 1 + print("RESULT %d (%s) differs: %s" % (pos, scenario[pos][0], diff)) + if len(py_res) != len(cpp_res): + failures += 1 + print("result count %d vs %d" % (len(py_res), len(cpp_res))) + py_dump, cpp_dump = settle(runs["python"]["dump"], py_norm), settle(runs["cpp"]["dump"], cpp_norm) + diff = first_diff(py_dump["schema"], cpp_dump["schema"]) + if diff: + failures += 1 + print("SCHEMA differs: %s" % diff) + for table in sorted(set(py_dump["tables"]) | set(cpp_dump["tables"])): + diff = first_diff(py_dump["tables"].get(table), cpp_dump["tables"].get(table)) + if diff: + failures += 1 + print("TABLE %s differs: %s" % (table, diff)) + diff = first_diff(py_dump["files"], cpp_dump["files"]) + if diff: + failures += 1 + print("FILES differ: %s" % diff) + diff = first_diff(runs["python"]["requests"], runs["cpp"]["requests"]) + if diff: + failures += 1 + print("NATIVE BACKEND REQUESTS differ: %s" % diff) + total_rows = sum(len(t["rows"]) for t in py_dump["tables"].values()) + print("%d operations, %d tables, %d rows, %d profile files compared, %d differences" % (len(py_res), len(py_dump["tables"]), total_rows, len(py_dump["files"]), failures)) + if opts.keep: + print("kept %s" % work) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tests/foundation_probe.cpp b/taicpp/tests/foundation_probe.cpp new file mode 100644 index 0000000..25714fb --- /dev/null +++ b/taicpp/tests/foundation_probe.cpp @@ -0,0 +1,517 @@ +// Scenario runner for tests/foundation_parity.py: replays a JSON list of +// Store operations against a real vault in $TAI_HOME and prints the results as +// sorted-key JSON. The python side replays the same scenario through tai.py; +// the driver diffs both results and both vault dumps. +// +// Usage: foundation_probe (needs TAI_HOME, TAI_SESSION) +#include +#include +#include +#include +#include +#include +#include +#include "config/config.h" +#include "config/prompts.h" +#include "config/settings_rules.h" +#include +#include +#include +#include "plexer/flow_validator.h" +#include "plexer/plexer.h" +#include "repl/repl.h" +#include "repl/repl_style.h" +#include "session/conversation.h" +#include "store/debuglog.h" +#include "plexer/plexer_flow.h" +#include "session/shell_context.h" +#include "store/fts.h" +#include "store/patterns.h" +#include "store/tag_rules.h" +#include "store/json.h" +#include "store/spill.h" +#include "store/store.h" +using detz::Config; +using detz::store::Json; +using detz::store::NullSeal; +using detz::store::Store; + +namespace { + +struct Args { + const Json* pos; + const Json* kw; + const Json* get(size_t i, const char* name) const { + if (pos && pos->is_array() && i < pos->size()) return &pos->items()[i]; + if (kw && kw->is_object()) return kw->find(name); + return nullptr; + } + bool present(size_t i, const char* name) const { const Json* v = get(i, name); return v != nullptr; } + std::string str(size_t i, const char* name, const std::string& d = "") const { const Json* v = get(i, name); return (v && v->is_string()) ? v->as_string() : d; } + long long num(size_t i, const char* name, long long d) const { const Json* v = get(i, name); return (v && v->is_number()) ? v->as_int() : d; } + std::vector list(size_t i, const char* name) const { + std::vector out; + const Json* v = get(i, name); + if (v && v->is_array()) for (auto& x : v->items()) out.push_back(x.as_string()); + return out; + } + Json json(size_t i, const char* name) const { const Json* v = get(i, name); return v ? *v : Json(); } + bool is_null(size_t i, const char* name) const { const Json* v = get(i, name); return !v || v->is_null(); } + std::optional opt_str(size_t i, const char* name) const { const Json* v = get(i, name); if (!v || v->is_null()) return std::nullopt; return v->as_string(); } +}; + +Json strs(const std::vector& v) { Json a = Json::array(); for (auto& s : v) a.push(Json(s)); return a; } +Json arr(const std::vector& v) { Json a = Json::array(); for (auto& s : v) a.push(s); return a; } +Json opt(const std::optional& v) { return v ? *v : Json(); } +Json opt_s(const std::optional& v) { return v ? Json(*v) : Json(); } + +Json value_json(const detz::SettingValue& v) { + switch (v.kind) { + case detz::SettingValue::Kind::Null: return Json(); + case detz::SettingValue::Kind::Bool: return Json(v.b); + case detz::SettingValue::Kind::Float: return Json(v.d); + default: return Json(v.s); + } +} + +Json setting_json(const detz::store::SettingRow& r) { + Json o = Json::object(); + o.set("key", r.key); o.set("scope", r.scope); o.set("type", r.type); o.set("default", value_json(r.def)); + o.set("required", r.required); o.set("nullable", r.nullable); + o.set("options", r.has_options ? strs(r.options) : Json()); + o.set("preferred", Json()); o.set("encrypted", r.encrypted); o.set("env", r.env); o.set("desc", r.desc); + o.set("stored", opt_s(r.stored)); o.set("value", value_json(r.value)); o.set("source", r.source); + o.set("created", opt_s(r.created)); o.set("updated", opt_s(r.updated)); + return o; +} + +Json rows_json(const std::vector& rows) { + Json a = Json::array(); + for (auto& r : rows) a.push(setting_json(r)); + return a; +} + +Json events_json(const std::vector& rows) { + Json a = Json::array(); + for (auto& r : rows) a.push(arr({Json(r.ts), Json(r.role), Json(r.kind), Json(r.text)})); + return a; +} + +using Handler = std::function; + +std::map handlers() { + std::map h; + h["add_record"] = [](Store& s, const Args& a) { return Json(s.add_record(a.str(0, "kind"), a.str(1, "title"), a.str(2, "content"), a.list(3, "tags"), a.str(4, "profile"))); }; + h["get_record"] = [](Store& s, const Args& a) { return opt(s.get_record(a.str(0, "record_id"), a.str(1, "profile"))); }; + h["read_record"] = [](Store& s, const Args& a) { return opt(s.read_record(a.str(0, "record_id"), a.num(1, "offset", 0), a.num(2, "limit", 4000), a.str(3, "profile"))); }; + h["search_records"] = [](Store& s, const Args& a) { return arr(s.search_records(a.str(0, "query"), a.str(1, "kind"), a.list(2, "tags"), static_cast(a.num(3, "limit", 8)), a.str(4, "profile"))); }; + h["delete_record"] = [](Store& s, const Args& a) { return Json(s.delete_record(a.str(0, "record_id"), a.str(1, "profile"))); }; + h["save_secret"] = [](Store& s, const Args& a) { + std::optional> tags; + if (!a.is_null(3, "tags")) tags = a.list(3, "tags"); + s.save_secret(a.str(0, "name"), a.str(1, "value"), a.json(2, "meta"), tags, a.str(4, "profile")); + return Json(); + }; + h["secret_meta"] = [](Store& s, const Args& a) { return s.secret_meta(a.str(0, "name"), a.str(1, "profile")); }; + h["list_secret_infos"] = [](Store& s, const Args& a) { return arr(s.list_secret_infos(a.str(0, "profile"))); }; + h["load_secret"] = [](Store& s, const Args& a) { return opt_s(s.load_secret(a.str(0, "name"), a.str(1, "profile"))); }; + h["delete_secret"] = [](Store& s, const Args& a) { return Json(s.delete_secret(a.str(0, "name"), a.str(1, "profile"))); }; + h["redact"] = [](Store& s, const Args& a) { return Json(s.redact(a.str(0, "text"), a.str(1, "profile"))); }; + h["known_tags"] = [](Store& s, const Args& a) { auto t = s.known_tags(a.str(0, "profile")); return strs(std::vector(t.begin(), t.end())); }; + h["canonical_tags"] = [](Store& s, const Args& a) { return strs(s.canonical_tags(a.list(0, "tags"), nullptr, a.str(2, "profile"))); }; + h["tag_counts"] = [](Store& s, const Args& a) { + Json out = Json::array(); + for (auto& [t, n] : s.tag_counts(a.str(0, "prefix"), static_cast(a.num(1, "limit", 50)), a.str(2, "profile"))) out.push(arr({Json(t), Json(static_cast(n))})); + return out; + }; + h["tag_item"] = [](Store& s, const Args& a) { s.tag_item(a.str(0, "item"), a.list(1, "tags"), a.str(2, "profile")); return Json(); }; + h["set_item_tags"] = [](Store& s, const Args& a) { s.set_item_tags(a.str(0, "item"), a.list(1, "tags"), a.str(2, "profile")); return Json(); }; + h["item_tags"] = [](Store& s, const Args& a) { return strs(s.item_tags(a.str(0, "item"), a.str(1, "profile"))); }; + h["tagged_items"] = [](Store& s, const Args& a) { auto r = s.tagged_items(a.list(0, "tags"), a.str(1, "profile")); return r ? strs(*r) : Json(); }; + h["add_edge"] = [](Store& s, const Args& a) { + try { return Json(s.add_edge(a.str(0, "src"), a.str(1, "dst"), a.str(2, "relation", "linked"), a.str(3, "profile"))); } + catch (const detz::store::ValueError& e) { Json o = Json::object(); o.set("ValueError", e.what()); return o; } + }; + h["edges_for"] = [](Store& s, const Args& a) { return arr(s.edges_for(a.str(0, "node"), a.str(1, "profile"))); }; + h["remove_edges"] = [](Store& s, const Args& a) { return Json(static_cast(s.remove_edges(a.str(0, "node"), a.str(1, "profile")))); }; + h["traverse"] = [](Store& s, const Args& a) { return arr(s.traverse(a.str(0, "start"), static_cast(a.num(1, "depth", 2)), static_cast(a.num(2, "limit", 50)), a.str(3, "profile"))); }; + h["add_chunks"] = [](Store& s, const Args& a) { s.add_chunks(a.str(0, "parent_kind"), a.str(1, "parent_id"), a.list(2, "texts"), a.str(3, "profile")); return Json(); }; + h["get_chunks_by_ids"] = [](Store& s, const Args& a) { + std::vector ids; + const Json* v = a.get(0, "ids"); + if (v && v->is_array()) for (auto& x : v->items()) ids.push_back(x.as_int()); + Json o = Json::object(); + for (auto& [id, d] : s.get_chunks_by_ids(ids, a.str(1, "profile"))) o.set(std::to_string(id), d); + return o; + }; + h["audit_event"] = [](Store& s, const Args& a) { + return Json(static_cast(s.audit_event(a.str(0, "actor"), a.str(1, "action"), a.str(2, "path"), a.str(3, "message"), a.opt_str(4, "old"), a.opt_str(5, "new"), a.list(6, "tags"), + a.is_null(7, "old_size") ? -1 : a.num(7, "old_size", -1), a.is_null(8, "new_size") ? -1 : a.num(8, "new_size", -1), a.str(9, "profile")))); + }; + h["audit_history"] = [](Store& s, const Args& a) { return arr(s.audit_history(a.str(0, "path"), a.str(1, "tag"), static_cast(a.num(2, "limit", 20)), a.str(3, "profile"))); }; + h["audit_get"] = [](Store& s, const Args& a) { return opt(s.audit_get(a.num(0, "row_id", 0), a.str(1, "profile"))); }; + h["audit_search"] = [](Store& s, const Args& a) { return arr(s.audit_search(a.str(0, "query"), a.str(1, "profile"), static_cast(a.num(2, "limit", 20)))); }; + h["upsert_file_record"] = [](Store& s, const Args& a) { return Json(s.upsert_file_record(a.str(0, "path"), a.str(1, "content"), a.is_null(2, "true_size") ? -1 : a.num(2, "true_size", -1), a.str(3, "profile"))); }; + h["file_record_id"] = [](Store& s, const Args& a) { return opt_s(s.file_record_id(a.str(0, "path"), a.str(1, "profile"))); }; + h["delete_file_record"] = [](Store& s, const Args& a) { return Json(s.delete_file_record(a.str(0, "path"), a.str(1, "profile"))); }; + h["log_event"] = [](Store& s, const Args& a) { s.log_event(a.str(0, "profile"), a.str(1, "role"), a.str(2, "kind"), a.str(3, "text"), a.list(4, "tags")); return Json(); }; + h["search_events"] = [](Store& s, const Args& a) { return events_json(s.search_events(a.str(0, "profile"), a.str(1, "query"), static_cast(a.num(2, "limit", 8)), a.list(3, "tags"))); }; + h["events_like"] = [](Store& s, const Args& a) { return events_json(s.events_like(a.str(0, "profile"), a.str(1, "query"), static_cast(a.num(2, "limit", 8)), a.list(3, "tags"))); }; + h["history_add"] = [](Store& s, const Args& a) { return Json(s.history_add(a.str(0, "profile"), a.str(1, "bot"), a.str(2, "text"))); }; + h["history_load"] = [](Store& s, const Args& a) { return strs(s.history_load(a.str(0, "profile"), static_cast(a.num(1, "limit", 1000)))); }; + h["history_last"] = [](Store& s, const Args& a) { return opt_s(s.history_last(a.str(0, "profile"))); }; + h["add_schedule"] = [](Store& s, const Args& a) { + std::optional> tags; + if (!a.is_null(6, "tags")) tags = a.list(6, "tags"); + return Json(static_cast(s.add_schedule(a.str(0, "name"), a.str(1, "prompt"), a.str(2, "profile"), a.num(3, "every_sec", 0), a.str(4, "next_run"), a.num(5, "timeout", 0), tags))); + }; + h["remove_schedule"] = [](Store& s, const Args& a) { return Json(s.remove_schedule(a.num(0, "row_id", 0), a.str(1, "profile"))); }; + h["list_schedules"] = [](Store& s, const Args& a) { return arr(s.list_schedules(a.str(0, "profile"))); }; + h["fts_search"] = [](Store& s, const Args& a) { + Json out = Json::array(); + for (auto& hit : s.fts_search(a.str(0, "query"), a.list(1, "kinds"), a.str(2, "profile"), static_cast(a.num(3, "limit", 10)))) { + Json o = Json::object(); + if (hit.item_numeric) o.set("item", static_cast(std::stoll(hit.item))); else o.set("item", hit.item); o.set("kind", hit.kind); o.set("sub", hit.sub); o.set("title", hit.title); o.set("snippet", hit.snippet); o.set("rank", hit.rank); + out.push(o); + } + return out; + }; + h["save_system"] = [](Store& s, const Args& a) { s.save_system(a.str(0, "name"), a.str(1, "text")); return Json(); }; + h["load_system"] = [](Store& s, const Args& a) { return opt_s(s.load_system(a.str(0, "name"))); }; + h["save_session"] = [](Store& s, const Args& a) { + std::vector msgs; + const Json* v = a.get(1, "messages"); + if (v && v->is_array()) msgs = v->items(); + s.save_session(a.str(0, "name"), msgs, a.str(2, "bot", "main"), a.str(3, "ctx")); + return Json(); + }; + h["load_session"] = [](Store& s, const Args& a) { return arr(s.load_session(a.str(0, "name"), a.str(1, "bot", "main"), a.str(2, "ctx"))); }; + h["save_turn_state"] = [](Store& s, const Args& a) { s.save_turn_state(a.str(0, "profile"), a.str(1, "bot"), a.json(2, "state"), a.str(3, "ctx")); return Json(); }; + h["load_turn_state"] = [](Store& s, const Args& a) { return s.load_turn_state(a.str(0, "profile"), a.str(1, "bot", "main"), a.str(2, "ctx")); }; + h["save_bots"] = [](Store& s, const Args& a) { s.save_bots(a.str(0, "profile"), a.json(1, "bots")); return Json(); }; + h["load_bots"] = [](Store& s, const Args& a) { return s.load_bots(a.str(0, "profile")); }; + h["list_bots"] = [](Store& s, const Args& a) { return arr(s.list_bots(a.str(0, "profile"))); }; + h["resolve_bot"] = [](Store& s, const Args& a) { return opt_s(s.resolve_bot(a.str(0, "profile"), a.str(1, "mention"))); }; + h["save_bot_system"] = [](Store& s, const Args& a) { s.save_bot_system(a.str(0, "profile"), a.str(1, "bot"), a.str(2, "text")); return Json(); }; + h["load_bot_system"] = [](Store& s, const Args& a) { return opt_s(s.load_bot_system(a.str(0, "profile"), a.str(1, "bot"))); }; + h["list_profiles"] = [](Store& s, const Args&) { return strs(s.list_profiles()); }; + h["get_or_create_settings"] = [](Store& s, const Args& a) { + try { + std::optional> keys; + if (!a.is_null(0, "keys")) { + const Json* v = a.get(0, "keys"); + keys = v->is_string() ? std::vector{v->as_string()} : a.list(0, "keys"); + } + return rows_json(s.get_or_create_settings(keys, a.str(1, "scope", "global"))); + } catch (const detz::store::ValueError& e) { Json o = Json::object(); o.set("ValueError", e.what()); return o; } + }; + h["get_all_settings"] = [](Store& s, const Args& a) { + try { return rows_json(s.get_all_settings(a.str(0, "scope", "global"))); } + catch (const detz::store::ValueError& e) { Json o = Json::object(); o.set("ValueError", e.what()); return o; } + }; + h["upsert_settings"] = [](Store& s, const Args& a) { + try { + std::map items; + Json v = a.json(0, "items"); + if (v.is_object()) for (auto& m : v.members()) items[m.first] = m.second; + return rows_json(s.upsert_settings(items, a.str(1, "scope", "global"))); + } catch (const detz::store::ValueError& e) { Json o = Json::object(); o.set("ValueError", e.what()); return o; } + }; + h["mcp_save_server"] = [](Store& s, const Args& a) { + try { s.mcp_save_server(a.json(0, "entry")); return Json(); } + catch (const detz::store::ValueError& e) { Json o = Json::object(); o.set("ValueError", e.what()); return o; } + }; + h["mcp_get_server"] = [](Store& s, const Args& a) { return opt(s.mcp_get_server(a.str(0, "name"))); }; + h["mcp_list_servers"] = [](Store& s, const Args&) { return arr(s.mcp_list_servers()); }; + h["mcp_remove_server"] = [](Store& s, const Args& a) { s.mcp_remove_server(a.str(0, "name")); return Json(); }; + h["mcp_set_enabled"] = [](Store& s, const Args& a) { const Json* v = a.get(2, "enabled"); s.mcp_set_enabled(a.str(0, "server"), a.str(1, "profile"), v && v->is_bool() && v->as_bool()); return Json(); }; + h["mcp_server_enabled"] = [](Store& s, const Args& a) { return Json(s.mcp_server_enabled(a.str(0, "server"), a.str(1, "profile"))); }; + h["mcp_enabled_servers"] = [](Store& s, const Args& a) { return arr(s.mcp_enabled_servers(a.str(0, "profile"))); }; + h["mcp_store_tools"] = [](Store& s, const Args& a) { + std::vector tools; + const Json* v = a.get(1, "tools"); + if (v && v->is_array()) tools = v->items(); + s.mcp_store_tools(a.str(0, "server"), tools); + return Json(); + }; + h["mcp_cached_tools"] = [](Store& s, const Args& a) { return arr(s.mcp_cached_tools(a.str(0, "server"))); }; + h["mcp_tool_pins"] = [](Store& s, const Args& a) { Json o = Json::object(); for (auto& [k, v] : s.mcp_tool_pins(a.str(0, "server"))) o.set(k, v); return o; }; + h["load_dir_defaults"] = [](Store& s, const Args& a) { Json o = Json::object(); for (auto& [k, v] : s.load_dir_defaults(a.str(0, "directory"))) o.set(k, v); return o; }; + h["save_dir_defaults"] = [](Store& s, const Args& a) { s.save_dir_defaults(a.str(0, "directory"), a.str(1, "provider"), a.str(2, "model"), a.str(3, "bot"), a.list(4, "clear")); return Json(); }; + h["spill_or_truncate"] = [](Store& s, const Args& a) { return Json(detz::store::Spill::spill_or_truncate(&s, a.str(0, "kind"), a.str(1, "title"), a.str(2, "text"), a.list(3, "tags"), a.str(4, "profile"))); }; + return h; +} + + +// Replays scripted replies for PlexerFlow and logs every screen it was asked. +class FakeController : public detz::plexer::PlexerController { + public: + explicit FakeController(Json replies) : replies_(std::move(replies)) {} + Json send_keys(const std::string&, bool) override { return Json(); } + Json read_pane(int) override { return Json(""); } + Json status() override { return Json(); } + Json ask(const Json& screen, const Json&) override { + asked.push(screen); + Json r = replies_.items()[next_++]; + return r; + } + Json asked = Json::array(); + private: + Json replies_; + size_t next_{0}; +}; + +Json err_json(const char* what) { Json o = Json::object(); o.set("ValueError", what); return o; } + +// Captures everything written to fd 1 while alive (the REPL prints with printf). +class StdoutCapture { + public: + StdoutCapture() { + std::fflush(stdout); + saved_ = ::dup(1); + file_ = std::tmpfile(); + ::dup2(::fileno(file_), 1); + } + std::string finish() { + std::fflush(stdout); + ::dup2(saved_, 1); + ::close(saved_); + std::string out; + std::rewind(file_); + char buf[4096]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, file_)) > 0) out.append(buf, n); + std::fclose(file_); + return out; + } + private: + int saved_; + std::FILE* file_; +}; + +class NullHost : public detz::repl::ReplHost { + public: + std::string run_turn(const std::string&) override { return ""; } +}; + +void add_pure_handlers(std::map& h) { + using namespace detz; + h["statusbar"] = [](Store&, const Args& a) { return Json(repl::ReplStyle::statusbar(a.str(0, "version"), a.str(1, "profile"), a.str(2, "directory"), a.json(3, "yolo").as_bool(), static_cast(a.num(4, "agents", 0)), static_cast(a.num(5, "width", 0)))); }; + h["strip_ansi"] = [](Store&, const Args& a) { return Json(repl::ReplStyle::strip_ansi(a.str(0, "text"))); }; + h["truncate_ansi"] = [](Store&, const Args& a) { return Json(repl::ReplStyle::truncate_ansi(a.str(0, "text"), static_cast(a.num(1, "width", 0)))); }; + h["glow_prompt"] = [](Store&, const Args&) { return Json(repl::ReplStyle::glow_prompt()); }; + h["glow_title"] = [](Store&, const Args& a) { return Json(repl::ReplStyle::glow_title(a.str(0, "version"), a.str(1, "profile"))); }; + h["sleep"] =[](Store&, const Args& a) { std::this_thread::sleep_for(std::chrono::duration(a.json(0, "seconds").as_double())); return Json(); }; + h["native_ctl"] = [](Store&, const Args& a) { + plexer::NativePlexerController ctl(a.str(0, "sock"), a.str(1, "pane")); + std::string method = a.str(2, "method"); + Json args = a.json(3, "args"); + try { + if (method == "type_text") { ctl.type_text(args.items()[0].as_string(), 0.0, args.items()[1].as_bool()); return Json(); } + if (method == "call") return ctl.call(args.items()[0]); + if (method == "send_keys") return ctl.send_keys(args.items()[0].as_string(), args.items()[1].as_bool()); + if (method == "read_pane") return ctl.read_pane(static_cast(args.items()[0].as_int())); + if (method == "status") return ctl.status(); + return ctl.ask(args.items()[0], args.items()[1]); + } catch (const plexer::PlexerError& e) { + Json o = Json::object(); + o.set("PlexerError", e.what()); + return o; + } + }; + h["tmux_ctl"] = [](Store&, const Args& a) { + plexer::TmuxPlexerController ctl(a.str(0, "pane")); + std::string method = a.str(1, "method"); + Json args = a.json(2, "args"); + try { + if (method == "type_text") { ctl.type_text(args.items()[0].as_string(), 0.0, args.items()[1].as_bool()); return Json(); } + if (method == "send_keys") return ctl.send_keys(args.items()[0].as_string(), args.items()[1].as_bool()); + if (method == "read_pane") return ctl.read_pane(static_cast(args.items()[0].as_int())); + if (method == "status") return ctl.status(); + return ctl.ask(args.items()[0], args.items()[1]); + } catch (const plexer::PlexerError& e) { + Json o = Json::object(); + o.set("PlexerError", e.what()); + return o; + } + }; + h["log_query"] = [](Store& s, const Args& a) { + Json f = a.json(0, "filter"); + store::LogFilter lf; + lf.level = f.get_string("level"); lf.component = f.get_string("component"); lf.event = f.get_string("event"); + lf.profile = f.get_string("profile"); lf.bot = f.get_string("bot"); lf.chat = f.get_string("chat"); lf.run = f.get_string("run"); + if (const Json* l = f.find("limit")) lf.limit = static_cast(l->as_int()); + return arr(s.log_query(lf)); + }; + h["debug_log"] = [](Store& s, const Args& a) { + auto& log = store::DebugLog::Global(); + log.configure(a.str(0, "sink"), a.str(1, "fmt"), s.config().home, &s); + for (auto& ev : a.json(2, "events").items()) { + const auto& it = ev.items(); + Json kw = it[4]; + log.event(it[0].as_string(), it[1].as_string(), it[2].as_string(), it[3].as_string(), kw.get_string("profile"), kw.get_string("bot"), kw.get_string("chat"), + kw.has("meta") ? *kw.find("meta") : Json()); + } + std::string run = log.run; + log.close(); + log.reset(); + std::ifstream in(s.config().home + "/debug.log", std::ios::binary); + std::stringstream ss; + ss << in.rdbuf(); + return arr({Json(run), Json(ss.str())}); + }; + h["repl_script"] = [](Store& s, const Args& a) { + Config cfg = s.config(); + cfg.profile = "default"; + const char* tai_session = std::getenv("TAI_SESSION"); + session::Conversation conv(s, cfg, true, false, false, tai_session ? tai_session : ""); + NullHost host; + repl::Repl::Options opts; + repl::Repl app(conv, host, opts); + Json outputs = Json::array(); + for (auto& text : a.json(0, "script").items()) { + StdoutCapture cap; + bool cont = app.handle_command(text.as_string()); + outputs.push(arr({Json(cap.finish()), Json(cont)})); + } + Json msgs = Json::array(); + for (size_t k = 1; k < conv.messages().size(); k++) msgs.push(conv.messages()[k]); + return arr({outputs, arr({Json(conv.profile()), Json(conv.bot()), msgs})}); + }; + h["prompt"] = [](Store&, const Args& a) { + std::string n = a.str(0, "name"); + if (n == "DEFAULT_SYSTEM") return Json(Prompts::DefaultSystem()); + if (n == "MERGE_SYSTEM") return Json(Prompts::MergeSystem()); + if (n == "COMPACT_SYSTEM") return Json(Prompts::CompactSystem()); + if (n == "AUTONOMOUS_NOTE") return Json(Prompts::AutonomousNote()); + if (n == "HELP_TEXT") return Json(Prompts::HelpText()); + if (n == "LEGACY_PASSWORD_NOTE") return Json(Prompts::LegacyPasswordNote()); + return Json(Prompts::VaultMemoryNote()); + }; + h["capability_block"] = [](Store&, const Args& a) { return Json(Prompts::CapabilityBlock(a.json(0, "yolo").is_bool() && a.json(0, "yolo").as_bool(), a.json(1, "auto").is_bool() && a.json(1, "auto").as_bool())); }; + h["orientation_block"] = [](Store&, const Args& a) { return Json(Prompts::OrientationBlock(a.str(0, "directory"), a.json(1, "sandbox").is_bool() && a.json(1, "sandbox").as_bool())); }; + h["settings_catalog"] = [](Store&, const Args&) { + Json o = Json::object(); + for (auto& [k, s] : SettingsCatalog()) { + Json e = Json::object(); + e.set("key", s.key); e.set("env", s.env); e.set("type", s.type); e.set("default", value_json(s.def)); e.set("desc", s.help); + e.set("required", s.required); e.set("nullable", s.nullable); e.set("encrypted", s.encrypted); + e.set("options", s.hasOptions ? strs(s.options) : Json()); + e.set("preferred", Json()); + e.set("true_words", s.trueWords.empty() ? Json() : strs(s.trueWords)); + e.set("false_words", s.falseWords.empty() ? Json() : strs(s.falseWords)); + o.set(k, e); + } + return o; + }; + h["json_dumps"] = [](Store&, const Args& a) { Json v = a.json(0, "value"); return arr({Json(v.dumps()), Json(v.dumps(true))}); }; + h["json_roundtrip"] = [](Store&, const Args& a) { + try { return Json(Json::loads(a.str(0, "text")).dumps(true)); } + catch (const store::JsonError&) { return err_json("x"); } + }; + h["float_repr"] = [](Store&, const Args& a) { return Json(Json::float_repr(a.json(0, "v").as_double())); }; + h["normalize_tag"] = [](Store&, const Args& a) { return Json(store::TagRules::normalize_tag(a.str(0, "raw"))); }; + h["normalize_tags"] = [](Store&, const Args& a) { return strs(store::TagRules::normalize_tags(a.list(0, "items"))); }; + h["singular_noun"] = [](Store&, const Args& a) { return Json(store::TagRules::singular_noun(a.str(0, "word"))); }; + h["tag_variants"] = [](Store&, const Args& a) { return strs(store::TagRules::tag_variants(a.str(0, "tag"))); }; + h["parse_node_id"] = [](Store&, const Args& a) { auto r = store::TagRules::parse_node_id(a.str(0, "raw")); return r ? arr({Json(r->first), Json(r->second)}) : Json(); }; + h["fts_match_variants"] = [](Store&, const Args& a) { auto t = store::Fts::escape(a.str(0, "query")); return t ? strs(store::Fts::match_variants(*t)) : Json(); }; + h["fts_rank"] = [](Store&, const Args& a) { return Json(store::Fts::rank(a.str(0, "table"))); }; + h["profile_re"] = [](Store&, const Args& a) { return Json(store::Patterns::profile(a.str(0, "s"))); }; + h["bot_re"] = [](Store&, const Args& a) { return Json(store::Patterns::bot(a.str(0, "s"))); }; + h["secret_re"] = [](Store&, const Args& a) { return Json(store::Patterns::secret(a.str(0, "s"))); }; + h["mcp_server_re"] = [](Store&, const Args& a) { return Json(store::Patterns::mcp_server(a.str(0, "s"))); }; + h["normalize_setting_value"] = [](Store&, const Args& a) { + try { + auto r = SettingsRules::Normalize(a.str(0, "key"), SettingsCatalog().at(a.str(0, "key")), a.json(1, "value")); + return opt_s(r); + } catch (const store::ValueError& e) { return err_json(e.what()); } + }; + h["parse_setting_env"] = [](Store&, const Args& a) { + try { return value_json(SettingsRules::ParseEnv(SettingsCatalog().at(a.str(0, "key")), a.str(1, "raw"))); } + catch (const store::ValueError& e) { return err_json(e.what()); } + }; + h["parse_setting_stored"] = [](Store&, const Args& a) { + try { return value_json(SettingsRules::ParseStored(SettingsCatalog().at(a.str(0, "key")), a.str(1, "stored"))); } + catch (const store::ValueError&) { return err_json("could not convert"); } + }; + h["parse_backends"] = [](Store&, const Args& a) { + Json out = Json::array(); + for (auto& b : ParseBackends(a.str(0, "spec"))) { + Json o = Json::object(); + o.set("label", b.label); o.set("base", b.base); o.set("key", b.key); o.set("model", b.model); o.set("profile", b.profile); + out.push(o); + } + return out; + }; + h["split_models"] = [](Store&, const Args& a) { return strs(SplitModels(a.str(0, "spec"))); }; + h["chunk_lines"] = [](Store&, const Args& a) { return strs(store::Spill::chunk_lines(a.str(0, "text"), static_cast(a.num(1, "max_lines", 60)), static_cast(a.num(2, "overlap_lines", 3)), static_cast(a.num(3, "max_chars", 4000)))); }; + h["truncate"] = [](Store&, const Args& a) { return Json(store::Spill::truncate(a.str(0, "text"), static_cast(a.num(1, "head", 1500)), static_cast(a.num(2, "tail", 500)))); }; + h["shell_current"] = [](Store&, const Args& a) { + std::map env; + Json e = a.json(0, "env"); + if (e.is_object()) for (auto& m : e.members()) env[m.first] = m.second.as_string(); + auto c = session::ShellContext::current(&env, a.num(1, "pid", 0), a.str(2, "proc_root"), a.json(3, "now").as_double(), a.num(4, "clk_tck", 100)); + Json chain = Json::array(); + for (auto& x : c.chain) { + Json o = Json::object(); + o.set("pid", static_cast(x.pid)); o.set("ppid", static_cast(x.ppid)); o.set("comm", x.comm); o.set("argv0", x.argv0); + o.set("name", x.name); o.set("start", x.start); o.set("start_iso", x.start_iso); o.set("sid", x.sid); + chain.push(o); + } + return arr({Json(c.sid), Json(c.shell), Json(static_cast(c.pid)), Json(c.started), c.parent ? Json(*c.parent) : Json(), chain}); + }; + h["sanitize_sid"] = [](Store&, const Args& a) { return Json(session::ShellContext::sanitize(a.str(0, "value"))); }; + h["validate_flow_spec"] = [](Store&, const Args& a) { + auto r = plexer::FlowValidator::spec(a.json(0, "spec")); + return arr({r.second.empty() ? r.first : Json(), Json(r.second)}); + }; + h["run_flow"] = [](Store&, const Args& a) { + auto v = plexer::FlowValidator::spec(a.json(0, "spec")); + if (!v.second.empty()) return arr({Json(), Json(v.second)}); + FakeController fake(a.json(1, "replies")); + plexer::PlexerFlow flow(v.first); + Json outcome = flow.run(fake); + return arr({outcome, fake.asked}); + }; +} + +Json substitute(const Json& v, const std::vector& results) { + if (v.is_string() && v.as_string().size() > 2 && v.as_string().rfind("$R", 0) == 0) { + size_t idx = std::stoul(v.as_string().substr(2)); + if (idx < results.size()) return results[idx]; + } + if (v.is_array()) { Json a = Json::array(); for (auto& x : v.items()) a.push(substitute(x, results)); return a; } + if (v.is_object()) { Json o = Json::object(); for (auto& m : v.members()) o.set(m.first, substitute(m.second, results)); return o; } + return v; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { std::fprintf(stderr, "usage: foundation_probe scenario.json\n"); return 2; } + std::ifstream in(argv[1], std::ios::binary); + std::stringstream ss; + ss << in.rdbuf(); + Json scenario = Json::loads(ss.str()); + Config config = Config::Build("default", false, false); + NullSeal seal; + Store store(config, seal); + auto table = handlers(); + add_pure_handlers(table); + std::vector results; + for (auto& step : scenario.items()) { + std::string op = step.items()[0].as_string(); + Json pos = step.size() > 1 ? substitute(step.items()[1], results) : Json::array(); + Json kw = step.size() > 2 ? substitute(step.items()[2], results) : Json::object(); + auto it = table.find(op); + if (it == table.end()) { std::fprintf(stderr, "unknown op %s\n", op.c_str()); return 3; } + Args args{&pos, &kw}; + results.push_back(it->second(store, args)); + } + store.close(); + Json out = Json::array(); + for (auto& r : results) out.push(r); + std::cout << out.dumps(true) << std::endl; + return 0; +} diff --git a/taicpp/tests/foundation_test.cpp b/taicpp/tests/foundation_test.cpp new file mode 100644 index 0000000..d1439cb --- /dev/null +++ b/taicpp/tests/foundation_test.cpp @@ -0,0 +1,246 @@ +// Unit tests for the foundation area that need no python oracle: hash +// vectors, UTF-8 helpers, the seal-enabled store paths (with a stand-in +// SealPort), boot settings, directory defaults, plexer detection, line input. +// The behaviour parity with tai.py lives in tests/foundation_parity.py. +#include +#include +#include +#include +#include +#include +#include +#include "config/boot_settings.h" +#include "config/config.h" +#include "plexer/plexer.h" +#include "repl/interrupt.h" +#include "repl/line_input.h" +#include "repl/repl.h" +#include "session/conversation.h" +#include "session/dir_defaults.h" +#include "session/md5.h" +#include "session/shell_context.h" +#include "store/clock.h" +#include "store/json.h" +#include "store/sha256.h" +#include "store/store.h" +#include "store/utf8.h" +#include "store/vault_files.h" +using namespace detz; +namespace fs = std::filesystem; + +static int g_checks = 0, g_fail = 0; +#define CHECK(cond) do { g_checks++; if (!(cond)) { g_fail++; std::printf("FAIL %s:%d %s\n", __FILE__, __LINE__, #cond); } } while (0) + +// A reversible stand-in for the vault crypto: enough to exercise every sealed code path of Store. +class FakeSeal : public store::SealPort { + public: + bool enabled() const override { return true; } + std::string lock(const std::string& t) const override { + if (is_sealed(t)) return t; + std::string out = kPrefix; + for (unsigned char c : t) { char b[4]; std::snprintf(b, sizeof b, "%02x", static_cast(c ^ 0x5a)); out += b; } + return out; + } + std::string unlock(const std::string& t) const override { + if (!is_sealed(t)) throw store::SealError("not sealed"); + std::string out; + for (size_t i = 5; i + 1 < t.size(); i += 2) out += static_cast(std::stoi(t.substr(i, 2), nullptr, 16) ^ 0x5a); + return out; + } +}; + +static Config temp_config(const std::string& profile = "default") { + fs::path home = fs::temp_directory_path() / ("detz-ft-" + store::Clock::uuid_hex().substr(0, 8)); + fs::create_directories(home / "profiles"); + Config c = Config::FromEnv(); + c.home = home.string(); c.profilesDir = (home / "profiles").string(); c.audioDir = (home / "audio").string(); + c.dbPath = (home / "memory.db").string(); c.historyPath = (home / "history").string(); c.profile = profile; + return c; +} + +static void test_hashes() { + CHECK(store::Sha256::hex("") == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + CHECK(store::Sha256::hex("abc") == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK(store::Sha256::hex(std::string(1000, 'a')) == "41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3"); + CHECK(session::Md5::hex("") == "d41d8cd98f00b204e9800998ecf8427e"); + CHECK(session::Md5::hex("abc") == "900150983cd24fb0d6963f7d28e17f72"); + CHECK(session::Md5::hex("1700000000.0").size() == 32); +} + +static void test_utf8_and_json() { + using store::Utf8; + CHECK(Utf8::length("h\xc3\xa9llo \xf0\x9f\x98\x80") == 7); + CHECK(Utf8::slice("a\xc3\xa9z", 1, 2) == "\xc3\xa9"); + CHECK(Utf8::suffix("ab\xc3\xa9", 2) == "b\xc3\xa9"); + CHECK(Utf8::strip("\xc2\xa0 x \xe2\x80\x83") == "x"); + auto sp = Utf8::split_first(" cmd the rest "); + CHECK(sp.first == "cmd" && sp.second == "the rest "); + CHECK(Utf8::sanitize("a\xff" "b") == "a\xef\xbf\xbd" "b"); + CHECK(store::Json::float_repr(0.1) == "0.1" && store::Json::float_repr(1e16) == "1e+16" && store::Json::float_repr(100.0) == "100.0"); + store::Json j = store::Json::loads("{\"b\": [1, 2.5, \"\\u00e9\"], \"a\": null}"); + CHECK(j.dumps() == "{\"b\": [1, 2.5, \"\\u00e9\"], \"a\": null}"); + CHECK(j.dumps(true) == "{\"a\": null, \"b\": [1, 2.5, \"\\u00e9\"]}"); + bool threw = false; + try { store::Json::loads("{\"a\":"); } catch (const store::JsonError&) { threw = true; } + CHECK(threw); + CHECK(std::regex_match(store::Clock::now_iso(), std::regex("\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d\\+00:00"))); + CHECK(std::regex_match(store::Clock::uuid_hex(), std::regex("[0-9a-f]{32}"))); +} + +static void test_sealed_store() { + Config cfg = temp_config(); + store::NullSeal plain; + { + store::Store st(cfg, plain); + st.log_event("default", "user", "say", "plain fox event"); + st.save_secret("k", "secret-value-1"); + st.save_session("default", {store::Json::loads("{\"role\": \"user\", \"content\": \"hi\"}")}, "main", "ctx"); + st.save_system("default", "sys text"); + st.history_add("default", "main", "typed line"); + st.close(); + } + FakeSeal seal; + { + store::Store st(cfg, seal); // upgrades the plaintext vault in place + auto raw = st.db().execute("SELECT text FROM events").rows; + CHECK(!raw.empty() && store::SealPort::is_sealed(raw[0][0].text())); + CHECK(store::SealPort::is_sealed(st.db().execute("SELECT value FROM secrets").rows[0][0].text())); + CHECK(st.load_secret("k") == std::optional("secret-value-1")); + CHECK(st.history_load("default") == std::vector{"typed line"}); + CHECK(st.load_system("default") == std::optional("sys text")); + auto msgs = st.load_session("default", "main", "ctx"); + CHECK(msgs.size() == 1 && msgs[0].get_string("content") == "hi"); + { // seal_files() only upgrades *.sys.md, *.session.json and *.bots.json, so a ctx-scoped file stays plain until it is saved again (as in tai.py) + std::ifstream f(st.session_path("default", "ctx")); + std::stringstream ss; ss << f.rdbuf(); + CHECK(!store::SealPort::is_sealed(ss.str())); + } + st.save_session("default", msgs, "main", "ctx"); + { + std::ifstream f(st.session_path("default", "ctx")); + std::stringstream ss; ss << f.rdbuf(); + CHECK(store::SealPort::is_sealed(ss.str())); + CHECK(st.load_session("default", "main", "ctx").size() == 1); + } + { + std::ifstream f(st.profile_path("default")); + std::stringstream ss; ss << f.rdbuf(); + CHECK(store::SealPort::is_sealed(ss.str())); // default.sys.md was upgraded in place + } + auto hits = st.search_events("default", "fox"); + CHECK(hits.size() == 1 && hits[0].text == "plain fox event"); // through the in-memory index + st.log_event("default", "user", "say", "second fox event"); + CHECK(st.search_events("default", "fox").size() == 2); + CHECK(st.fts_search("fox", {"event"}).size() == 2); + st.close(); + } + { + bool refused = false; + try { store::Store st(cfg, plain); } catch (const store::SealError& e) { refused = std::string(e.what()).find("sealed") != std::string::npos; } + CHECK(refused); // a sealed vault without the seal is refused loudly + } +} + +static void test_boot_settings_and_dir_defaults() { + ::unsetenv("TAI_THEME"); ::unsetenv("TAI_VOICE"); ::unsetenv("TAI_MUSE_BIN"); ::unsetenv("MUSE_BIN"); + Config cfg = temp_config(); + store::NullSeal seal; + store::Store st(cfg, seal); + st.upsert_settings({{"voice", store::Json("vault-voice")}, {"theme", store::Json("nord")}, {"muse_bin", store::Json("/opt/muse")}}); + auto applied = BootSettings::Boot(cfg, st); + CHECK(cfg.voice == "vault-voice"); + bool has_voice = false, has_theme = false; + for (auto& a : applied) { has_voice |= a == "voice"; has_theme |= a == "theme"; } + CHECK(has_voice && has_theme); + CHECK(std::string(std::getenv("TAI_THEME")) == "nord" && std::string(std::getenv("MUSE_BIN")) == "/opt/muse"); + // env wins: a second boot with the env already exported changes nothing + ::setenv("TAI_VOICE", "env-voice", 1); + Config again = cfg; + again.voice = "x"; + auto second = again.ApplyStoreSettings(st); + bool voice_again = false; + for (auto& a : second) voice_again |= a == "voice"; + CHECK(!voice_again && again.voice == "x"); + ::unsetenv("TAI_VOICE"); ::unsetenv("TAI_THEME"); ::unsetenv("MUSE_BIN"); + + session::Conversation conv(st, cfg, true, false, false, "ctxdd"); + st.save_bots("default", store::Json::loads("{\"helper\": {\"nicknames\": []}}")); + st.save_dir_defaults(session::ShellContext::dir_key(), "prov", "mod", "helper"); + std::string pinned_provider, pinned_model; + session::DirDefaults::Hooks hooks; + hooks.offered_providers = [] { return std::set{"prov"}; }; + hooks.pin_provider = [&](const std::string& p) { pinned_provider = p; }; + hooks.pin_model = [&](const std::string& m) { pinned_model = m; }; + std::string line = session::DirDefaults::apply(conv, hooks); + CHECK(pinned_provider == "prov" && pinned_model == "mod" && conv.bot() == "helper"); + CHECK(line == "dir defaults for " + session::ShellContext::dir_key() + ": provider=prov model=mod bot=helper"); +} + +static void test_conversation_turns() { + Config cfg = temp_config(); + store::NullSeal seal; + store::Store st(cfg, seal); + session::Conversation conv(st, cfg, true, false, false, "ctxturn"); + CHECK(conv.messages().size() == 1 && conv.messages()[0].get_string("role") == "system"); + conv.begin_turn("do the thing"); + conv.add_message("user", "do the thing"); + conv.checkpoint(); + auto open = st.load_turn_state("default", "main", "ctxturn"); + CHECK(open.find("open") && open.find("open")->as_bool() && open.get_string("goal") == "do the thing"); + conv.add_message("assistant", "done"); + conv.close_turn(); + CHECK(!st.load_turn_state("default", "main", "ctxturn").find("open")->as_bool()); + session::Conversation fresh(st, cfg, true, false, false, "ctxturn"); + CHECK(fresh.messages().size() == 3); // system + the two restored messages + session::Conversation other(st, cfg, true, false, false, "ctxother"); + CHECK(other.messages().size() == 1); + session::Conversation volatile_run(st, cfg, false, false, false, "ctxturn"); + CHECK(volatile_run.messages().size() == 1); // persist=false never restores + CHECK(conv.switch_bot("nobody", true).rfind("unknown bot", 0) == 0); +} + +static void test_plexer_detection() { + ::unsetenv("TAI_PLEXER_SOCKET"); ::unsetenv("TMUX_PANE"); ::unsetenv("TAI_PLEXER_PANE"); + CHECK(plexer::PlexerControllerFactory::detect() == nullptr); + ::setenv("TMUX_PANE", "%4", 1); + auto t = plexer::PlexerControllerFactory::detect(); + CHECK(t && t->kind() == "tmux"); + ::setenv("TAI_PLEXER_SOCKET", "/tmp/nonexistent.sock", 1); + ::setenv("TAI_PLEXER_PANE", "9", 1); + auto n = plexer::PlexerControllerFactory::detect(); + CHECK(n && n->kind() == "native" && dynamic_cast(n.get())->pane() == "9"); + ::unsetenv("TAI_PLEXER_SOCKET"); ::unsetenv("TMUX_PANE"); ::unsetenv("TAI_PLEXER_PANE"); + CHECK(plexer::Plexer::shlex_quote("a b") == "'a b'" && plexer::Plexer::shlex_quote("") == "''" && plexer::Plexer::shlex_quote("it's") == "'it'\"'\"'s'"); +} + +static void test_line_input() { + int fds[2]; + CHECK(::pipe(fds) == 0); + const char data[] = "first\r\nsecond\nlast-no-newline"; + CHECK(::write(fds[1], data, sizeof data - 1) > 0); + ::close(fds[1]); + repl::LineInput in(fds[0]); + std::string line; + CHECK(in.read("", line) == repl::LineInput::Status::Line && line == "first"); + CHECK(in.read("", line) == repl::LineInput::Status::Line && line == "second"); + CHECK(in.read("", line) == repl::LineInput::Status::Line && line == "last-no-newline"); + CHECK(in.read("", line) == repl::LineInput::Status::Eof); + repl::Interrupt::raise(); + int fds2[2]; + CHECK(::pipe(fds2) == 0); + repl::LineInput quiet(fds2[0]); + CHECK(quiet.read("", line) == repl::LineInput::Status::Interrupted && !repl::Interrupt::requested()); + ::close(fds[0]); ::close(fds2[0]); ::close(fds2[1]); +} + +int main() { + test_hashes(); + test_utf8_and_json(); + test_sealed_store(); + test_boot_settings_and_dir_defaults(); + test_conversation_turns(); + test_plexer_detection(); + test_line_input(); + std::printf("%d checks, %d failures\n", g_checks, g_fail); + return g_fail ? 1 : 0; +} diff --git a/taicpp/tests/repl_persist.py b/taicpp/tests/repl_persist.py new file mode 100644 index 0000000..35b77d7 --- /dev/null +++ b/taicpp/tests/repl_persist.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Conversation history survives a restart in the same pane. + +Starts the real REPL binary (build/repl_probe: Repl + Conversation + Store, a +scripted host in place of the model) repeatedly, each time as a child of a +long-lived shell that stands for one terminal pane (TMUX_PANE and +TAI_PLEXER_PANE set, a throwaway TAI_HOME, no TAI_SESSION). Every assertion +reads vault state (profiles/*.session.*.json, *.turn.*.json, memory.db), never +the program's output. + +tai.py keys a conversation by the nearest shell ancestor (ShellSession.current), +and a pane is exactly one such shell, so: + - same pane, restart -> same session file, history resumes + - different pane -> its own session file, history independent + - TAI_SESSION= -> panes share one pinned context + - interrupted turn -> --continue picks it up + +Usage: python3 tests/repl_persist.py [--probe PATH] +""" +import argparse +import glob +import json +import os +import shlex +import sqlite3 +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) + + +class Pane: + """One long-lived shell = one terminal pane.""" + + def __init__(self, home, probe, number, extra_env=None): + self.probe = probe + env = {key: value for key, value in os.environ.items() if not key.startswith(("TAI_", "TMUX"))} + env.update({"TAI_HOME": home, "TAI_PASSPHRASE": "", "NO_COLOR": "1", "HOME": home, + "TMUX_PANE": "%%%d" % number, "TAI_PLEXER_PANE": str(number)}) + env.update(extra_env or {}) + self.shell = subprocess.Popen(["bash", "--norc", "--noprofile"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=env, universal_newlines=True) + + def run(self, lines, *flags): + """Run the probe in this pane with `lines` on stdin; returns its exit code.""" + feed = "".join(line + "\n" for line in lines) + command = "printf %%s %s | %s %s; echo __RC=$?\n" % (shlex.quote(feed), shlex.quote(self.probe), " ".join(shlex.quote(f) for f in flags)) + self.shell.stdin.write(command) + self.shell.stdin.flush() + for raw in self.shell.stdout: + if "__RC=" in raw: # the prompt has no newline, so the marker can follow it on the same line + return int(raw.strip().split("__RC=")[1]) + raise RuntimeError("pane shell died") + + def close(self): + try: + self.shell.stdin.close() + self.shell.wait(timeout=10) + except Exception: + self.shell.kill() + + +def session_files(home, profile="default"): + return sorted(glob.glob(os.path.join(home, "profiles", profile + ".session.*.json"))) + + +def read_json(path): + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def contents(path): + return [(item["role"], item["content"]) for item in read_json(path)] + + +def history(home): + db = sqlite3.connect(os.path.join(home, "memory.db")) + try: + return [row[0] for row in db.execute("SELECT text FROM input_history WHERE profile = 'default' ORDER BY id").fetchall()] + finally: + db.close() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--probe", default=os.path.join(os.path.dirname(HERE), "build", "repl_probe")) + opts = parser.parse_args() + home = tempfile.mkdtemp(prefix="detz-repl-") + checks = 0 + failures = [] + + def check(label, ok, detail=""): + nonlocal checks + checks += 1 + if not ok: + failures.append("%s %s" % (label, detail)) + print("FAIL %s %s" % (label, detail)) + + pane_a = Pane(home, opts.probe, 11) + pane_b = Pane(home, opts.probe, 12) + try: + # first start in pane A + check("A1 exit", pane_a.run(["hello one", "hello two", "/quit"]) == 0) + files = session_files(home) + check("A1 one session file", len(files) == 1, str(files)) + sid_file = files[0] + check("A1 file is keyed by a shell id", ".session.p" not in sid_file and "-p" in os.path.basename(sid_file), sid_file) + expect = [("user", "hello one"), ("assistant", "echo: hello one"), ("user", "hello two"), ("assistant", "echo: hello two")] + check("A1 messages saved", contents(sid_file) == expect, str(contents(sid_file))) + check("A1 system message not stored", all(role != "system" for role, _ in contents(sid_file))) + + # quit and restart in the same pane: the same file resumes + check("A2 exit", pane_a.run(["after restart", "/quit"]) == 0) + files = session_files(home) + check("A2 still one session file", files == [sid_file], str(files)) + check("A2 history resumed and extended", contents(sid_file) == expect + [("user", "after restart"), ("assistant", "echo: after restart")], str(contents(sid_file))) + + # another pane: independent conversation + check("B1 exit", pane_b.run(["other pane"]) == 0) + files = session_files(home) + check("B1 second session file", len(files) == 2, str(files)) + other = [path for path in files if path != sid_file][0] + check("B1 own history only", contents(other) == [("user", "other pane"), ("assistant", "echo: other pane")], str(contents(other))) + check("B1 pane A untouched", len(contents(sid_file)) == 6) + + # pane A again, one-shot path also resumes and saves + check("A3 oneshot exit", pane_a.run([], "--oneshot", "one shot q") == 0) + check("A3 oneshot appended", contents(sid_file)[-2:] == [("user", "one shot q"), ("assistant", "echo: one shot q")] and len(contents(sid_file)) == 8, str(contents(sid_file))) + check("A3 input history in the vault", history(home)[:3] == ["hello one", "hello two", "/quit"], str(history(home))) + + # shell_sessions rows registered, keyed like the files + db = sqlite3.connect(os.path.join(home, "memory.db")) + sids = {row[0] for row in db.execute("SELECT sid FROM shell_sessions").fetchall()} + db.close() + sid_a = os.path.basename(sid_file)[len("default.session."):-len(".json")] + check("shell_sessions row for pane A", sid_a in sids, "%s not in %s" % (sid_a, sids)) + + # python reads what C++ wrote (same files, same keys) + sys.path.insert(0, ROOT) + env_backup = dict(os.environ) + os.environ["TAI_HOME"] = home + import argparse as ap + import tai + config = tai.Config(ap.Namespace(profile="default", yes=False, no_opencode=False)) + store = tai.Store(config, tai.Seal(home, "")) + loaded = store.load_session("default", ctx=sid_a) + check("python loads the C++ session", [(m["role"], m["content"]) for m in loaded] == contents(sid_file)) + # and C++ resumes a session python wrote + store.save_session("default", [{"role": "system", "content": "s"}, {"role": "user", "content": "from python"}, {"role": "assistant", "content": "ok"}], ctx=sid_a) + store.close() + os.environ.clear() + os.environ.update(env_backup) + check("A4 exit", pane_a.run(["and back"]) == 0) + check("A4 C++ resumed the python-written session", contents(sid_file) == [("user", "from python"), ("assistant", "ok"), ("user", "and back"), ("assistant", "echo: and back")], str(contents(sid_file))) + finally: + pane_a.close() + pane_b.close() + + # interrupted turn and --continue, in a fresh pane + home2 = tempfile.mkdtemp(prefix="detz-repl-") + pane_c = Pane(home2, opts.probe, 21) + try: + check("C1 crash exit code", pane_c.run(["CRASH"]) == 9) + files = session_files(home2) + check("C1 checkpoint saved the user message", len(files) == 1 and contents(files[0]) == [("user", "CRASH")], str(files)) + turns = glob.glob(os.path.join(home2, "profiles", "default.turn.*.json")) + check("C1 turn left open", len(turns) == 1 and read_json(turns[0]).get("open") is True and read_json(turns[0]).get("goal") == "CRASH", str(turns)) + check("C2 continue exit", pane_c.run(["/quit"], "--continue") == 0) + got = contents(files[0]) + check("C2 resume turn ran", got[0] == ("user", "CRASH") and got[1][0] == "user" and got[1][1].startswith("Continue where you left off: the interrupted request was: CRASH"), str(got)) + check("C2 turn closed", read_json(turns[0]).get("open") is False) + finally: + pane_c.close() + + # a pinned TAI_SESSION is shared by different panes + home3 = tempfile.mkdtemp(prefix="detz-repl-") + pane_d = Pane(home3, opts.probe, 31, {"TAI_SESSION": "pinned1"}) + pane_e = Pane(home3, opts.probe, 32, {"TAI_SESSION": "pinned1"}) + try: + check("D1 exit", pane_d.run(["from d"]) == 0) + check("D2 exit", pane_e.run(["from e"]) == 0) + files = session_files(home3) + check("D shared pinned context", [os.path.basename(f) for f in files] == ["default.session.pinned1.json"], str(files)) + check("D both panes wrote it", contents(files[0]) == [("user", "from d"), ("assistant", "echo: from d"), ("user", "from e"), ("assistant", "echo: from e")], str(contents(files[0]))) + finally: + pane_d.close() + pane_e.close() + + print("%d checks, %d failures" % (checks, len(failures))) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tests/repl_probe.cpp b/taicpp/tests/repl_probe.cpp new file mode 100644 index 0000000..c732034 --- /dev/null +++ b/taicpp/tests/repl_probe.cpp @@ -0,0 +1,67 @@ +// The real Repl on the real Conversation and Store, with a scripted host in +// place of the model loop (no model is called). Used by tests/repl_persist.py, +// which starts it repeatedly under chosen shells and checks the vault. +// +// repl_probe [--profile NAME] [--continue] [--yolo] [--oneshot TEXT] +// +// Host behaviour: every input line becomes a user message plus the assistant +// reply "echo: "; the input "CRASH" saves a checkpoint with the turn +// still open and kills the process (an interrupted turn for --continue). +#include +#include +#include +#include +#include "config/config.h" +#include "repl/boot.h" +#include "repl/repl.h" +#include "session/conversation.h" +#include "session/shell_context.h" +#include "store/store.h" +using namespace detz; + +class EchoHost : public repl::ReplHost { + public: + explicit EchoHost(session::Conversation& conv) : conv_(conv) {} + std::string run_turn(const std::string& text) override { + conv_.begin_turn(text); + conv_.add_message("user", text); + if (text == "CRASH") { + conv_.checkpoint(); + std::fflush(stdout); + ::_exit(9); + } + std::string reply = "echo: " + text; + conv_.add_message("assistant", reply); + conv_.close_turn(); + return reply; + } + + private: + session::Conversation& conv_; +}; + +int main(int argc, char** argv) { + std::string profile = "default", oneshot; + bool resume = false, yolo = false, has_oneshot = false; + for (int i = 1; i < argc; i++) { + std::string a = argv[i]; + if (a == "--profile" && i + 1 < argc) profile = argv[++i]; + else if (a == "--continue") resume = true; + else if (a == "--yolo") yolo = true; + else if (a == "--oneshot" && i + 1 < argc) { oneshot = argv[++i]; has_oneshot = true; } + } + repl::BootOptions boot; + boot.profile = profile; + boot.yolo = yolo; + boot.version = "probe"; + boot.mode = has_oneshot ? "one-shot" : "repl"; + auto session = repl::Boot::open(boot); + session::Conversation& conv = *session->conversation; + EchoHost host(conv); + repl::Repl::Options options; + options.version = "probe"; + options.resume = resume; + repl::Repl app(conv, host, options); + if (has_oneshot) return app.oneshot(oneshot); + return app.run(); +} diff --git a/taicpp/tests/store_test.cpp b/taicpp/tests/store_test.cpp index 7ebdcf8..6657b39 100644 --- a/taicpp/tests/store_test.cpp +++ b/taicpp/tests/store_test.cpp @@ -1,25 +1,23 @@ -#include "../src/store/store.h" -#include +// Store smoke test (record CRUD, search, spill). The full parity suite is +// tests/foundation_parity.py; this one has no python dependency. #include +#include +#include +#include "../src/store/spill.h" +#include "../src/store/store.h" int main() { using namespace detz::store; - Store st(":memory:"); - assert(st.ok() && st.schema_version() == 1); - int64_t id = st.add_record("note", "hello", "the quick brown fox jumps"); - auto r = st.get_record(id); - assert(r && r->title == "hello"); - assert(st.update_record(id, "hello2", "the quick brown fox jumps high")); - auto hits = st.search("fox"); - assert(!hits.empty()); - assert(st.delete_record(id)); - assert(!st.get_record(id)); - Store st2(":memory:"); - std::string big(5000, 'x'); - std::string v = spill_or_truncate(&st2, "note", "big", big); - assert(v.find("spilled to") != std::string::npos); - assert(truncate_text("abc", 4000) == "abc"); - DebugLog dl; dl.info("a"); dl.warn("b"); dl.error("c"); - assert(dl.lines().size() == 3); - std::puts("M3 OK"); + auto st = Store::Open(":memory:"); + std::string id = st->add_record("note", "hello", "the quick brown fox jumps", {"animals"}); + auto r = st->get_record(id); + if (!r || r->get_string("title") != "hello") { std::puts("get_record failed"); return 1; } + if (st->search_records("fox").size() != 1) { std::puts("search_records failed"); return 1; } + if (st->fts_search("fox").empty()) { std::puts("fts_search failed"); return 1; } + if (!st->delete_record(id) || st->get_record(id)) { std::puts("delete_record failed"); return 1; } + std::string big(7000, 'x'); + std::string view = Spill::spill_or_truncate(st.get(), "note", "big", big); + if (view.find("chars spilled to mem:") == std::string::npos) { std::puts("spill failed"); return 1; } + if (Spill::truncate("abc") != "abc") { std::puts("truncate failed"); return 1; } + std::puts("store OK"); return 0; }