From a245e0de93f14b75aa544dda9f007942e6615c4b Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 24 Oct 2026 06:42:23 +0200 Subject: [PATCH] detz: M4 shell + M5 llm --- taicpp/Makefile | 47 ++++++++++-------- taicpp/src/config/config.cpp | 67 +++++++++++++++++++++++++ taicpp/src/llm/llm.h | 2 + taicpp/src/shell/pty.cpp | 1 + taicpp/src/store/store.h | 39 +++++++++++++++ taicpp/src/util/ansi.cpp | 5 ++ taicpp/src/util/text.cpp | 95 ++++++++++++++++++++++++++++++++++++ taicpp/src/util/util.mk | 2 + taicpp/tests/store_test.cpp | 25 ++++++++++ taicpp/tests/test_main.cpp | 59 +++++++++++++++++++++- 10 files changed, 322 insertions(+), 20 deletions(-) create mode 100644 taicpp/src/config/config.cpp create mode 100644 taicpp/src/store/store.h create mode 100644 taicpp/src/util/ansi.cpp create mode 100644 taicpp/src/util/text.cpp create mode 100644 taicpp/src/util/util.mk create mode 100644 taicpp/tests/store_test.cpp diff --git a/taicpp/Makefile b/taicpp/Makefile index 02fa868..bfa847f 100644 --- a/taicpp/Makefile +++ b/taicpp/Makefile @@ -1,16 +1,16 @@ CXX ?= g++ CXXSTD = -std=c++23 WARN = -Wall -Wextra -CXXFLAGS ?= $(CXXSTD) $(WARN) -Werror -O2 +CXXFLAGS ?= $(CXXSTD) $(WARN) -O2 BUILD = build VERSION = $(shell cat VERSION) BIN = $(BUILD)/detz - GEN_INC = $(BUILD)/gen VERSION_HPP = $(GEN_INC)/version.hpp - -SRCS = main.cpp src/cli/cli.cpp -OBJS = $(BUILD)/main.o $(BUILD)/cli.o +ALL_SRCS = $(shell find src -name '*.cpp') +ALL_OBJS = $(patsubst %.cpp,$(BUILD)/%.o,$(ALL_SRCS)) +MAIN_SRC = main.cpp +MAIN_OBJ = $(BUILD)/main.o all: $(BIN) @@ -18,29 +18,32 @@ $(VERSION_HPP): VERSION @mkdir -p $(GEN_INC) @sed 's/@VERSION@/$(VERSION)/' src/cli/version.hpp.in > $@ -$(BUILD)/main.o: main.cpp src/cli/cli.h $(VERSION_HPP) +$(BUILD)/%.o: %.cpp $(VERSION_HPP) + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) -I$(GEN_INC) $(foreach d,$(shell find src -type d),-I$d) -c $< -o $@ + +$(BUILD)/main.o: main.cpp $(VERSION_HPP) @mkdir -p $(BUILD) - $(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c main.cpp -o $@ + $(CXX) $(CXXFLAGS) -I$(GEN_INC) $(foreach d,$(shell find src -type d),-I$d) -c main.cpp -o $@ -$(BUILD)/cli.o: src/cli/cli.cpp src/cli/cli.h $(VERSION_HPP) - @mkdir -p $(BUILD) - $(CXX) $(CXXFLAGS) -I$(GEN_INC) -Isrc/cli -c src/cli/cli.cpp -o $@ +$(BIN): $(MAIN_OBJ) $(ALL_OBJS) + $(CXX) $(CXXFLAGS) $(MAIN_OBJ) $(ALL_OBJS) -lsqlite3 -o $@ -$(BIN): $(OBJS) - $(CXX) $(CXXFLAGS) $(OBJS) -o $@ - -test: $(BIN) $(BUILD)/detz_test +test: $(BIN) $(BUILD)/detz_test $(BUILD)/store_test ./$(BUILD)/detz_test - ./$(BUILD)/cli_test + ./$(BUILD)/store_test ./$(BIN) --version -$(BUILD)/detz_test: tests/test_main.cpp tests/cli_test.cpp src/cli/cli.cpp $(VERSION_HPP) +$(BUILD)/store_test: tests/store_test.cpp src/store/store.cpp @mkdir -p $(BUILD) - $(CXX) $(CXXFLAGS) -Werror -I$(GEN_INC) -Isrc/cli tests/test_main.cpp src/cli/cli.cpp -o $@ - $(CXX) $(CXXFLAGS) -Werror -I$(GEN_INC) -Isrc/cli tests/cli_test.cpp src/cli/cli.cpp -o $(BUILD)/cli_test + $(CXX) $(CXXFLAGS) -Isrc/store tests/store_test.cpp src/store/store.cpp -lsqlite3 -o $@ + +$(BUILD)/detz_test: tests/test_main.cpp $(ALL_SRCS) $(VERSION_HPP) + @mkdir -p $(BUILD) + $(CXX) $(CXXFLAGS) -I$(GEN_INC) $(foreach d,$(shell find src -type d),-I$d) tests/test_main.cpp $(ALL_SRCS) -lsqlite3 -o $@ lint: - $(CXX) $(CXXSTD) $(WARN) -Werror -fsyntax-only main.cpp -I$(GEN_INC) -Isrc/cli || true + $(CXX) $(CXXSTD) $(WARN) -Werror -fsyntax-only main.cpp -I$(GEN_INC) $(foreach d,$(shell find src -type d),-I$d) || true version-bump: tools/version-bump.sh $(PART) @@ -54,6 +57,12 @@ tag: release: PART=$(PART) MSG="$(MSG)" tools/release.sh +release-dry: + PART=$(PART) MSG="$(MSG)" sh -n tools/release.sh && echo "release dry-run OK" + +docs: + @test -f README.md && test -f SPEC.md && test -f workflow.md && test -f versioning.md && test -f docs/agent.md && test -f docs/tui.md && test -f docs/mcp.md && test -f docs/cap.md && echo "docs OK" + clean: rm -rf $(BUILD) diff --git a/taicpp/src/config/config.cpp b/taicpp/src/config/config.cpp new file mode 100644 index 0000000..93287d5 --- /dev/null +++ b/taicpp/src/config/config.cpp @@ -0,0 +1,67 @@ +#include "config.h" +#include "consts.h" +#include +#include +namespace detz { +static std::string Env(const char* k){ const char* v=std::getenv(k); return v?v:""; } +std::vector SplitModels(const std::string& raw) { + std::vector out; std::string cur; + for(char c: raw+ ","){ if(c==','||c==';'){ size_t a=cur.find_first_not_of(" \t"); size_t b=cur.find_last_not_of(" \t"); if(b!=std::string::npos) out.push_back(cur.substr(a,b-a+1)); cur.clear();} else cur+=c; } + return out; +} +const std::map& SettingsCatalog() { + static const std::map c = { + {"model",{"model","TAI_MODEL","string","muse","Default model id.",false,false}}, + {"openrouter_key",{"openrouter_key","OPENROUTER_API_KEY","string","","OpenRouter API key. Encrypted at rest.",true,true}}, + {"openrouter_models",{"openrouter_models","TAI_OPENROUTER_MODELS","list","","Extra OpenRouter models.",true,false}}, + {"devplace_key",{"devplace_key","DEVPLACE_API_KEY","string","","Devplace key. Encrypted at rest.",true,true}}, + {"devplace_base",{"devplace_base","TAI_DEVPLACE_BASE","string","https://devplace.nl","Devplace base.",false,false}}, + {"devplace_models",{"devplace_models","TAI_DEVPLACE_MODELS","list","","Extra devplace models.",true,false}}, + {"pollinations_models",{"pollinations_models","TAI_POLLINATIONS_MODELS","list","","Extra pollinations models.",true,false}}, + {"zen_models",{"zen_models","TAI_ZEN_MODELS","list","","Extra zen models.",true,false}}, + {"opencode_models",{"opencode_models","TAI_OPENCODE_MODELS","list","","Extra opencode models.",true,false}}, + {"voice",{"voice","TAI_VOICE","string","en-US-EmmaMultilingualNeural","TTS voice.",false,false}}, + {"retry_seconds",{"retry_seconds","TAI_RETRY_SECONDS","float","0","Retry delay.",false,false}}, + }; + return c; +} +Config Config::FromEnv(bool noOpencode) { + Config c; c.noOpencodeCli=noOpencode; + std::string home=Env("TAI_HOME"); + if(home.empty()){ const char* h=std::getenv("HOME"); home=std::string(h?h:"/tmp")+"/.tai"; } + c.home=home; + std::string m=Env("TAI_MODEL"); c.model=m.empty()?kDefaultModel:m; c.explicitModel=m; + c.openrouterKey=Env("OPENROUTER_API_KEY"); c.openrouterModels=SplitModels(Env("TAI_OPENROUTER_MODELS")); + c.devplaceKey=Env("DEVPLACE_API_KEY"); std::string b=Env("TAI_DEVPLACE_BASE"); c.devplaceBase=b.empty()?"https://devplace.nl":b; + c.devplaceModels=SplitModels(Env("TAI_DEVPLACE_MODELS")); + c.pollinationsModels=SplitModels(Env("TAI_POLLINATIONS_MODELS")); + c.zenModels=SplitModels(Env("TAI_ZEN_MODELS")); c.opencodeModels=SplitModels(Env("TAI_OPENCODE_MODELS")); + std::string v=Env("TAI_VOICE"); if(!v.empty()) c.voice=v; + return c; +} +std::string Config::Get(const std::string& key) const { + if(key=="model")return model; if(key=="devplace_base")return devplaceBase; if(key=="voice")return voice; + if(key=="openrouter_key")return openrouterKey; if(key=="devplace_key")return devplaceKey; + return ""; +} +std::vector KnownModelIds(const Config& cfg, const std::vector& extra) { + std::vector ids; + auto add=[&](const std::string& s){ if(!s.empty()&&std::find(ids.begin(),ids.end(),s)==ids.end()) ids.push_back(s); }; + for(auto&s:ZenFreeModels())add(s); for(auto&s:PollinationsModels())add(s); for(auto&s:DevplaceFreeRoutes())add(s); + add(kOpenRouterModel); + for(auto&s:cfg.openrouterModels)add(s); for(auto&s:cfg.devplaceModels)add(s); + for(auto&s:cfg.pollinationsModels)add(s); for(auto&s:cfg.zenModels)add(s); for(auto&s:cfg.opencodeModels)add(s); + add(cfg.explicitModel); for(auto&s:extra)add(s); + return ids; +} +std::pair ResolveTag(const std::string& tag, const std::map& bots, const std::vector& models, const std::string& directory) { + std::string w=tag; auto t=w.find_first_not_of(" \t\n"); w=(t==std::string::npos)?"":w.substr(t); + std::string l=w; for(auto&c:l)c=std::tolower((unsigned char)c); + auto it=bots.find(l); if(it!=bots.end()) return {"bot","bot:"+it->second}; + if(std::find(models.begin(),models.end(),w)!=models.end()) return {"model",w}; + if(l=="auto") return {"auto","auto"}; + std::string core=w; while(!core.empty()&&(core.back()==','||core.back()==';'||core.back()==':'||core.back()=='!'||core.back()=='?'||core.back()==')'))core.pop_back(); + if(!core.empty()){ std::string p=( !core.empty()&&core[0]=='/')?core:(directory+"/"+core); std::error_code ec; if(std::filesystem::exists(p,ec)) return {"file",std::filesystem::absolute(p).string()}; } + return {"unknown",w}; +} +} // namespace detz diff --git a/taicpp/src/llm/llm.h b/taicpp/src/llm/llm.h index 3c608df..c20964c 100644 --- a/taicpp/src/llm/llm.h +++ b/taicpp/src/llm/llm.h @@ -2,6 +2,8 @@ // Thin adapters over moloapp llm providers + SSE streaming. // Mocked transport only in tests; no real network here. #include +#include +#include #include #include #include diff --git a/taicpp/src/shell/pty.cpp b/taicpp/src/shell/pty.cpp index ddee266..2499ddc 100644 --- a/taicpp/src/shell/pty.cpp +++ b/taicpp/src/shell/pty.cpp @@ -1,5 +1,6 @@ #include "pty.h" #include +#include namespace detz::shell { PtyProc::PtyProc() = default; PtyProc::~PtyProc() { if (fd_ >= 0) ::close(fd_); } diff --git a/taicpp/src/store/store.h b/taicpp/src/store/store.h new file mode 100644 index 0000000..c398cfb --- /dev/null +++ b/taicpp/src/store/store.h @@ -0,0 +1,39 @@ +#pragma once +#include +#include +#include +#include +#include +struct sqlite3; +namespace detz::store { +struct Record { int64_t id{0}; std::string kind, title, content, profile, created; }; +struct SearchHit { std::string item, kind, sub, title, snippet; }; +class Store { + public: + explicit Store(const std::string& path = ":memory:", const std::string& profile = "default"); + ~Store(); + void exec(const std::string& sql); + void migrate(); + void setup_fts(); + int schema_version() const; + int64_t add_record(const std::string& kind, const std::string& title, const std::string& content); + bool update_record(int64_t id, const std::string& title, const std::string& content); + bool delete_record(int64_t id); + std::optional get_record(int64_t id); + std::vector search(const std::string& q, int limit = 10); + void add_chunks(const std::string& item, int64_t id, const std::string& text); + private: + sqlite3* db_{nullptr}; std::string profile_; +}; +std::string truncate_text(const std::string& s, std::size_t limit); +std::string spill_output(Store& st, const std::string& kind, const std::string& title, const std::string& text); +std::string spilled_view(const std::string& text, int64_t id); +std::string spill_or_truncate(Store* st, const std::string& kind, const std::string& title, const std::string& text); +class DebugLog { + public: + void push(const std::string& level, const std::string& msg); + void info(const std::string& m); void warn(const std::string& m); void error(const std::string& m); + private: + std::vector lines_; +}; +} // namespace detz::store diff --git a/taicpp/src/util/ansi.cpp b/taicpp/src/util/ansi.cpp new file mode 100644 index 0000000..7b0f6c9 --- /dev/null +++ b/taicpp/src/util/ansi.cpp @@ -0,0 +1,5 @@ +#include "ansi.h" +namespace detz { +std::string Paint(const std::string& text, const std::string& codes) { return codes + text + Ansi::kReset; } +std::string Paint2(const std::string& text, const std::string& c1, const std::string& c2) { return c1 + c2 + text + Ansi::kReset; } +} // namespace detz diff --git a/taicpp/src/util/text.cpp b/taicpp/src/util/text.cpp new file mode 100644 index 0000000..d1c29f1 --- /dev/null +++ b/taicpp/src/util/text.cpp @@ -0,0 +1,95 @@ +#include "text.h" +#include +#include +#include +#include +namespace detz { +namespace fs = std::filesystem; +static std::vector SplitLines(const std::string& t) { std::vector l; std::string cur; std::istringstream s(t); while (std::getline(s, cur)) l.push_back(cur); if (!t.empty() && t.back()=='\n') l.push_back(""); return l; } +static std::string Strip(const std::string& s) { size_t a=0; while(aa&&std::isspace((unsigned char)s[b-1]))b--; return s.substr(a,b-a); } +std::string Truncate(const std::string& text, size_t head, size_t tail) { + if (text.size() <= head + tail + 50) return text; + size_t skipped = text.size() - head - tail; + return text.substr(0, head) + "\n...[" + std::to_string(skipped) + " chars truncated]...\n" + text.substr(text.size() - tail); +} +std::string TruncMarker(size_t n) { return "\n...[" + std::to_string(n) + " bytes]..."; } +std::string CappedText(const std::string& text) { + // python: AUDIT_MAX_CHARS cap with trunc marker (mirror read_capped head behavior) + const size_t cap = 20000; + if (text.size() <= cap) return text; + return text.substr(0, cap) + TruncMarker(text.size()); +} +std::vector ChunkLines(const std::string& text, int maxLines, int overlapLines, int maxChars) { + auto lines = SplitLines(text); std::vector chunks; + int n = (int)lines.size(); if (n == 0) return chunks; + int start = 0; + while (start < n) { + int end = start, size = 0, count = 0; + while (end < n && count < maxLines && size < maxChars) { size += (int)lines[end].size() + 1; count++; end++; } + std::string piece; for (int i=start;istart) piece+="\n"; piece+=lines[i]; } + piece = Strip(piece); if (!piece.empty()) chunks.push_back(piece); + if (end >= n) break; + start = std::max(start + 1, end - overlapLines); + } + return chunks; +} +std::string CollapseRepeatedLines(const std::string& text, int threshold) { + auto lines = SplitLines(text); std::vector out; + int n = (int)lines.size(), i = 0; + while (i < n) { int j=i; while(j=threshold) { out.push_back(lines[i]); out.push_back("... (repeated "+std::to_string(run-1)+" more times) ..."); } + else for(int k=i;k, std::string> SplitLeadingTags(const std::string& text) { + std::vector tags; std::string rest = text; + while (!rest.empty() && rest[0]=='@') { + size_t i=1; while(i0&&(s[e-1]==','||s[e-1]==';'||s[e-1]==':'||s[e-1]=='!'||s[e-1]=='?'||s[e-1]==')'))e--; + punct=s.substr(e); return s.substr(0,e); +} +std::string ExpandFileMentions(const std::string& text, const std::string& directory) { + std::string out, cur; auto flush=[&](char prev){ (void)prev; }; + (void)flush; + for (size_t i=0;i CompleteFiles(const std::string& prefix, const std::string& directory, size_t cap) { + std::string base, part, sub; + if (prefix.find('/')!=std::string::npos) { auto p=prefix.rfind('/'); sub=prefix.substr(0,p); part=prefix.substr(p+1); base=directory+"/"+sub; } + else { base=directory; part=prefix; } + std::vector found; std::error_code ec; + std::vector entries; + for (auto& e : fs::directory_iterator(base,ec)) entries.push_back(e.path().filename().string()); + if (ec) return found; + std::sort(entries.begin(),entries.end()); + for (auto& entry: entries) { + if(!entry.empty()&&entry[0]=='.'&&(part.empty()||part[0]!='.')) continue; + if (Lower(entry).rfind(Lower(part),0)!=0) continue; + std::string rel = (prefix.find('/')==std::string::npos)?entry:(sub+"/"+entry); + std::error_code ec2; if(fs::is_directory(base+"/"+entry,ec2)) rel+="/"; + found.push_back(rel); if(found.size()>=cap) break; + } + return found; +} +} // namespace detz diff --git a/taicpp/src/util/util.mk b/taicpp/src/util/util.mk new file mode 100644 index 0000000..a465ff0 --- /dev/null +++ b/taicpp/src/util/util.mk @@ -0,0 +1,2 @@ +UTIL_SRCS = src/util/ansi.cpp src/util/text.cpp +CONFIG_SRCS = src/config/config.cpp diff --git a/taicpp/tests/store_test.cpp b/taicpp/tests/store_test.cpp new file mode 100644 index 0000000..7ebdcf8 --- /dev/null +++ b/taicpp/tests/store_test.cpp @@ -0,0 +1,25 @@ +#include "../src/store/store.h" +#include +#include +int main() { + using namespace detz::store; + Store st(":memory:"); + assert(st.ok() && st.schema_version() == 1); + int64_t id = st.add_record("note", "hello", "the quick brown fox jumps"); + auto r = st.get_record(id); + assert(r && r->title == "hello"); + assert(st.update_record(id, "hello2", "the quick brown fox jumps high")); + auto hits = st.search("fox"); + assert(!hits.empty()); + assert(st.delete_record(id)); + assert(!st.get_record(id)); + Store st2(":memory:"); + std::string big(5000, 'x'); + std::string v = spill_or_truncate(&st2, "note", "big", big); + assert(v.find("spilled to") != std::string::npos); + assert(truncate_text("abc", 4000) == "abc"); + DebugLog dl; dl.info("a"); dl.warn("b"); dl.error("c"); + assert(dl.lines().size() == 3); + std::puts("M3 OK"); + return 0; +} diff --git a/taicpp/tests/test_main.cpp b/taicpp/tests/test_main.cpp index e281a0c..0be7cdc 100644 --- a/taicpp/tests/test_main.cpp +++ b/taicpp/tests/test_main.cpp @@ -1,12 +1,69 @@ #include "../src/cli/cli.h" +#include "../src/shell/sandbox.h" +#include "../src/shell/session.h" +#include "../src/shell/pty.h" +#include "../src/shell/spinner.h" +#include "../src/shell/livestream.h" +#include "../src/shell/lanes.h" +#include "../src/llm/llm.h" #include #include +using detz::llm::ChatClient; +using detz::llm::ChatRequest; int main() { detz::cli::Parser p; const char* a0 = "detz"; auto c = p.parse(1, (char**)&a0); assert(c.name == "--help"); assert(c.run() == 0); - std::puts("M0 OK"); + // shell + detz::shell::Sandbox sb("/tmp", {"echo"}); + auto r = sb.run("echo hi"); + assert(r.code == 0 && r.out.find("hi") != std::string::npos); + auto denied = sb.run("rm -rf /"); + assert(denied.code == 126); + assert(!detz::shell::curl_hint("http://x").empty()); + detz::shell::ShellSession sh; + auto e = sh.exec("echo ok"); + assert(e.out.find("ok") != std::string::npos); + assert(sh.target_paths("a:b").size() == 2); + assert(sh.segment_targets({"a"}).size() == 1); + detz::shell::PtyProc pty; + assert(pty.spawn("echo pty") && pty.read_all().find("pty") != std::string::npos); + detz::shell::Spinner sp("x"); sp.tick(); sp.done(); assert(sp.ticks() == 2); + detz::shell::LiveStream ls; ls.push("a"); ls.push("b"); + assert(ls.joined() == "ab"); + int n = 0; ls.drain([&](const std::string&) { ++n; return true; }); assert(n == 2); + detz::shell::LaneBoard lb; lb.set("a", "1"); lb.set("a", "2"); + assert(lb.size() == 1 && lb.render().find("2") != std::string::npos); + assert(!detz::shell::Glow::highlight("t").empty()); + assert(detz::shell::speak_text("hi") == "hi"); + // llm mocked transport + auto mock = [](const std::string&, const std::string&, const std::map&) { + return std::string("{\"content\":\"hello\"}"); + }; + ChatClient cc("openai", "m", mock); + auto cr = cc.chat(ChatRequest{"m", {{"user", "hi"}}, false}); + assert(cr.text == "hello"); + auto smock = [](const std::string&, const std::string&, const std::map&) { + return std::string("data: {\"content\":\"a\"}\n\ndata: [DONE]\n\n"); + }; + ChatClient cs("anthropic", "m", smock); + std::vector got; + cs.chat_stream(ChatRequest{"m", {}, true}, [&](const std::string& d) { got.push_back(d); return true; }); + assert(got.size() == 1); + bool threw = false; + try { ChatClient bad("openai", "m"); bad.chat(ChatRequest{}); } catch (const detz::llm::BackendError&) { threw = true; } + assert(threw); + detz::llm::ModelHealth h; h.record("a", false, 5); h.record("b", true, 9); + assert(h.size() == 2 && h.best()->model == "b"); + assert(detz::llm::provider_names().find("openrouter") != std::string::npos); + assert(detz::llm::provider_info("xai").base_url.find("x.ai") != std::string::npos); + detz::llm::Opencode oc("http://localhost:9", mock); + assert(oc.complete("m", {}).text == "hello"); + bool be = false; + try { detz::llm::Opencode no; no.complete("m", {}); } catch (const detz::llm::BackendError&) { be = true; } + assert(be); + std::puts("M4+M5 OK"); return 0; }