tai/taicpp/tests/shell_test.cpp

// Unit tests for src/shell and src/util. Real processes, a real pty, real files.
// Build and run (outputs under build/ only):
//   g++ -std=c++23 -Wall -Wextra -Isrc/util -Isrc/shell -pthread tests/shell_test.cpp
//       (every src/util/*.cpp except http.cpp) src/shell/*.cpp -o build/shell_test && build/shell_test
// The byte-for-byte comparison against tai.py lives in tests/shell_parity.py.
#include "ansi.h"
#include "diffrender.h"
#include "boxdata.h"
#include "glow.h"
#include "jsonvalue.h"
#include "lanes.h"
#include "livestream.h"
#include "markdown.h"
#include "pty.h"
#include "pyglob.h"
#include "runlive.h"
#include "sandbox.h"
#include "sequencematcher.h"
#include "session.h"
#include "shellio.h"
#include "shlex.h"
#include "spinner.h"
#include "targets.h"
#include "text.h"
#include "utf8.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <thread>
#include <unistd.h>
using namespace detz;
using namespace detz::shell;
namespace fs = std::filesystem;

static int gFailures = 0, gChecks = 0;
#define CHECK(cond) do { ++gChecks; if (!(cond)) { ++gFailures; std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } } while (0)
#define CHECK_EQ(a, b) do { ++gChecks; auto va = (a); auto vb = (b); if (!(va == vb)) { ++gFailures; std::printf("FAIL %s:%d: %s == %s\n", __FILE__, __LINE__, #a, #b); } } while (0)


static void TestUtf8() {
  CHECK_EQ(Utf8::Len("caf\xc3\xa9"), (size_t)4);
  CHECK_EQ(Utf8::Slice("a\xc3\xa9z", 1, 2), std::string("\xc3\xa9"));
  CHECK_EQ(Utf8::Tail("abcdef", 2), std::string("ef"));
  CHECK_EQ(Utf8::Strip(" \t x\xc2\xa0\n"), std::string("x"));
  CHECK_EQ(Utf8::SplitLines("a\r\nb\rc\n").size(), (size_t)3);
  CHECK_EQ(Utf8::ExpandTabs("a\tb", 8), std::string("a       b"));
  CHECK_EQ(Utf8::Encode(Utf8::Decode("x\xff y")), std::string("x\xef\xbf\xbd y"));
}

static void TestText() {
  CHECK_EQ(Truncate("abc"), std::string("abc"));
  CHECK(Truncate(std::string(3000, 'a'), 100, 100).find("2800 chars truncated") != std::string::npos);
  // text[-0:] is the whole text in Python, so tail=0 keeps everything after the marker
  CHECK_EQ(Truncate(std::string(3000, 'a'), 5, 0).size(), (size_t)5 + std::string("\n...[2995 chars truncated]...\n").size() + 3000);
  CHECK_EQ(TruncMarker(7), std::string("\n...[truncated, full size 7 bytes]..."));
  Capped c = CappedText(std::string(20001, 'x'));
  CHECK(c.present && c.size == 20001 && c.text.size() == 20000 + TruncMarker(20001).size());
  CHECK(!CappedText(std::nullopt).present);
  CHECK_EQ(ChunkLines("a\nb\nc", 2, 1, 4000).size(), (size_t)2);
  CHECK_EQ(CollapseRepeatedLines("x\nx\nx\ny"), std::string("x\n... (repeated 2 more times) ...\ny"));
  auto t = SplitLeadingTags("@a @b  rest @c");
  CHECK(t.first.size() == 2 && t.second == "rest @c");
  CHECK_EQ(HumanSize(2048), std::string("2.0 KB"));
  CHECK_EQ(FormatDelay(7200), std::string("2h"));
  CHECK_EQ(ParseDuration("3 m"), 180L);
  CHECK_EQ(ParseDuration("x"), -1L);
  CHECK_EQ(AsInt("1_0", 5), 10L);
  CHECK_EQ(AsInt("1__0", 5), 5L);
  CHECK_EQ(ToolNameClean("shell<|channel|>x"), std::string("shell"));
  fs::path dir = fs::temp_directory_path() / "detz_shell_test_text";
  fs::remove_all(dir); fs::create_directories(dir / "Sub");
  std::ofstream(dir / "alpha.txt") << "x";
  std::ofstream(dir / ".dot") << "x";
  CHECK_EQ(ExpandFileMentions("see @alpha.txt, ok", dir.string()), "see " + (dir / "alpha.txt").string() + ", ok");
  CHECK_EQ(ExpandFileMentions("mail me@alpha.txt", dir.string()), std::string("mail me@alpha.txt"));
  auto cf = CompleteFiles("", dir.string());
  CHECK(cf.size() == 2 && cf[0] == "Sub/" && cf[1] == "alpha.txt");
  CHECK_EQ(CompleteFiles(".", dir.string()).size(), (size_t)1);
  fs::remove_all(dir);
  ReadCappedResult rc = ReadCapped("/nonexistent/file");
  CHECK(!rc.ok);
}

static void TestAnsi() {
  CHECK_EQ(AnsiText::Strip("\033[31mred\033[0m"), std::string("red"));
  CHECK_EQ(AnsiText::Truncate("hello world", 5), std::string("hello\033[0m"));
  CHECK_EQ(AnsiText::Truncate("hi", 5), std::string("hi"));
  CHECK_EQ(AnsiText::StyleSpan("a\033[0mb", {"\033[1m"}), std::string("\033[1ma\033[0m\033[1mb\033[0m"));
  auto w = AnsiText::Wrap("one two three four", 9);
  CHECK(w.size() == 3 && w[0] == "one two" && w[1] == "three" && w[2] == "four");
  CHECK_EQ(AnsiText::PadTinted("ab\033[0m", 5), std::string("ab   \033[0m"));
}

static void TestJson() {
  bool ok = false;
  JsonValue v = JsonValue::Parse("{\"b\": [1, 2.5, \"x\\u00e9\"], \"a\": null, \"a\": true}", &ok);
  CHECK(ok);
  CHECK_EQ(v.Dump(false), std::string("{\"b\": [1, 2.5, \"x\\u00e9\"], \"a\": true}"));
  CHECK_EQ(v.Dump(true), std::string("{\"a\": true, \"b\": [1, 2.5, \"x\\u00e9\"]}"));
  CHECK_EQ(v.Repr(), std::string("{'b': [1, 2.5, 'x\xc3\xa9'], 'a': True}"));
  JsonValue::Parse("{'a': 1}", &ok); CHECK(!ok);
  JsonValue::Parse("[1,]", &ok); CHECK(!ok);
  CHECK_EQ(JsonValue::Parse("123456789012345678901234567890", &ok).Dump(), std::string("123456789012345678901234567890"));
  CHECK_EQ(JsonValue::Parse("1e16", &ok).Dump(), std::string("1e+16"));
}

static void TestMarkdown() {
  CHECK_EQ(Markdown::Inline("a **b** c"), std::string("a \033[1mb\033[0m c"));
  CHECK_EQ(Markdown::Inline("snake_case_word"), std::string("snake_case_word"));
  CHECK_EQ(Markdown::Inline("`x`"), std::string("\033[36mx\033[0m"));
  CHECK_EQ(Markdown::Render("plain", 40, 0), std::string("plain"));
  CHECK_EQ(Markdown::Render("# T", 40, 1), std::string("\033[1m\033[36mT\033[0m"));
  CHECK(Markdown::SplitRow("| a | b\\|c |").size() == 2);
  CHECK(Markdown::IsSeparator("|:--|--:|"));
  CHECK(!Markdown::IsSeparator("| a |"));
}

static void TestDiff() {
  SequenceMatcher m({"a", "b", "c"}, {"a", "x", "c"});
  auto ops = m.Opcodes();
  CHECK(ops.size() == 3 && ops[1].tag == "replace");
  std::string out = DiffRender::Render("f.txt", std::string("a\nb\n"), std::string("a\nc\n"), 40, false);
  CHECK(out.find("\xe2\x94\x82 \xe2\x94\x80\xe2\x94\x80 f.txt (+1 -1)") == 0);
  CHECK_EQ(DiffRender::Render("f", std::string("x\n"), std::string("x\n"), 40, true), std::string(""));
}

static void TestShlexGlobTargets() {
  std::vector<std::string> t;
  CHECK(Shlex::Split("a 'b c' \"d\\\"e\" f\\ g ''", &t));
  CHECK(t.size() == 5 && t[1] == "b c" && t[2] == "d\"e" && t[3] == "f g" && t[4] == "");
  CHECK(!Shlex::Split("a 'unclosed", &t));
  CHECK(!Shlex::Split("trailing\\", &t));
  CHECK(PyGlob::FnmatchCase("abc.txt", "a*.t?t"));
  CHECK(PyGlob::FnmatchCase("b", "[a-c]"));
  CHECK(!PyGlob::FnmatchCase("d", "[!a-d]"));
  fs::path dir = fs::temp_directory_path() / "detz_shell_test_targets";
  fs::remove_all(dir); fs::create_directories(dir / "sub");
  std::ofstream(dir / "a.txt") << "x";
  std::ofstream(dir / "b.txt") << "x";
  std::ofstream(dir / "sub" / "c.txt") << "x";
  auto p = ShellTargets::Paths("rm *.txt && echo hi > sub/c.txt", dir.string());
  CHECK(p.size() == 3 && p[0] == (dir / "a.txt").string() && p[2] == (dir / "sub" / "c.txt").string());
  CHECK(ShellTargets::Paths("rm 'unclosed", dir.string()).empty());
  auto seg = ShellTargets::Segment({"sudo", "env", "A=1", "tee", "f", "-"});
  CHECK(seg.size() == 1 && seg[0] == "f");
  CHECK_EQ(ShellTargets::Segment({"truncate", "-s", "5", "f"}).size(), (size_t)1);
  CHECK_EQ(ShellTargets::Segment({"dd", "if=in", "of=out"})[0], std::string("out"));
  fs::remove_all(dir);
}

static void TestProcAndRun() {
  // Popen error text follows Python's OSError
  try { PopenOptions o; o.argv = {"/nonexistent/prog"}; Popen p(o); CHECK(false); }
  catch (const OsError& e) { CHECK_EQ(std::string(e.what()), std::string("[Errno 2] No such file or directory: '/nonexistent/prog'")); }
  try { PopenOptions o; o.argv = {"true"}; o.cwd = "/nonexistent/dir"; Popen p(o); CHECK(false); }
  catch (const OsError& e) { CHECK_EQ(std::string(e.what()), std::string("[Errno 2] No such file or directory: '/nonexistent/dir'")); }
  // subprocess.run semantics
  SubprocessOptions so;
  so.popen.argv = {"sh", "-c", "cat; echo err >&2; exit 4"};
  so.popen.out = StdMode::Pipe; so.popen.err = StdMode::Pipe; so.input = "in-data";
  CompletedProcess done = Subprocess::Run(so);
  CHECK(done.returncode == 4 && done.out == "in-data" && done.err == "err\n");
  SubprocessOptions slow;
  slow.popen.argv = {"sleep", "5"}; slow.timeout = 0.3;
  try { Subprocess::Run(slow); CHECK(false); }
  catch (const TimeoutExpired& e) { CHECK_EQ(std::string(e.what()), std::string("Command '['sleep', '5']' timed out after 0.3 seconds")); }
  // incremental decoding / newline translation
  IncrementalUtf8 dec;
  CHECK_EQ(dec.Decode("\xc3"), std::string(""));
  CHECK_EQ(dec.Decode("\xa9"), std::string("\xc3\xa9"));
  UniversalNewlines nl;
  CHECK_EQ(nl.Feed("a\r"), std::string("a"));
  CHECK_EQ(nl.Feed("\nb\rc"), std::string("\nb\nc"));
  CHECK_EQ(Subprocess::Which("sh").empty(), false);

  // run_live pipe path
  RunLiveOptions opt;
  RunResult r = RunLive::Run(Command::Shell("echo hi; echo err >&2; exit 3"), opt);
  CHECK(r.code == 3 && r.output == "hi\nerr\n" && !r.timedOut);
  r = RunLive::Run(Command::Shell("printf 'a\\rb\\r\\nc'"), opt);
  CHECK_EQ(r.output, std::string("a\nb\nc"));
  opt.timeout = 0.5;
  auto t0 = std::chrono::steady_clock::now();
  r = RunLive::Run(Command::Shell("echo before; sleep 5"), opt);
  double took = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
  // sh is killed but its sleep child keeps the pipe open: run_live joins the reader (up to 10s), like tai.py
  CHECK(r.timedOut && r.code == -9 && r.output == "before\n" && took > 4.0 && took < 13.0);
  try { RunLive::Run(Command::Argv({"/nonexistent/prog"}), opt); CHECK(false); } catch (const OsError&) { CHECK(true); }

  // big output through the pipe path
  opt.timeout = 60;
  r = RunLive::Run(Command::Shell("seq 1 200000"), opt);
  CHECK(r.code == 0 && r.output.size() == 1288895);

  // pty path: prompt detection, secret, decline, wait time not counted
  struct Rec : StreamSink { std::vector<std::string> lines; int pauses = 0; void Feed(const std::string& l) override { lines.push_back(l); } void Pause() override { ++pauses; } void Close(const std::string&) override {} };
  Rec rec;
  std::vector<std::pair<std::string, bool>> asked;
  RunLiveOptions po;
  po.timeout = 20; po.stream = &rec;
  po.ask = [&](const std::string& text, bool secret) -> std::optional<std::string> { asked.emplace_back(text, secret); return std::string("hunter2"); };
  r = RunLive::Run(Command::Shell("printf 'Password: '; read -r x; echo \"got:$x\""), po);
  CHECK(r.output == "Password: hunter2\ngot:hunter2\n" && asked.size() == 1 && asked[0].first == "Password:" && !asked[0].second && rec.pauses == 1);
  CHECK(rec.lines.size() == 3 && rec.lines[0].find("\xc2\xbb Password: hunter2") == 0);
  asked.clear(); rec.lines.clear();
  r = RunLive::Run(Command::Shell("stty -echo; printf 'Secret: '; read -r x; stty echo; echo; echo \"len:${#x}\""), po);
  CHECK(asked.size() == 1 && asked[0].second && rec.lines[0] == "\xc2\xbb Secret: (hidden)\n");
  asked.clear();
  po.ask = [&](const std::string& text, bool secret) -> std::optional<std::string> { asked.emplace_back(text, secret); return std::nullopt; };
  r = RunLive::Run(Command::Shell("printf 'Name: '; read -r x; echo \"read:$x:\""), po);
  CHECK(asked.size() == 1 && r.output.find("read::\n") != std::string::npos);
  po.timeout = 2;
  po.ask = [&](const std::string&, bool) -> std::optional<std::string> { std::this_thread::sleep_for(std::chrono::seconds(3)); return std::string("slow"); };
  r = RunLive::Run(Command::Shell("printf 'Q: '; read -r x; echo \"hi $x\""), po);
  CHECK(!r.timedOut && r.output == "Q: slow\nhi slow\n");
  po.ask = [&](const std::string&, bool) -> std::optional<std::string> { return std::string("x"); };
  po.timeout = 1;
  r = RunLive::Run(Command::Shell("echo start; sleep 6"), po);
  CHECK(r.timedOut && r.code == -9);
  // control sequences are stripped from pty output
  po.timeout = 10;
  r = RunLive::Run(Command::Shell("printf '\\033[31mred\\033[0m \\033]0;t\\007ok\\n'"), po);
  CHECK_EQ(r.output, std::string("red ok\n"));
  // RUN_OUTPUT_CAP bounds what comes back from the pty path
  po.timeout = 60;
  r = RunLive::Run(Command::Shell("seq 1 700000"), po);
  CHECK(r.output.size() < 4200000 && r.output.size() > 3900000);
  CHECK_EQ(RunLive::StripControl("a\033[1;2Hb\033]0;x\007c\x01"), std::string("abc"));
  CHECK(RunLive::LooksLikePrompt("Password:") && RunLive::LooksLikePrompt("Continue? (y/n)") && !RunLive::LooksLikePrompt("done"));
}

static void TestBurst() {
  // many concurrent runs through both paths must neither deadlock nor mix output
  std::atomic<int> bad{0};
  std::vector<std::thread> threads;
  for (int i = 0; i < 16; ++i) {
    threads.emplace_back([i, &bad] {
      RunLiveOptions o; o.timeout = 60;
      RunResult r = RunLive::Run(Command::Shell("seq 1 " + std::to_string(20000 + i)), o);
      if (r.code != 0 || r.output.size() < 100000) ++bad;
      o.ask = [](const std::string&, bool) -> std::optional<std::string> { return std::string("y"); };
      RunResult p = RunLive::Run(Command::Shell("seq 1 " + std::to_string(20000 + i)), o);
      if (p.code != 0 || p.output.size() < 100000) ++bad;
    });
  }
  for (auto& t : threads) t.join();
  CHECK_EQ(bad.load(), 0);
}

static void TestStreams() {
  StringSink sink;
  LiveStream ls(2, 20, &sink, true);
  ls.Feed("one\n"); ls.Feed("two\n"); ls.Feed("three\n");
  ls.Close("done");
  CHECK_EQ(sink.str(), std::string("\xe2\x94\x82 one\n\xe2\x94\x82 two\n\033[2A\r\033[K\xe2\x94\x82 two\n\r\033[K\xe2\x94\x82 three\n\xe2\x94\x82 done\n"));
  StringSink empty;
  LiveStream e(4, 20, &empty, true);
  e.Close("exit 0");
  CHECK_EQ(empty.str(), std::string("\xe2\x94\x82 (no output)\n\xe2\x94\x82 exit 0\n"));
  CHECK_EQ(StreamStatus(3, "a\nb", 1.25, false), std::string("exit 3 \xc2\xb7 1.2s \xc2\xb7 2 lines \xc2\xb7 3 B"));
  CHECK_EQ(StreamStatus(0, "", 0.0, true), std::string("timed out after 120s (killed) \xc2\xb7 0 lines \xc2\xb7 0 B"));
}

static void TestGlow() {
  CHECK_EQ(Glow::Ellipsis(std::string(100, 'x'), 60).size(), (size_t)59 + 3);
  CHECK_EQ(Glow::IconFor("shell"), std::string("\xe2\x96\xb6"));
  auto d = Glow::DescribeCall("shell", "{\"command\": \"ls   -la\"}");
  CHECK(d.verb == "Run" && d.target == "ls -la" && d.extra == "shell" && d.preview == "ls -la");
  d = Glow::DescribeCall("store_secret", "{\"name\": \"k\", \"value\": \"s3\"}");
  CHECK(d.preview.find("s3") == std::string::npos);
  d = Glow::DescribeCall("nope", "x y");
  CHECK(d.verb == "nope" && d.preview == "x y");
  std::vector<AgentRow> rows(2);
  rows[0].id = 1; rows[0].status = "running"; rows[0].task = "t";
  rows[1].id = 2; rows[1].status = "done"; rows[1].task = "u"; rows[1].tools = 0;
  auto lanes = Glow::Lanes(rows, 6, 0, "");
  CHECK(lanes.size() == 3 && lanes[0].find("lanes") != std::string::npos);
  // LaneBoard paints twice (start, stop) and degrades to static lines when the row count changes
  std::vector<std::vector<AgentRow>> seq = {{rows[0]}, rows};
  size_t call = 0;
  StringSink out;
  LaneBoard board("p", [&](const std::string&) { return seq[std::min(call++, seq.size() - 1)]; }, 60, 6, &out);
  board.Start(); board.Stop();
  CHECK_EQ(board.painted(), 3);
  CHECK(out.str().find("\033[") != std::string::npos);
}

static void TestSession() {
  fs::path root = fs::temp_directory_path() / "detz_shell_test_proc";
  fs::remove_all(root);
  fs::create_directories(root / "sys/kernel/random");
  std::ofstream(root / "stat") << "btime 1760000000\n";
  std::ofstream(root / "sys/kernel/random/boot_id") << "ABCDEF12-3456\n";
  fs::create_directories(root / "200"); fs::create_directories(root / "100");
  std::ofstream(root / "200/stat") << "200 (bash) S 100 " << "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5000 0 0\n";
  std::ofstream(root / "200/cmdline") << std::string("bash\0--norc\0", 12);
  std::ofstream(root / "100/stat") << "100 (init) S 0 " << "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0\n";
  std::ofstream(root / "100/cmdline") << std::string("/sbin/init\0", 11);
  EnvMap env;
  ShellSession s = ShellSession::Current(&env, 200, root.string(), 1760001000, 100);
  CHECK(s.shell == "bash" && s.pid == 200 && !s.parent && s.sid.find("-p200-abcdef12") != std::string::npos);
  EnvMap pinned; pinned.Set("TAI_SESSION", "shared word");
  ShellSession w = ShellSession::Current(&pinned, 200, root.string(), 1760001000, 100);
  CHECK(w.sid == "sharedword" && w.pid == 200);
  EnvMap nested; nested.Set("TAI_SESSION", "20250101-000000-p999-abcdef12");
  ShellSession n = ShellSession::Current(&nested, 200, root.string(), 1760001000, 100);
  CHECK(n.sid == s.sid && n.parent && *n.parent == "20250101-000000-p999-abcdef12");
  CHECK_EQ(ShellSession::Md5Hex("abc"), std::string("900150983cd24fb0d6963f7d28e17f72"));
  CHECK(ShellSession::ParseStat("1 (a b) S 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 77 0").ticks == 77);
  CHECK(!ShellSession::ParseStat("garbage").ok);
  fs::remove_all(root);
}

static void TestSandbox() {
  CHECK_EQ(CurlHint("curl https://x"), std::string("\n") + kCurlHint);
  CHECK_EQ(CurlHint("mycurl http"), std::string(""));
  CHECK_EQ(Sandbox::RunArgs().size(), (size_t)6);
  CHECK_EQ(Sandbox::UpdateArgs().size(), (size_t)4);
  std::string e = Sandbox::ContainerEngine();
  CHECK(e.empty() || e == "podman" || e == "docker");
  CHECK_EQ(Sandbox::BoxState("/nonexistent/engine"), std::string("missing"));
  CHECK(std::string(BoxData::BoxContainerfile()).find("FROM python:3.12-slim") == 0);
}

int main() {
  TestUtf8(); TestText(); TestAnsi(); TestJson(); TestMarkdown(); TestDiff(); TestShlexGlobTargets();
  TestProcAndRun(); TestBurst(); TestStreams(); TestGlow(); TestSession(); TestSandbox();
  std::printf("%d checks, %d failures\n", gChecks, gFailures);
  return gFailures ? 1 : 0;
}