tai/taicpp/tests/mplex_foundation_test.cpp

// Unit tests for the Mplex foundation libraries (mplex_common, mplex_py, mplex_json, mplex_toml).
// Every expected value in mplex_foundation_cases.inc was produced by CPython 3.14 through
// mplex_foundation_gen.py; this file only replays them against the C++ port.
//
// Build and run (from taicpp/):
//   g++ -std=c++23 -Wall -Wextra -Isrc/mplex tests/mplex_foundation_test.cpp src/mplex/mplex_common.cpp
//   (then list src/mplex/mplex_py.cpp src/mplex/mplex_json.cpp src/mplex/mplex_toml.cpp -o build/f1/foundation_test)
//   TZ=UTC build/f1/foundation_test
#include <unistd.h>

#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>

#include "mplex_common.h"
#include "mplex_json.h"
#include "mplex_py.h"
#include "mplex_toml.h"

using namespace detz::mplex;
using detz::mplex::py::Regex;
using detz::mplex::py::RegexError;
using detz::mplex::py::regex_escape;

namespace {

struct Case {
  std::string suite;
  std::vector<std::string> a;
};

struct RawCase {
  std::string_view suite;
  std::vector<std::string_view> a;
};

using namespace std::literals;

const RawCase kCases[] = {
#include "mplex_foundation_cases.inc"
};

int g_checked = 0;
int g_failed = 0;
std::map<std::string, int> g_by_suite;
std::map<std::string, int> g_failed_by_suite;

std::string show(const std::string& s) {
  std::string out;
  char buf[8];
  for (unsigned char c : s) {
    if (c >= 32 && c < 127) {
      out.push_back(static_cast<char>(c));
    } else {
      std::snprintf(buf, sizeof buf, "\\x%02x", c);
      out += buf;
    }
  }
  return out;
}

void check(const Case& c, const std::string& got, const std::string& want) {
  ++g_checked;
  ++g_by_suite[c.suite];
  if (got == want) return;
  ++g_failed;
  ++g_failed_by_suite[c.suite];
  if (g_failed <= 60) {
    std::cerr << "FAIL " << c.suite << " [";
    for (size_t i = 0; i + 1 < c.a.size(); ++i) std::cerr << (i ? " | " : "") << show(c.a[i]);
    std::cerr << "]\n   got:  " << show(got) << "\n   want: " << show(want) << "\n";
  }
}

void expect(const Case& c, bool cond, const std::string& note) {
  ++g_checked;
  ++g_by_suite[c.suite];
  if (cond) return;
  ++g_failed;
  ++g_failed_by_suite[c.suite];
  if (g_failed <= 60) std::cerr << "FAIL " << c.suite << " " << show(c.a.empty() ? "" : c.a[0]) << ": " << note << "\n";
}

std::string lst(const std::vector<std::string>& items) {
  std::string out = std::to_string(items.size()) + ":";
  for (size_t i = 0; i < items.size(); ++i) {
    if (i) out.push_back('\x1e');
    out += items[i];
  }
  return out;
}

std::string hex_of(const std::string& s) {
  std::string out;
  char buf[4];
  for (unsigned char c : s) {
    std::snprintf(buf, sizeof buf, "%02x", c);
    out += buf;
  }
  return out;
}
std::string unhex(const std::string& h) {
  std::string out;
  for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(static_cast<char>(std::stoi(h.substr(i, 2), nullptr, 16)));
  return out;
}

std::string span(long s, long e) { return std::to_string(s) + "," + std::to_string(e); }

std::string toml_ser(const TomlValue& v) {
  switch (v.kind) {
    case TomlValue::Kind::Bool: return v.boolean ? "true" : "false";
    case TomlValue::Kind::Int: return v.text.empty() ? std::to_string(v.integer) : v.text;
    case TomlValue::Kind::Float:
      if (std::isnan(v.number)) return "nan";
      if (std::isinf(v.number)) return v.number < 0 ? "-inf" : "inf";
      return py::format_float(v.number);
    case TomlValue::Kind::String: return Json::string(v.text).dumps();
    case TomlValue::Kind::Datetime: return "@" + v.text;
    case TomlValue::Kind::Array: {
      std::string out = "[";
      for (size_t i = 0; i < v.array.size(); ++i) out += (i ? "," : "") + toml_ser(v.array[i]);
      return out + "]";
    }
    case TomlValue::Kind::Table: {
      std::string out = "{";
      bool first = true;
      for (const auto& e : v.table) {
        out += (first ? "" : ",") + Json::string(e.first).dumps() + ":" + toml_ser(e.second);
        first = false;
      }
      return out + "}";
    }
  }
  return "";
}

void run_case(const Case& c) {
  const std::string suite = c.suite;
  const auto& a = c.a;
  if (suite == "lower") check(c, py::lower(a[0]), a[1]);
  else if (suite == "upper") check(c, py::upper(a[0]), a[1]);
  else if (suite == "strip") check(c, py::strip(a[0]), a[1]);
  else if (suite == "rstrip") check(c, py::rstrip(a[0]), a[1]);
  else if (suite == "split_ws") check(c, lst(py::split_ws(a[0])), a[1]);
  else if (suite == "splitlines") check(c, lst(py::splitlines(a[0])), a[1]);
  else if (suite == "len") check(c, std::to_string(py::len(a[0])), a[1]);
  else if (suite == "repr") check(c, py::repr(a[0]), a[1]);
  else if (suite == "slice") check(c, py::slice(a[0], std::stol(a[1]), std::stol(a[2])), a[3]);
  else if (suite == "ljust") check(c, py::ljust(a[0], static_cast<size_t>(std::stoul(a[1]))), a[2]);
  else if (suite == "strip_chars") check(c, py::strip(a[0], a[1]), a[2]);
  else if (suite == "lstrip_chars") check(c, py::lstrip(a[0], a[1]), a[2]);
  else if (suite == "rstrip_chars") check(c, py::rstrip(a[0], a[1]), a[2]);
  else if (suite == "split") check(c, lst(py::split(a[0], a[1], std::stoi(a[2]))), a[3]);
  else if (suite == "rsplit") check(c, lst(py::rsplit(a[0], a[1], std::stoi(a[2]))), a[3]);
  else if (suite == "replace") check(c, py::replace(a[0], a[1], a[2]), a[3]);
  else if (suite == "shlex_quote") check(c, py::shlex_quote(a[0]), a[1]);
  else if (suite == "pred") {
    wchar_t ch = static_cast<wchar_t>(std::stoul(a[0]));
    std::string flags = std::string(py::isalpha(ch) ? "1" : "0") + (py::isdigit(ch) ? "1" : "0") + (py::isalnum(ch) ? "1" : "0") +
                        (py::isspace(ch) ? "1" : "0") + (py::isprintable(ch) ? "1" : "0");
    check(c, flags, a[1]);
    UStr one(1, ch);
    check(c, to_utf8(py::lower(one)), a[2]);
    check(c, to_utf8(py::upper(one)), a[3]);
  } else if (suite == "int") {
    bool ok = a[1] == "1";
    if (ok) {
      if (a[2] == "BIG") {
        expect(c, py::is_int_text(a[0]), "is_int_text");
      } else {
        check(c, py::is_int_text(a[0]) ? std::to_string(py::to_int(a[0])) : "notint", a[2]);
      }
    } else {
      expect(c, !py::is_int_text(a[0]), "is_int_text must be false");
      std::string msg = "no throw";
      try {
        py::to_int(a[0]);
      } catch (const ValueError& e) {
        msg = e.what();
      }
      check(c, msg, a[2]);
    }
  } else if (suite == "float") {
    if (a[1] == "1") {
      double v = 0;
      std::string got;
      try {
        v = py::to_float(a[0]);
        got = std::isnan(v) ? "nan" : py::format_float(v);
      } catch (const ValueError& e) {
        got = std::string("throws ") + e.what();
      }
      check(c, got, a[2]);
    } else {
      std::string msg = "no throw";
      try {
        py::to_float(a[0]);
      } catch (const ValueError& e) {
        msg = e.what();
      }
      check(c, msg, a[2]);
    }
  } else if (suite == "format_float") {
    double v = std::strtod(a[0].c_str(), nullptr);
    check(c, py::format_float(v), a[1]);
  } else if (suite == "wrap") {
    check(c, lst(py::textwrap_wrap(a[0], std::stoi(a[1]))), a[2]);
  } else if (suite == "wrap_bad") {
    std::string msg = "no throw";
    try {
      py::textwrap_wrap(a[0], std::stoi(a[1]));
    } catch (const ValueError& e) {
      msg = std::string("ValueError: ") + e.what();
    }
    check(c, msg, a[2]);
  } else if (suite == "fnmatch") check(c, py::fnmatchcase(a[0], a[1]) ? "1" : "0", a[2]);
  else if (suite == "basename") check(c, py::basename(a[0]), a[1]);
  else if (suite == "dirname") check(c, py::dirname(a[0]), a[1]);
  else if (suite == "join_path") check(c, py::join_path(a[0], a[1]), a[2]);
  else if (suite == "expanduser") check(c, py::expanduser(a[0]), a[1]);
  else if (suite == "strftime") check(c, py::strftime(a[0], std::stod(a[1])), a[2]);
  else if (suite == "re_search" || suite == "re_search_from") {
    try {
      Regex re(to_ustr(a[0]));
      size_t from = suite == "re_search" ? 0 : static_cast<size_t>(std::stoul(a[2]));
      Regex::Match m = re.search(to_ustr(a[1]), from);
      check(c, m.found ? span(m.start, m.end) : "none", a.back());
    } catch (const RegexError& e) {
      expect(c, false, std::string("RegexError: ") + e.what());
    }
  } else if (suite == "re_finditer") {
    try {
      Regex re(to_ustr(a[0]));
      std::vector<std::string> spans;
      for (const auto& m : re.finditer(to_ustr(a[1]))) spans.push_back(span(m.start, m.end));
      check(c, lst(spans), a[2]);
    } catch (const RegexError& e) {
      expect(c, false, std::string("RegexError: ") + e.what());
    }
  } else if (suite == "re_sub") {
    try {
      Regex re(to_ustr(a[0]));
      check(c, to_utf8(re.sub(to_ustr(a[2]), to_ustr(a[1]))), a[3]);
    } catch (const RegexError& e) {
      expect(c, false, std::string("RegexError: ") + e.what());
    }
  } else if (suite == "re_sub_err") {
    bool threw = false;
    try {
      Regex re(to_ustr(a[0]));
      re.sub(to_ustr(a[2]), to_ustr(a[1]));
    } catch (const RegexError&) {
      threw = true;
    }
    expect(c, threw, "expected RegexError");
  } else if (suite == "re_err") {
    bool threw = false;
    try {
      Regex re(to_ustr(a[0]));
    } catch (const RegexError&) {
      threw = true;
    }
    expect(c, threw, "expected RegexError");
  } else if (suite == "re_unsupported_ok") {
    try {
      Regex re(to_ustr(a[0]));
      re.search(L"abc");
    } catch (const RegexError&) {
    }
    expect(c, true, "");
  } else if (suite == "re_ic") {
    Regex re(to_ustr(a[0]), true);
    Regex::Match m = re.search(to_ustr(a[1]));
    check(c, m.found ? span(m.start, m.end) : "none", a[2]);
  } else if (suite == "re_escape") check(c, to_utf8(regex_escape(to_ustr(a[0]))), a[1]);
  else if (suite == "split_commas") check(c, lst(py::split_commas_ws(a[0])), a[1]);
  else if (suite == "decode") check(c, hex_of(to_utf8(to_ustr(unhex(a[0])))), a[1]);
  else if (suite == "json_ok") {
    try {
      check(c, Json::loads(a[0]).dumps(), a[1]);
    } catch (const JsonError& e) {
      expect(c, false, std::string("JsonError: ") + e.what());
    }
  } else if (suite == "json_err") {
    std::string msg = "no throw";
    try {
      Json::loads(a[0]);
    } catch (const JsonError& e) {
      msg = e.what();
    }
    check(c, msg, a[1]);
  } else if (suite == "json_bad_ok") {
    expect(c, false, "case generator produced an accepted document in the bad list");
  } else if (suite == "json_eq") {
    check(c, Json::loads(a[0]).equals(Json::loads(a[1])) ? "1" : "0", a[2]);
  } else if (suite == "json_truthy") {
    check(c, Json::loads(a[0]).truthy() ? "1" : "0", a[1]);
  } else if (suite == "toml_ok") {
    try {
      check(c, toml_ser(TomlValue::loads(a[0])), a[1]);
    } catch (const TomlError& e) {
      expect(c, false, std::string("TomlError: ") + e.what());
    }
  } else if (suite == "toml_err") {
    std::string msg = "no throw";
    try {
      TomlValue::loads(a[0]);
    } catch (const TomlError& e) {
      msg = e.what();
    }
    check(c, msg, a[1]);
  } else {
    std::cerr << "unknown suite " << suite << "\n";
    ++g_failed;
  }
}

// ---- hand written checks that need no CPython reference -------------------------------------

void hand_tests() {
  Case c{"hand", {"colors"}};
  for (const char* frag : {"0", "1", "30", "37", "40", "47", "90", "97", "100", "107", "38;5;0", "38;5;255", "48;5;0", "48;5;12", "38;2;0;0;0",
                           "38;2;255;255;255", "48;2;255;128;7", "48;2;1;2;3"}) {
    Color col = color_from_sgr(frag);
    expect(c, col != 0, std::string("nonzero for ") + frag);
    check(c, color_sgr(col), frag);
  }
  check(c, color_sgr(color_from_sgr("")), "");
  expect(c, color_from_sgr("") == 0, "empty is None");
  std::set<Color> seen;
  for (const char* frag : {"31", "41", "38;5;31", "48;5;31", "38;2;0;0;31", "48;2;0;0;31", "0"}) seen.insert(color_from_sgr(frag));
  expect(c, seen.size() == 7, "packings are distinct");
  expect(c, color_from_sgr("031") == 0 && color_from_sgr("38;5;256") == 0 && color_from_sgr("38;9") == 0, "non canonical fragments map to None");

  c = Case{"hand", {"short_error"}};
  check(c, short_error(std::runtime_error("boom")), "boom");
  check(c, short_error(std::runtime_error(""), "KeyError"), "KeyError");
  check(c, std::to_string(py::len(short_error(std::runtime_error(std::string(400, 'x'))))), "300");
  check(c, std::to_string(py::len(short_error(std::runtime_error(std::string(250, '\xc3') + std::string(250, '\xa9'))))), "300");

  c = Case{"hand", {"ordered_map"}};
  OrderedMap<int, std::string> om;
  om.set(3, "c");
  om.set(1, "a");
  om.set(3, "C");
  check(c, std::to_string(om.keys()[0]) + std::to_string(om.keys()[1]), "31");
  check(c, *om.find(3), "C");
  expect(c, om.erase(1) && !om.erase(1) && om.size() == 1, "erase");

  c = Case{"hand", {"json_api"}};
  Json o = Json::object();
  o.set("b", Json::integer(1));
  o.set("a", Json::string("\xc3\xa9\xf0\x9f\x98\x80"));
  o.set("b", Json::boolean(true));
  check(c, o.dumps(), "{\"b\": true, \"a\": \"\\u00e9\\ud83d\\ude00\"}");
  check(c, Json::array({Json::null(), Json::number(1.0), Json::number(std::nan(""))}).dumps(), "[null, 1.0, NaN]");
  check(c, Json::loads("123456789012345678901234567890").dumps(), "123456789012345678901234567890");
  expect(c, Json::loads("123456789012345678901234567890").is_big_int(), "big int flag");
  expect(c, Json::loads("5").is_int() && !Json::loads("5").is_big_int(), "small int");

  c = Case{"hand", {"subprocess"}};
  py::RunResult r = py::run_shell("echo hi; echo err 1>&2; exit 3", "", true, true, 5.0);
  check(c, r.out + "|" + r.err + "|" + std::to_string(r.returncode), "hi\n|err\n|3");
  r = py::run_shell("cat", "abc", true, false, 5.0, true);
  check(c, r.out, "abc");
  std::string big(3 << 20, 'z');
  r = py::run_shell("cat", big, true, false, 20.0, true);
  expect(c, r.out == big && !r.timed_out, "3 MiB through cat without deadlock");
  auto t0 = std::chrono::steady_clock::now();
  r = py::run_shell("sleep 5", "", true, true, 0.3);
  double took = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
  expect(c, r.timed_out && took < 2.0, "timeout kills the shell, took " + std::to_string(took));
  r = py::run_shell("kill -9 $$", "", true, true, 5.0);
  check(c, std::to_string(r.returncode), "-9");
  r = py::run_shell("echo out; echo err 1>&2", "", false, false, 5.0);
  expect(c, r.out.empty() && r.err.empty() && r.returncode == 0, "devnull output");
  r = py::run_argv({"printf", "%s", "x y"}, true, true, 5.0);
  check(c, r.out, "x y");
  bool threw = false;
  try {
    py::run_argv({"/nonexistent/binary"}, true, true, 5.0);
  } catch (const py::OsError& e) {
    threw = std::string(e.what()).find("No such file or directory") != std::string::npos;
  }
  expect(c, threw, "missing binary raises OsError");
  int leak[2];
  if (pipe(leak) == 0) {
    r = py::run_shell("ls /proc/self/fd | wc -l", "", true, true, 5.0);
    int n = std::atoi(r.out.c_str());
    expect(c, n <= 5, "child inherited descriptors: " + std::to_string(n));
    close(leak[0]);
    close(leak[1]);
  }

  c = Case{"hand", {"time"}};
  double m0 = py::monotonic();
  expect(c, py::monotonic() >= m0 && py::now() > 1.6e9, "clocks");
  check(c, py::strftime("%Y", 0.0), "1970");
  expect(c, !py::gethostname().empty(), "hostname");
}

}  // namespace

int main() {
  setenv("TZ", "UTC", 1);
  tzset();
  setenv("HOME", "/home/tester/", 1);
  for (const RawCase& raw : kCases) {
    Case c{std::string(raw.suite), {}};
    for (std::string_view v : raw.a) c.a.emplace_back(v);
    run_case(c);
  }
  hand_tests();
  std::printf("checked %d, failed %d\n", g_checked, g_failed);
  for (const auto& e : g_by_suite) std::printf("  %-18s %5d checks, %d failed\n", e.first.c_str(), e.second, g_failed_by_suite[e.first]);
  return g_failed == 0 ? 0 : 1;
}