// Unit tests for the Mplex foundation libraries (mplex_common, mplex_py, mplex_json, mplex_toml). // Every expected value in mplex_foundation_cases.inc was produced by CPython 3.14 through // mplex_foundation_gen.py; this file only replays them against the C++ port. // // Build and run (from taicpp/): // g++ -std=c++23 -Wall -Wextra -Isrc/mplex tests/mplex_foundation_test.cpp src/mplex/mplex_common.cpp // (then list src/mplex/mplex_py.cpp src/mplex/mplex_json.cpp src/mplex/mplex_toml.cpp -o build/f1/foundation_test) // TZ=UTC build/f1/foundation_test #include #include #include #include #include #include #include #include #include #include #include #include "mplex_common.h" #include "mplex_json.h" #include "mplex_py.h" #include "mplex_toml.h" using namespace detz::mplex; using detz::mplex::py::Regex; using detz::mplex::py::RegexError; using detz::mplex::py::regex_escape; namespace { struct Case { std::string suite; std::vector a; }; struct RawCase { std::string_view suite; std::vector a; }; using namespace std::literals; const RawCase kCases[] = { #include "mplex_foundation_cases.inc" }; int g_checked = 0; int g_failed = 0; std::map g_by_suite; std::map g_failed_by_suite; std::string show(const std::string& s) { std::string out; char buf[8]; for (unsigned char c : s) { if (c >= 32 && c < 127) { out.push_back(static_cast(c)); } else { std::snprintf(buf, sizeof buf, "\\x%02x", c); out += buf; } } return out; } void check(const Case& c, const std::string& got, const std::string& want) { ++g_checked; ++g_by_suite[c.suite]; if (got == want) return; ++g_failed; ++g_failed_by_suite[c.suite]; if (g_failed <= 60) { std::cerr << "FAIL " << c.suite << " ["; for (size_t i = 0; i + 1 < c.a.size(); ++i) std::cerr << (i ? " | " : "") << show(c.a[i]); std::cerr << "]\n got: " << show(got) << "\n want: " << show(want) << "\n"; } } void expect(const Case& c, bool cond, const std::string& note) { ++g_checked; ++g_by_suite[c.suite]; if (cond) return; ++g_failed; ++g_failed_by_suite[c.suite]; if (g_failed <= 60) std::cerr << "FAIL " << c.suite << " " << show(c.a.empty() ? "" : c.a[0]) << ": " << note << "\n"; } std::string lst(const std::vector& items) { std::string out = std::to_string(items.size()) + ":"; for (size_t i = 0; i < items.size(); ++i) { if (i) out.push_back('\x1e'); out += items[i]; } return out; } std::string hex_of(const std::string& s) { std::string out; char buf[4]; for (unsigned char c : s) { std::snprintf(buf, sizeof buf, "%02x", c); out += buf; } return out; } std::string unhex(const std::string& h) { std::string out; for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(static_cast(std::stoi(h.substr(i, 2), nullptr, 16))); return out; } std::string span(long s, long e) { return std::to_string(s) + "," + std::to_string(e); } std::string toml_ser(const TomlValue& v) { switch (v.kind) { case TomlValue::Kind::Bool: return v.boolean ? "true" : "false"; case TomlValue::Kind::Int: return v.text.empty() ? std::to_string(v.integer) : v.text; case TomlValue::Kind::Float: if (std::isnan(v.number)) return "nan"; if (std::isinf(v.number)) return v.number < 0 ? "-inf" : "inf"; return py::format_float(v.number); case TomlValue::Kind::String: return Json::string(v.text).dumps(); case TomlValue::Kind::Datetime: return "@" + v.text; case TomlValue::Kind::Array: { std::string out = "["; for (size_t i = 0; i < v.array.size(); ++i) out += (i ? "," : "") + toml_ser(v.array[i]); return out + "]"; } case TomlValue::Kind::Table: { std::string out = "{"; bool first = true; for (const auto& e : v.table) { out += (first ? "" : ",") + Json::string(e.first).dumps() + ":" + toml_ser(e.second); first = false; } return out + "}"; } } return ""; } void run_case(const Case& c) { const std::string suite = c.suite; const auto& a = c.a; if (suite == "lower") check(c, py::lower(a[0]), a[1]); else if (suite == "upper") check(c, py::upper(a[0]), a[1]); else if (suite == "strip") check(c, py::strip(a[0]), a[1]); else if (suite == "rstrip") check(c, py::rstrip(a[0]), a[1]); else if (suite == "split_ws") check(c, lst(py::split_ws(a[0])), a[1]); else if (suite == "splitlines") check(c, lst(py::splitlines(a[0])), a[1]); else if (suite == "len") check(c, std::to_string(py::len(a[0])), a[1]); else if (suite == "repr") check(c, py::repr(a[0]), a[1]); else if (suite == "slice") check(c, py::slice(a[0], std::stol(a[1]), std::stol(a[2])), a[3]); else if (suite == "ljust") check(c, py::ljust(a[0], static_cast(std::stoul(a[1]))), a[2]); else if (suite == "strip_chars") check(c, py::strip(a[0], a[1]), a[2]); else if (suite == "lstrip_chars") check(c, py::lstrip(a[0], a[1]), a[2]); else if (suite == "rstrip_chars") check(c, py::rstrip(a[0], a[1]), a[2]); else if (suite == "split") check(c, lst(py::split(a[0], a[1], std::stoi(a[2]))), a[3]); else if (suite == "rsplit") check(c, lst(py::rsplit(a[0], a[1], std::stoi(a[2]))), a[3]); else if (suite == "replace") check(c, py::replace(a[0], a[1], a[2]), a[3]); else if (suite == "shlex_quote") check(c, py::shlex_quote(a[0]), a[1]); else if (suite == "pred") { wchar_t ch = static_cast(std::stoul(a[0])); std::string flags = std::string(py::isalpha(ch) ? "1" : "0") + (py::isdigit(ch) ? "1" : "0") + (py::isalnum(ch) ? "1" : "0") + (py::isspace(ch) ? "1" : "0") + (py::isprintable(ch) ? "1" : "0"); check(c, flags, a[1]); UStr one(1, ch); check(c, to_utf8(py::lower(one)), a[2]); check(c, to_utf8(py::upper(one)), a[3]); } else if (suite == "int") { bool ok = a[1] == "1"; if (ok) { if (a[2] == "BIG") { expect(c, py::is_int_text(a[0]), "is_int_text"); } else { check(c, py::is_int_text(a[0]) ? std::to_string(py::to_int(a[0])) : "notint", a[2]); } } else { expect(c, !py::is_int_text(a[0]), "is_int_text must be false"); std::string msg = "no throw"; try { py::to_int(a[0]); } catch (const ValueError& e) { msg = e.what(); } check(c, msg, a[2]); } } else if (suite == "float") { if (a[1] == "1") { double v = 0; std::string got; try { v = py::to_float(a[0]); got = std::isnan(v) ? "nan" : py::format_float(v); } catch (const ValueError& e) { got = std::string("throws ") + e.what(); } check(c, got, a[2]); } else { std::string msg = "no throw"; try { py::to_float(a[0]); } catch (const ValueError& e) { msg = e.what(); } check(c, msg, a[2]); } } else if (suite == "format_float") { double v = std::strtod(a[0].c_str(), nullptr); check(c, py::format_float(v), a[1]); } else if (suite == "wrap") { check(c, lst(py::textwrap_wrap(a[0], std::stoi(a[1]))), a[2]); } else if (suite == "wrap_bad") { std::string msg = "no throw"; try { py::textwrap_wrap(a[0], std::stoi(a[1])); } catch (const ValueError& e) { msg = std::string("ValueError: ") + e.what(); } check(c, msg, a[2]); } else if (suite == "fnmatch") check(c, py::fnmatchcase(a[0], a[1]) ? "1" : "0", a[2]); else if (suite == "basename") check(c, py::basename(a[0]), a[1]); else if (suite == "dirname") check(c, py::dirname(a[0]), a[1]); else if (suite == "join_path") check(c, py::join_path(a[0], a[1]), a[2]); else if (suite == "expanduser") check(c, py::expanduser(a[0]), a[1]); else if (suite == "strftime") check(c, py::strftime(a[0], std::stod(a[1])), a[2]); else if (suite == "re_search" || suite == "re_search_from") { try { Regex re(to_ustr(a[0])); size_t from = suite == "re_search" ? 0 : static_cast(std::stoul(a[2])); Regex::Match m = re.search(to_ustr(a[1]), from); check(c, m.found ? span(m.start, m.end) : "none", a.back()); } catch (const RegexError& e) { expect(c, false, std::string("RegexError: ") + e.what()); } } else if (suite == "re_finditer") { try { Regex re(to_ustr(a[0])); std::vector spans; for (const auto& m : re.finditer(to_ustr(a[1]))) spans.push_back(span(m.start, m.end)); check(c, lst(spans), a[2]); } catch (const RegexError& e) { expect(c, false, std::string("RegexError: ") + e.what()); } } else if (suite == "re_sub") { try { Regex re(to_ustr(a[0])); check(c, to_utf8(re.sub(to_ustr(a[2]), to_ustr(a[1]))), a[3]); } catch (const RegexError& e) { expect(c, false, std::string("RegexError: ") + e.what()); } } else if (suite == "re_sub_err") { bool threw = false; try { Regex re(to_ustr(a[0])); re.sub(to_ustr(a[2]), to_ustr(a[1])); } catch (const RegexError&) { threw = true; } expect(c, threw, "expected RegexError"); } else if (suite == "re_err") { bool threw = false; try { Regex re(to_ustr(a[0])); } catch (const RegexError&) { threw = true; } expect(c, threw, "expected RegexError"); } else if (suite == "re_unsupported_ok") { try { Regex re(to_ustr(a[0])); re.search(L"abc"); } catch (const RegexError&) { } expect(c, true, ""); } else if (suite == "re_ic") { Regex re(to_ustr(a[0]), true); Regex::Match m = re.search(to_ustr(a[1])); check(c, m.found ? span(m.start, m.end) : "none", a[2]); } else if (suite == "re_escape") check(c, to_utf8(regex_escape(to_ustr(a[0]))), a[1]); else if (suite == "split_commas") check(c, lst(py::split_commas_ws(a[0])), a[1]); else if (suite == "decode") check(c, hex_of(to_utf8(to_ustr(unhex(a[0])))), a[1]); else if (suite == "json_ok") { try { check(c, Json::loads(a[0]).dumps(), a[1]); } catch (const JsonError& e) { expect(c, false, std::string("JsonError: ") + e.what()); } } else if (suite == "json_err") { std::string msg = "no throw"; try { Json::loads(a[0]); } catch (const JsonError& e) { msg = e.what(); } check(c, msg, a[1]); } else if (suite == "json_bad_ok") { expect(c, false, "case generator produced an accepted document in the bad list"); } else if (suite == "json_eq") { check(c, Json::loads(a[0]).equals(Json::loads(a[1])) ? "1" : "0", a[2]); } else if (suite == "json_truthy") { check(c, Json::loads(a[0]).truthy() ? "1" : "0", a[1]); } else if (suite == "toml_ok") { try { check(c, toml_ser(TomlValue::loads(a[0])), a[1]); } catch (const TomlError& e) { expect(c, false, std::string("TomlError: ") + e.what()); } } else if (suite == "toml_err") { std::string msg = "no throw"; try { TomlValue::loads(a[0]); } catch (const TomlError& e) { msg = e.what(); } check(c, msg, a[1]); } else { std::cerr << "unknown suite " << suite << "\n"; ++g_failed; } } // ---- hand written checks that need no CPython reference ------------------------------------- void hand_tests() { Case c{"hand", {"colors"}}; for (const char* frag : {"0", "1", "30", "37", "40", "47", "90", "97", "100", "107", "38;5;0", "38;5;255", "48;5;0", "48;5;12", "38;2;0;0;0", "38;2;255;255;255", "48;2;255;128;7", "48;2;1;2;3"}) { Color col = color_from_sgr(frag); expect(c, col != 0, std::string("nonzero for ") + frag); check(c, color_sgr(col), frag); } check(c, color_sgr(color_from_sgr("")), ""); expect(c, color_from_sgr("") == 0, "empty is None"); std::set seen; for (const char* frag : {"31", "41", "38;5;31", "48;5;31", "38;2;0;0;31", "48;2;0;0;31", "0"}) seen.insert(color_from_sgr(frag)); expect(c, seen.size() == 7, "packings are distinct"); expect(c, color_from_sgr("031") == 0 && color_from_sgr("38;5;256") == 0 && color_from_sgr("38;9") == 0, "non canonical fragments map to None"); c = Case{"hand", {"short_error"}}; check(c, short_error(std::runtime_error("boom")), "boom"); check(c, short_error(std::runtime_error(""), "KeyError"), "KeyError"); check(c, std::to_string(py::len(short_error(std::runtime_error(std::string(400, 'x'))))), "300"); check(c, std::to_string(py::len(short_error(std::runtime_error(std::string(250, '\xc3') + std::string(250, '\xa9'))))), "300"); c = Case{"hand", {"ordered_map"}}; OrderedMap om; om.set(3, "c"); om.set(1, "a"); om.set(3, "C"); check(c, std::to_string(om.keys()[0]) + std::to_string(om.keys()[1]), "31"); check(c, *om.find(3), "C"); expect(c, om.erase(1) && !om.erase(1) && om.size() == 1, "erase"); c = Case{"hand", {"json_api"}}; Json o = Json::object(); o.set("b", Json::integer(1)); o.set("a", Json::string("\xc3\xa9\xf0\x9f\x98\x80")); o.set("b", Json::boolean(true)); check(c, o.dumps(), "{\"b\": true, \"a\": \"\\u00e9\\ud83d\\ude00\"}"); check(c, Json::array({Json::null(), Json::number(1.0), Json::number(std::nan(""))}).dumps(), "[null, 1.0, NaN]"); check(c, Json::loads("123456789012345678901234567890").dumps(), "123456789012345678901234567890"); expect(c, Json::loads("123456789012345678901234567890").is_big_int(), "big int flag"); expect(c, Json::loads("5").is_int() && !Json::loads("5").is_big_int(), "small int"); c = Case{"hand", {"subprocess"}}; py::RunResult r = py::run_shell("echo hi; echo err 1>&2; exit 3", "", true, true, 5.0); check(c, r.out + "|" + r.err + "|" + std::to_string(r.returncode), "hi\n|err\n|3"); r = py::run_shell("cat", "abc", true, false, 5.0, true); check(c, r.out, "abc"); std::string big(3 << 20, 'z'); r = py::run_shell("cat", big, true, false, 20.0, true); expect(c, r.out == big && !r.timed_out, "3 MiB through cat without deadlock"); auto t0 = std::chrono::steady_clock::now(); r = py::run_shell("sleep 5", "", true, true, 0.3); double took = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); expect(c, r.timed_out && took < 2.0, "timeout kills the shell, took " + std::to_string(took)); r = py::run_shell("kill -9 $$", "", true, true, 5.0); check(c, std::to_string(r.returncode), "-9"); r = py::run_shell("echo out; echo err 1>&2", "", false, false, 5.0); expect(c, r.out.empty() && r.err.empty() && r.returncode == 0, "devnull output"); r = py::run_argv({"printf", "%s", "x y"}, true, true, 5.0); check(c, r.out, "x y"); bool threw = false; try { py::run_argv({"/nonexistent/binary"}, true, true, 5.0); } catch (const py::OsError& e) { threw = std::string(e.what()).find("No such file or directory") != std::string::npos; } expect(c, threw, "missing binary raises OsError"); int leak[2]; if (pipe(leak) == 0) { r = py::run_shell("ls /proc/self/fd | wc -l", "", true, true, 5.0); int n = std::atoi(r.out.c_str()); expect(c, n <= 5, "child inherited descriptors: " + std::to_string(n)); close(leak[0]); close(leak[1]); } c = Case{"hand", {"time"}}; double m0 = py::monotonic(); expect(c, py::monotonic() >= m0 && py::now() > 1.6e9, "clocks"); check(c, py::strftime("%Y", 0.0), "1970"); expect(c, !py::gethostname().empty(), "hostname"); } } // namespace int main() { setenv("TZ", "UTC", 1); tzset(); setenv("HOME", "/home/tester/", 1); for (const RawCase& raw : kCases) { Case c{std::string(raw.suite), {}}; for (std::string_view v : raw.a) c.a.emplace_back(v); run_case(c); } hand_tests(); std::printf("checked %d, failed %d\n", g_checked, g_failed); for (const auto& e : g_by_suite) std::printf(" %-18s %5d checks, %d failed\n", e.first.c_str(), e.second, g_failed_by_suite[e.first]); return g_failed == 0 ? 0 : 1; }