From 55b879718bbd8aae4b17464ab4412f1e60b3b56c Mon Sep 17 00:00:00 2001 From: retoor Date: Sun, 25 Oct 2026 01:21:03 +0200 Subject: [PATCH] taicpp parity: differential harness, ledger-generated PARITY.md with verified status, test_tai.py coverage map, seed cases Co-Authored-By: Claude Sonnet 5.5 --- taicpp/PARITY.md | 305 ++++++ taicpp/docs/parity.md | 126 +++ taicpp/docs/wiring/verify.md | 24 + taicpp/tests/diff/cases/cli_generated.json | 1137 ++++++++++++++++++++ taicpp/tests/diff/cases/repl.json | 46 + taicpp/tools/parity.sh | 21 + taicpp/tools/parity_diff.py | 602 +++++++++++ taicpp/tools/parity_gen_cases.py | 127 +++ taicpp/tools/parity_ledger.py | 367 +++++-- taicpp/tools/parity_tests.py | 115 ++ 10 files changed, 2793 insertions(+), 77 deletions(-) create mode 100644 taicpp/PARITY.md create mode 100644 taicpp/docs/parity.md create mode 100644 taicpp/docs/wiring/verify.md create mode 100644 taicpp/tests/diff/cases/cli_generated.json create mode 100644 taicpp/tests/diff/cases/repl.json create mode 100755 taicpp/tools/parity.sh create mode 100755 taicpp/tools/parity_diff.py create mode 100755 taicpp/tools/parity_gen_cases.py mode change 100644 => 100755 taicpp/tools/parity_ledger.py create mode 100755 taicpp/tools/parity_tests.py diff --git a/taicpp/PARITY.md b/taicpp/PARITY.md new file mode 100644 index 0000000..7f2a402 --- /dev/null +++ b/taicpp/PARITY.md @@ -0,0 +1,305 @@ +# Parity detz vs tai.py (generated, do not edit) + +Generated by `tools/parity_ledger.py` from the symbol ledger and the last differential run. +Only a passing differential case (`tools/parity_diff.py`) makes a symbol `verified`; a facet +after the symbol says what that case proved (for example `parse` means argument parsing only). +`name-only` means an identifier matches in the C++ tree and NOTHING about behavior is known. + +- tai.py sha256: `7e817bba8d48f4ab` +- detz sha256: `a7a705c9b8bd5ed0` +- differential run: 2026-10-25T01:20:14 (tai.py `7e817bba8d48f4ab`, detz `a7a705c9b8bd5ed0`) + +## Summary + +| kind | total | verified | name-only | missing | +|---|---|---|---|---| +| class | 264 | 0 | 28 | 236 | +| method | 2140 | 0 | 109 | 2031 | +| function | 413 | 0 | 25 | 388 | +| tool | 61 | 0 | 61 | 0 | +| flag | 45 | 3 | 42 | 0 | +| setting | 28 | 0 | 13 | 15 | +| command | 31 | 0 | 24 | 7 | +| **all** | 2982 | 3 | 302 | 2677 | + +## Differential cases + +Counts: fail=54, pass=9, xfail=1 + +| case | status | reason | +|---|---|---| +| boot.vault | fail | vault differs (44 diff lines) | +| cli.cap_manifest | fail | stdout differs (61 diff lines) | +| cli.err.ambiguous.cap | fail | stderr differs (18 diff lines) | +| cli.err.ambiguous.debug | fail | stderr differs (18 diff lines) | +| cli.err.ambiguous.install | fail | stderr differs (18 diff lines) | +| cli.err.ambiguous.tui | fail | stderr differs (18 diff lines) | +| cli.err.auto.explicit | fail | stderr differs (18 diff lines) | +| cli.err.cap-cwd.missing | fail | stdout differs (12 diff lines) | +| cli.err.cap-manifest.explicit | fail | stderr differs (18 diff lines) | +| cli.err.cap-max-turns.int | fail | stderr differs (19 diff lines) | +| cli.err.cap-max-turns.missing | fail | stdout differs (12 diff lines) | +| cli.err.cap-model.missing | fail | stdout differs (12 diff lines) | +| cli.err.cap-permission-mode.choice | fail | stderr differs (18 diff lines) | +| cli.err.cap-permission-mode.missing | fail | stdout differs (12 diff lines) | +| cli.err.cap-speak.explicit | fail | stderr differs (18 diff lines) | +| cli.err.continue.explicit | fail | stderr differs (18 diff lines) | +| cli.err.debug-format.choice | fail | stderr differs (18 diff lines) | +| cli.err.debug-format.missing | fail | stderr differs (17 diff lines) | +| cli.err.debug.choice | fail | stderr differs (18 diff lines) | +| cli.err.demo-shot.explicit | fail | stderr differs (18 diff lines) | +| cli.err.demo.explicit | fail | stderr differs (18 diff lines) | +| cli.err.env.missing | fail | stdout differs (12 diff lines) | +| cli.err.error_before_help | fail | stderr differs (18 diff lines) | +| cli.err.eval-retrieval.explicit | fail | stderr differs (18 diff lines) | +| cli.err.install-telegram.explicit | fail | stderr differs (18 diff lines) | +| cli.err.install.explicit | fail | stderr differs (18 diff lines) | +| cli.err.mcp-bind.missing | fail | stdout differs (12 diff lines) | +| cli.err.mcp-port.int | fail | stderr differs (19 diff lines) | +| cli.err.mcp-port.missing | fail | stdout differs (12 diff lines) | +| cli.err.no-opencode.explicit | fail | stderr differs (18 diff lines) | +| cli.err.plexer-wrap.explicit | fail | stderr differs (18 diff lines) | +| cli.err.profile.missing | fail | stdout differs (12 diff lines) | +| cli.err.scheduler.explicit | fail | stderr differs (18 diff lines) | +| cli.err.serve-mcp.missing | fail | stdout differs (12 diff lines) | +| cli.err.stats-rag.explicit | fail | stderr differs (18 diff lines) | +| cli.err.telegram.explicit | fail | stderr differs (18 diff lines) | +| cli.err.theme.missing | fail | stdout differs (12 diff lines) | +| cli.err.tui-local.explicit | fail | stderr differs (18 diff lines) | +| cli.err.tui.explicit | fail | stderr differs (18 diff lines) | +| cli.err.uninstall-telegram.explicit | fail | stderr differs (18 diff lines) | +| cli.err.unknown.long | fail | stderr differs (17 diff lines) | +| cli.err.unknown.short | fail | stderr differs (17 diff lines) | +| cli.err.update-models.explicit | fail | stderr differs (18 diff lines) | +| cli.err.version.explicit | fail | stderr differs (18 diff lines) | +| cli.err.yes.explicit | fail | stderr differs (18 diff lines) | +| cli.err.yolo.explicit | fail | stderr differs (18 diff lines) | +| cli.help.wins_over_error | fail | stdout differs (87 diff lines) | +| cli.prefix.version | fail | stdout differs (5 diff lines) | +| persist.restart_history | xfail | needs the foundation agent: vault events persistence and history resume across process restarts; detz has no REPL/agent | +| repl.help | fail | stdout differs (123 diff lines) | +| repl.profiles | fail | stdout differs (94 diff lines) | +| repl.skills_secret_schedules | fail | stdout differs (103 diff lines) | +| repl.tools | fail | stdout differs (139 diff lines) | +| repl.tools_named | fail | stdout differs (92 diff lines) | +| repl.unknown_command | fail | stdout differs (89 diff lines) | + +## Python test-suite coverage + +150 test classes, 1233 tests; tests reproduced by a passing diff case: 0 (in 0 classes); classes with no covered test: 150. +Full table: `build/parity/tests-map.md`. + +## flags (45) + +| name | status | facets | cases | +|---|---|---|---| +| `--allow-tool` | name-only | | | +| `--argv` | name-only | | | +| `--auto` | name-only | | | +| `--auto-answer` | name-only | | | +| `--binary` | name-only | | | +| `--cap` | name-only | | | +| `--cap-cwd` | name-only | | | +| `--cap-manifest` | name-only | | | +| `--cap-max-turns` | name-only | | | +| `--cap-model` | name-only | | | +| `--cap-permission-mode` | verified | parse | cli.version.with_choice,cli.ok.cap-permission-mode.none,cli.ok.cap-permission-mode.confirm | +| `--cap-speak` | name-only | | | +| `--continue` | name-only | | | +| `--cwd` | name-only | | | +| `--debug` | name-only | | | +| `--debug-format` | verified | parse | cli.ok.debug-format.sentence,cli.ok.debug-format.json | +| `--demo` | name-only | | | +| `--demo-shot` | name-only | | | +| `--deny-tool` | name-only | | | +| `--driver` | name-only | | | +| `--env` | name-only | | | +| `--eval-retrieval` | name-only | | | +| `--install` | name-only | | | +| `--install-telegram` | name-only | | | +| `--max-risk` | name-only | | | +| `--mcp-bind` | name-only | | | +| `--mcp-port` | name-only | | | +| `--model` | name-only | | | +| `--mplex` | name-only | | | +| `--no-opencode` | name-only | | | +| `--plexer-wrap` | name-only | | | +| `--profile` | name-only | | | +| `--scheduler` | name-only | | | +| `--serve-mcp` | name-only | | | +| `--stats-rag` | name-only | | | +| `--task` | name-only | | | +| `--telegram` | name-only | | | +| `--theme` | name-only | | | +| `--tui` | name-only | | | +| `--tui-local` | name-only | | | +| `--uninstall-telegram` | name-only | | | +| `--update-models` | name-only | | | +| `--version` | verified | output | cli.version.long | +| `--yes` | name-only | | | +| `--yolo` | name-only | | | + +## tools (61) + +| name | status | facets | cases | +|---|---|---|---| +| `ask_user` | name-only | | | +| `audit` | name-only | | | +| `create_bot` | name-only | | | +| `create_skill` | name-only | | | +| `delete_file` | name-only | | | +| `delete_secret` | name-only | | | +| `describe_image` | name-only | | | +| `edit_file` | name-only | | | +| `export_terminal_log` | name-only | | | +| `fork` | name-only | | | +| `get_current_terminal_content` | name-only | | | +| `graph_link` | name-only | | | +| `graph_query` | name-only | | | +| `install` | name-only | | | +| `install_skill` | name-only | | | +| `list_secrets` | name-only | | | +| `list_terminals` | name-only | | | +| `listen` | name-only | | | +| `load_skill` | name-only | | | +| `mcp_add` | name-only | | | +| `mcp_call` | name-only | | | +| `mcp_disable` | name-only | | | +| `mcp_enable` | name-only | | | +| `mcp_list` | name-only | | | +| `mcp_login` | name-only | | | +| `mcp_remove` | name-only | | | +| `mcp_tools` | name-only | | | +| `plexer_read` | name-only | | | +| `plexer_send` | name-only | | | +| `plexer_show` | name-only | | | +| `plexer_status` | name-only | | | +| `plexer_type` | name-only | | | +| `poll` | name-only | | | +| `read_file` | name-only | | | +| `recall` | name-only | | | +| `record_delete` | name-only | | | +| `record_read` | name-only | | | +| `record_save` | name-only | | | +| `record_search` | name-only | | | +| `release` | name-only | | | +| `remember` | name-only | | | +| `restore` | name-only | | | +| `schedule` | name-only | | | +| `schedules` | name-only | | | +| `search` | name-only | | | +| `shell` | name-only | | | +| `speak` | name-only | | | +| `store_secret` | name-only | | | +| `sysinfo` | name-only | | | +| `tags` | name-only | | | +| `tail_terminals` | name-only | | | +| `tui_click` | name-only | | | +| `tui_goto` | name-only | | | +| `tui_keys` | name-only | | | +| `tui_result` | name-only | | | +| `tui_screen` | name-only | | | +| `tui_type` | name-only | | | +| `unschedule` | name-only | | | +| `web_fetch` | name-only | | | +| `web_search` | name-only | | | +| `write_file` | name-only | | | + +## settings (28) + +| name | status | facets | cases | +|---|---|---|---| +| `backends` | name-only | | | +| `box_caps` | missing | | | +| `claude_bin` | missing | | | +| `codex_bin` | missing | | | +| `codex_sandbox` | missing | | | +| `devplace_base` | name-only | | | +| `devplace_key` | name-only | | | +| `devplace_models` | name-only | | | +| `gemini_bin` | missing | | | +| `gemini_plan` | missing | | | +| `grok_bin` | missing | | | +| `model` | name-only | | | +| `muse_bin` | missing | | | +| `opencode_bin` | missing | | | +| `opencode_client_name` | missing | | | +| `opencode_client_ua` | missing | | | +| `opencode_enabled` | missing | | | +| `opencode_models` | name-only | | | +| `openrouter_enabled` | missing | | | +| `openrouter_key` | name-only | | | +| `openrouter_models` | name-only | | | +| `pollinations_enabled` | missing | | | +| `pollinations_models` | name-only | | | +| `retry_seconds` | name-only | | | +| `theme` | name-only | | | +| `voice` | name-only | | | +| `zen_enabled` | missing | | | +| `zen_models` | name-only | | | + +## commands (31) + +| name | status | facets | cases | +|---|---|---|---| +| `/agent` | name-only | | | +| `/agents` | missing | | | +| `/audit` | name-only | | | +| `/bot` | missing | | | +| `/bots` | name-only | | | +| `/clear` | name-only | | | +| `/compact` | missing | | | +| `/ctx` | missing | | | +| `/demo` | name-only | | | +| `/env` | missing | | | +| `/fork` | name-only | | | +| `/graph` | name-only | | | +| `/help` | name-only | | | +| `/install` | name-only | | | +| `/models` | name-only | | | +| `/profile` | name-only | | | +| `/profiles` | missing | | | +| `/quit` | missing | | | +| `/record` | name-only | | | +| `/records` | name-only | | | +| `/release` | name-only | | | +| `/restore` | name-only | | | +| `/schedule` | name-only | | | +| `/schedules` | name-only | | | +| `/search` | name-only | | | +| `/secret` | name-only | | | +| `/skills` | name-only | | | +| `/sysinfo` | name-only | | | +| `/tags` | name-only | | | +| `/tools` | name-only | | | +| `/unschedule` | name-only | | | + +## Classes with the most missing methods (top 25 of 241) + +| class | missing methods | +|---|---| +| MplexCopyMode | 128 | +| MplexCommands | 92 | +| Store | 90 | +| MplexServer | 87 | +| Tools | 85 | +| TuiApp | 62 | +| GrrChatView | 55 | +| TuiPanel | 50 | +| GrrShell | 50 | +| CapAcpDriver | 44 | +| TuiVtScreen | 35 | +| ChatClient | 28 | +| Agent | 26 | +| CapA2aDriver | 24 | +| McpOAuth | 21 | +| MplexPane | 20 | +| TuiCompositor | 20 | +| TuiTextBox | 19 | +| CapPtyDriver | 19 | +| CapFleet | 19 | +| GrrPage | 18 | +| CapDriver | 17 | +| Glow | 16 | +| DebugLog | 16 | +| TuiTerminal | 16 | diff --git a/taicpp/docs/parity.md b/taicpp/docs/parity.md new file mode 100644 index 0000000..a4d0646 --- /dev/null +++ b/taicpp/docs/parity.md @@ -0,0 +1,126 @@ +# parity: how detz is proven equal to tai.py + +Goal: detz behaves exactly like `../tai.py`. This document is the spec of the proof +machinery. It states what exists today; gaps are listed at the end. + +## Pipeline + +`tools/parity.sh run|check` runs, in order: + +1. `tools/parity_gen_cases.py` - regenerates `tests/diff/cases/cli_generated.json` from + `build_parser()` in tai.py (every argparse error and choice message). `--check` fails if stale. +2. `tools/parity_diff.py` - the differential harness (below). +3. `tools/parity_tests.py` - mines `../test_tai.py` into `build/parity/tests-map.{json,md}`. +4. `tools/parity_ledger.py --publish` - builds the symbol ledger and writes `PARITY.md` + (generated, never hand-edited). `--check` makes the pipeline exit 1 while anything is + missing or unverified. + +All output goes to `build/parity/`; only `PARITY.md` is published to the tree. + +## Symbol statuses + +| status | meaning | who sets it | +|---|---|---| +| missing | no identifier in `src/` or `main.cpp` matches the normalized name | ledger scan | +| name-only | an identifier matches; behavior unknown | ledger scan | +| verified | a passing differential case names the symbol | harness result only | + +`verified` is never inferred. A case names a symbol as `kind:name` or `kind:name#facet` +(`#parse`, `#output`, `#listed`, ...). The facet is printed in PARITY.md, so "verified" always +says what was proven (`flag:--debug-format#parse` is argument parsing, not the flag's effect). +Evidence is rejected as stale when the sha256 of tai.py or of the detz binary differs from +the one recorded in `diff-results.json`, and a filtered run (`--filter`) writes +`diff-results-partial.json`, which the ledger ignores. A case naming a symbol the ledger does not +know is an error. Kinds: class, method, function, tool, flag, setting, command (a `/command` of +`HELP_TEXT`). + +## Case format (`tests/diff/cases/*.json`) + +```json +{"cases": [{ + "id": "repl.help", + "doc": "what this proves", + "symbols": ["command:/help"], + "symbols_from_stdout": [{"prefix": "tool:", "pattern": "^ ([a-z_]+)$", "facet": "listed", "step": 0}], + "tests": ["CapCliTests.test_cap_manifest_flag_prints_valid_toml"], + "steps": [{"argv": [], "stdin": "", "env": {}, "files": {"rel/in/cwd": "text"}, + "home_files": {"rel/in/home": "text"}, "mode": "pipe|pty", "timeout": 60}], + "compare": {"stdout": true, "stderr": true, "exit": true, "vault": false, + "files": [{"root": "home|cwd", "glob": "**/*"}], + "vault_ignore_tables": [], "vault_ignore_why": ""}, + "normalize": [{"on": ["stdout"], "pattern": "regex", "replace": "", "dotall": false, "why": "required"}], + "llm": false, "attempts": 1, "min_pass": 1, "checks": [], "expected_fail": "reason" +}]} +``` + +Semantics: + +- Each side (`python3 ../tai.py`, `build/detz`) runs every step in order, sharing one throwaway + `TAI_HOME` and cwd per side (`build/parity/work//-/`). `HOME` is a third + throwaway dir so provider CLIs never touch the real one. Inherited `TAI_*`, `CAP_*`, `TMUX*` + variables are dropped (they leak the caller's session); `COLUMNS=80`, `LINES=24`, + `TERM=dumb`, `TAI_PASSPHRASE=` are set. A step's `env` overrides. +- Compared channels: stdout, stderr, exit code per step, vault, declared files. `mode: pty` uses + `tai_test.ManagedProcess` (stderr is merged into stdout there). +- Vault = a deterministic dump of every `*.db` under the home: table DDL, rows sorted as JSON, + index names. sqlite internals and `fts_docs_*` shadow tables are skipped globally. Anything + else skipped must be listed in `vault_ignore_tables` with `vault_ignore_why`. +- Normalization is per case, per channel, regex only, and every rule needs a `why`. There is + no global scrubbing. Rules that matched nothing on either side are listed in the report + as suspect. +- `symbols_from_stdout` adds symbols read out of the reference's own stdout (for example every + tool `/tools` prints); they count only if the case passes, so detz had to print the same names. +- `tests` lists test_tai.py tests (`Class` or `Class.test_name`) whose assertions the case + reproduces through the real binaries. This is a claim for a reviewer to read, not inferred. +- `expected_fail` marks a known gap with its reason. It still counts as not verified and as + unverified in `parity-check`; it only keeps the harness exit code at 0. If such a case + passes it is reported `xpass` and fails the run, forcing the marker to be removed. + +### LLM cases + +`"llm": true` follows tests/README.md: a real model call on both sides, no mocks, no database +writes standing in for the agent. Outputs are not equal-compared. Instead `checks` must hold +on each side independently: `{"kind":"exit","step":0,"value":0}`, +`{"kind":"stdout_regex","step":1,"pattern":"..."}` (also `stderr_regex`), +`{"kind":"vault_sql","sql":"select count(*) ...","op":">=","value":2}`; add `"side":"ref"|"cpp"` +to scope one. The free rotating backend is flaky by design: `attempts` (cap 5) and `min_pass` +report a ratio per side (`2/3`) in the result instead of demanding 100 percent. `--no-llm` +skips them; skipped cases are never verified. + +## Caps + +jobs 6, steps per case 8, step timeout 300 s, captured output 2 MB per stream, diff lines shown +40 per channel, LLM attempts 5. A timed-out step kills the whole process group. + +## Test-suite map + +`build/parity/tests-map.md` lists every test class of `../test_tai.py` (150 classes, 1233 +tests when written), the tai.py symbols it touches (ast, from `tai.X`, `tai.X.y` and +`"tai.X.y"` patch strings; names that are not code symbols show as `data:`), the number of its +tests reproduced by a passing case, and the case ids or `uncovered`. + +## Seed cases (what exists today) + +- 56 generated cli cases: every missing-value, invalid-choice, invalid-int, explicit-value + error, unknown flags, ambiguous prefixes, valid prefix, `--version`, `-h`, `--help`, + `--cap-manifest`, each choice accepted. +- REPL cases: `/help`, `/tools` (names), `/tools `, unknown command, profiles and bots, + empty listings, plus `boot.vault` (seeded settings catalog, schema, input_history). +- `persist.restart_history`: LLM-driven restart persistence with the same `TAI_PLEXER_PANE` + in both runs. Marked `expected_fail` until the foundation agent lands the store and history + resume. On tai.py it passed 2 of 3 attempts on the first run (free model, flaky). + +## Gaps (each with what closes it) + +- Full tool schema JSON and `SETTINGS_CATALOG` entries are not exposed by any CLI flag, so only + tool names (`/tools`) and the seeded settings rows (`boot.vault`) are diffable today. Closing + it needs a read-only dump command in BOTH implementations, or a Python-side oracle (a script + importing tai.py) plus a detz dump flag. +- Startup banner lines (self-backup name, ctx id, live backend probes) are stripped by a + declared per-case rule; they cannot be compared byte for byte. +- tai.py snapshots itself into the vault on boot (audit, records, chunks, tags, fts_docs). detz + has no such file; `boot.vault` excludes those tables on purpose and says so. Needs an owner + decision, not a harness change. +- Module constants, TUI rendering, mplex, grr need their own cases (pty replay); none exist. +- Ledger "name-only" is a normalized-identifier match, weak by nature; it only separates + "nothing written" from "something written". diff --git a/taicpp/docs/wiring/verify.md b/taicpp/docs/wiring/verify.md new file mode 100644 index 0000000..425b919 --- /dev/null +++ b/taicpp/docs/wiring/verify.md @@ -0,0 +1,24 @@ +# wiring: parity verification targets (for the integrator) + +Add these to `taicpp/Makefile`. Nothing else in the tree needs to change; the tools live in +`tools/` and write only under `build/parity/` (plus the generated `PARITY.md`). + +```make +parity: $(BIN) + DETZ_BIN=$(abspath $(BIN)) tools/parity.sh run --detz $(abspath $(BIN)) + +parity-check: $(BIN) + DETZ_BIN=$(abspath $(BIN)) tools/parity.sh check --detz $(abspath $(BIN)) +``` + +- `make parity` regenerates cases, runs the differential harness against `python3 ../tai.py` + and `build/detz`, mines `../test_tai.py`, rebuilds the ledger and `PARITY.md`. + Exit 1 if a non-expected case failed. +- `make parity-check` does the same and exits non-zero while any symbol is missing or + unverified, a case names an unknown symbol, or evidence is stale. Expect it to fail for a long time. +- Extra arguments are passed to `tools/parity_diff.py`: `--no-llm`, `--filter TEXT`, `--jobs N`. + Example: `tools/parity.sh run --no-llm --filter cli.err`. A filtered run + never feeds the ledger (it writes `diff-results-partial.json`); a `--no-llm` run does feed it but parity-check reports it as a problem. +- Requires python3 (stdlib only), network for LLM cases and for the REPL startup probes. + Cases take about 4 s each because tai.py probes its backends on startup; `--jobs 6` is the cap. +- Do not run `make parity-check` in a gate that must stay green until the symbol count says so. diff --git a/taicpp/tests/diff/cases/cli_generated.json b/taicpp/tests/diff/cases/cli_generated.json new file mode 100644 index 0000000..0810846 --- /dev/null +++ b/taicpp/tests/diff/cases/cli_generated.json @@ -0,0 +1,1137 @@ +{ + "cases": [ + { + "id": "cli.err.profile.missing", + "doc": "--profile with no value", + "symbols": [ + "flag:--profile#parse" + ], + "steps": [ + { + "argv": [ + "--profile" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.yes.explicit", + "doc": "--yes given an explicit value", + "symbols": [ + "flag:--yes#parse" + ], + "steps": [ + { + "argv": [ + "--yes=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.yolo.explicit", + "doc": "--yolo given an explicit value", + "symbols": [ + "flag:--yolo#parse" + ], + "steps": [ + { + "argv": [ + "--yolo=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.auto.explicit", + "doc": "--auto given an explicit value", + "symbols": [ + "flag:--auto#parse" + ], + "steps": [ + { + "argv": [ + "--auto=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.continue.explicit", + "doc": "--continue given an explicit value", + "symbols": [ + "flag:--continue#parse" + ], + "steps": [ + { + "argv": [ + "--continue=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.debug.choice", + "doc": "--debug with a value outside its choices", + "symbols": [ + "flag:--debug#parse" + ], + "steps": [ + { + "argv": [ + "--debug", + "zzz-not-a-choice" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.debug-format.missing", + "doc": "--debug-format with no value", + "symbols": [ + "flag:--debug-format#parse" + ], + "steps": [ + { + "argv": [ + "--debug-format" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.debug-format.choice", + "doc": "--debug-format with a value outside its choices", + "symbols": [ + "flag:--debug-format#parse" + ], + "steps": [ + { + "argv": [ + "--debug-format", + "zzz-not-a-choice" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.env.missing", + "doc": "--env with no value", + "symbols": [ + "flag:--env#parse" + ], + "steps": [ + { + "argv": [ + "--env" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.no-opencode.explicit", + "doc": "--no-opencode given an explicit value", + "symbols": [ + "flag:--no-opencode#parse" + ], + "steps": [ + { + "argv": [ + "--no-opencode=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.update-models.explicit", + "doc": "--update-models given an explicit value", + "symbols": [ + "flag:--update-models#parse" + ], + "steps": [ + { + "argv": [ + "--update-models=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.eval-retrieval.explicit", + "doc": "--eval-retrieval given an explicit value", + "symbols": [ + "flag:--eval-retrieval#parse" + ], + "steps": [ + { + "argv": [ + "--eval-retrieval=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.stats-rag.explicit", + "doc": "--stats-rag given an explicit value", + "symbols": [ + "flag:--stats-rag#parse" + ], + "steps": [ + { + "argv": [ + "--stats-rag=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.version.explicit", + "doc": "--version given an explicit value", + "symbols": [ + "flag:--version#parse" + ], + "steps": [ + { + "argv": [ + "--version=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-manifest.explicit", + "doc": "--cap-manifest given an explicit value", + "symbols": [ + "flag:--cap-manifest#parse" + ], + "steps": [ + { + "argv": [ + "--cap-manifest=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-speak.explicit", + "doc": "--cap-speak given an explicit value", + "symbols": [ + "flag:--cap-speak#parse" + ], + "steps": [ + { + "argv": [ + "--cap-speak=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-cwd.missing", + "doc": "--cap-cwd with no value", + "symbols": [ + "flag:--cap-cwd#parse" + ], + "steps": [ + { + "argv": [ + "--cap-cwd" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-model.missing", + "doc": "--cap-model with no value", + "symbols": [ + "flag:--cap-model#parse" + ], + "steps": [ + { + "argv": [ + "--cap-model" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-permission-mode.missing", + "doc": "--cap-permission-mode with no value", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-permission-mode.choice", + "doc": "--cap-permission-mode with a value outside its choices", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode", + "zzz-not-a-choice" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-max-turns.missing", + "doc": "--cap-max-turns with no value", + "symbols": [ + "flag:--cap-max-turns#parse" + ], + "steps": [ + { + "argv": [ + "--cap-max-turns" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.cap-max-turns.int", + "doc": "--cap-max-turns with a non-integer", + "symbols": [ + "flag:--cap-max-turns#parse" + ], + "steps": [ + { + "argv": [ + "--cap-max-turns", + "abc" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.serve-mcp.missing", + "doc": "--serve-mcp with no value", + "symbols": [ + "flag:--serve-mcp#parse" + ], + "steps": [ + { + "argv": [ + "--serve-mcp" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.mcp-port.missing", + "doc": "--mcp-port with no value", + "symbols": [ + "flag:--mcp-port#parse" + ], + "steps": [ + { + "argv": [ + "--mcp-port" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.mcp-port.int", + "doc": "--mcp-port with a non-integer", + "symbols": [ + "flag:--mcp-port#parse" + ], + "steps": [ + { + "argv": [ + "--mcp-port", + "abc" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.mcp-bind.missing", + "doc": "--mcp-bind with no value", + "symbols": [ + "flag:--mcp-bind#parse" + ], + "steps": [ + { + "argv": [ + "--mcp-bind" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.telegram.explicit", + "doc": "--telegram given an explicit value", + "symbols": [ + "flag:--telegram#parse" + ], + "steps": [ + { + "argv": [ + "--telegram=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.scheduler.explicit", + "doc": "--scheduler given an explicit value", + "symbols": [ + "flag:--scheduler#parse" + ], + "steps": [ + { + "argv": [ + "--scheduler=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.install.explicit", + "doc": "--install given an explicit value", + "symbols": [ + "flag:--install#parse" + ], + "steps": [ + { + "argv": [ + "--install=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.install-telegram.explicit", + "doc": "--install-telegram given an explicit value", + "symbols": [ + "flag:--install-telegram#parse" + ], + "steps": [ + { + "argv": [ + "--install-telegram=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.uninstall-telegram.explicit", + "doc": "--uninstall-telegram given an explicit value", + "symbols": [ + "flag:--uninstall-telegram#parse" + ], + "steps": [ + { + "argv": [ + "--uninstall-telegram=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.plexer-wrap.explicit", + "doc": "--plexer-wrap given an explicit value", + "symbols": [ + "flag:--plexer-wrap#parse" + ], + "steps": [ + { + "argv": [ + "--plexer-wrap=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.demo.explicit", + "doc": "--demo given an explicit value", + "symbols": [ + "flag:--demo#parse" + ], + "steps": [ + { + "argv": [ + "--demo=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.demo-shot.explicit", + "doc": "--demo-shot given an explicit value", + "symbols": [ + "flag:--demo-shot#parse" + ], + "steps": [ + { + "argv": [ + "--demo-shot=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.tui.explicit", + "doc": "--tui given an explicit value", + "symbols": [ + "flag:--tui#parse" + ], + "steps": [ + { + "argv": [ + "--tui=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.tui-local.explicit", + "doc": "--tui-local given an explicit value", + "symbols": [ + "flag:--tui-local#parse" + ], + "steps": [ + { + "argv": [ + "--tui-local=1" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.theme.missing", + "doc": "--theme with no value", + "symbols": [ + "flag:--theme#parse" + ], + "steps": [ + { + "argv": [ + "--theme" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.unknown.long", + "doc": "unknown long flag", + "symbols": [], + "steps": [ + { + "argv": [ + "--bogus" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.unknown.short", + "doc": "unknown short flag", + "symbols": [], + "steps": [ + { + "argv": [ + "-z" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.ambiguous.debug", + "doc": "--deb matches --debug and --debug-format", + "symbols": [ + "flag:--debug#parse" + ], + "steps": [ + { + "argv": [ + "--deb" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.ambiguous.cap", + "doc": "--cap-m matches --cap-model and --cap-max-turns", + "symbols": [ + "flag:--cap-model#parse" + ], + "steps": [ + { + "argv": [ + "--cap-m" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.ambiguous.install", + "doc": "--ins matches --install, --install-telegram", + "symbols": [ + "flag:--install#parse" + ], + "steps": [ + { + "argv": [ + "--ins" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.ambiguous.tui", + "doc": "--tu matches --tui and --tui-local", + "symbols": [ + "flag:--tui#parse" + ], + "steps": [ + { + "argv": [ + "--tu" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.prefix.version", + "doc": "unique prefix --vers resolves to --version", + "symbols": [ + "flag:--version#parse" + ], + "steps": [ + { + "argv": [ + "--vers" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.version.long", + "doc": "--version", + "symbols": [ + "flag:--version#output" + ], + "steps": [ + { + "argv": [ + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.version.with_choice", + "doc": "--version alongside a valid choice flag", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--version", + "--cap-permission-mode", + "none" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.help.long", + "doc": "--help text, exact", + "symbols": [], + "steps": [ + { + "argv": [ + "--help" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.help.short", + "doc": "-h text, exact", + "symbols": [], + "steps": [ + { + "argv": [ + "-h" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.help.wins_over_error", + "doc": "-h before a bad flag prints help, exit 0", + "symbols": [], + "steps": [ + { + "argv": [ + "-h", + "--bogus" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.err.error_before_help", + "doc": "bad choice before -h is an error, exit 2", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode", + "zzz", + "-h" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.cap_manifest", + "doc": "--cap-manifest text, exact", + "symbols": [ + "flag:--cap-manifest#output" + ], + "steps": [ + { + "argv": [ + "--cap-manifest" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + }, + "tests": [ + "CapCliTests.test_cap_manifest_flag_prints_valid_toml" + ] + }, + { + "id": "cli.ok.debug-format.sentence", + "doc": "--debug-format sentence accepted, then --version", + "symbols": [ + "flag:--debug-format#parse" + ], + "steps": [ + { + "argv": [ + "--debug-format", + "sentence", + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.ok.debug-format.json", + "doc": "--debug-format json accepted, then --version", + "symbols": [ + "flag:--debug-format#parse" + ], + "steps": [ + { + "argv": [ + "--debug-format", + "json", + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.ok.cap-permission-mode.none", + "doc": "--cap-permission-mode none accepted, then --version", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode", + "none", + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.ok.cap-permission-mode.confirm", + "doc": "--cap-permission-mode confirm accepted, then --version", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode", + "confirm", + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + }, + { + "id": "cli.ok.cap-permission-mode.interactive", + "doc": "--cap-permission-mode interactive accepted, then --version", + "symbols": [ + "flag:--cap-permission-mode#parse" + ], + "steps": [ + { + "argv": [ + "--cap-permission-mode", + "interactive", + "--version" + ], + "timeout": 30 + } + ], + "compare": { + "stdout": true, + "stderr": true, + "exit": true + } + } + ] +} diff --git a/taicpp/tests/diff/cases/repl.json b/taicpp/tests/diff/cases/repl.json new file mode 100644 index 0000000..58ffbd8 --- /dev/null +++ b/taicpp/tests/diff/cases/repl.json @@ -0,0 +1,46 @@ +{"cases": [ + {"id": "boot.vault", "doc": "Fresh home, REPL started then EOF. Compares the seeded settings catalog rows, schema of every table, input_history. audit/records/chunks/tags/edges/shell_sessions are excluded because tai.py snapshots ITSELF into the vault on boot (path of tai.py, 50 KB source as a record); that is a deliberate per-binary difference, to be decided by the foundation agent, not hidden.", + "symbols": ["function:ensure_schema#schema", "function:boot_settings#seed"], + "steps": [{"argv": [], "stdin": "", "timeout": 60}], + "compare": {"stdout": false, "stderr": true, "exit": true, "vault": true, + "vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"], + "vault_ignore_why": "tai.py self-snapshot (audit/records/chunks/tags/edges/fts_docs) and shell session tree (shell_sessions) are volatile per binary and host pid"}, + "normalize": [ + {"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"\"", "why": "created/updated/ts columns hold wall-clock ISO timestamps"} + ]}, + {"id": "repl.help", "doc": "/help text in the REPL", "symbols": ["command:/help"], + "steps": [{"argv": [], "stdin": "/help\n", "timeout": 60}], + "normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner holds self-backup file name with timestamp, ctx id with pid, and live backend probe results (model counts, latency ms)"}]}, + {"id": "repl.tools", "doc": "/tools lists core and lazy tool names", "symbols": ["command:/tools"], + "symbols_from_stdout": [{"prefix": "tool:", "pattern": "^ ([a-z_]+)$", "facet": "listed", "step": 0}], + "steps": [{"argv": [], "stdin": "/tools\n", "timeout": 60}], + "normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]}, + {"id": "repl.tools_named", "doc": "/tools shell shows one tool", "symbols": ["command:/tools"], + "steps": [{"argv": [], "stdin": "/tools shell\n/tools nosuchtool\n", "timeout": 60}], + "normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]}, + {"id": "repl.unknown_command", "doc": "unknown slash command message", "symbols": [], + "steps": [{"argv": [], "stdin": "/nonsense\n", "timeout": 60}], + "normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]}, + {"id": "repl.profiles", "doc": "/profiles, /profile, /bots on a fresh home", "symbols": ["command:/profiles", "command:/profile", "command:/bots"], + "steps": [{"argv": [], "stdin": "/profiles\n/profile\n/bots\n/profile work\n/profiles\n", "timeout": 60}], + "compare": {"vault": true, "vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"], "vault_ignore_why": "tai.py self-snapshot and shell session tree, see boot.vault"}, + "normalize": [ + {"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}, + {"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"\"", "why": "wall-clock timestamps"} + ]}, + {"id": "repl.skills_secret_schedules", "doc": "listing commands on an empty vault", "symbols": ["command:/skills", "command:/secret", "command:/schedules", "command:/tags", "command:/records"], + "steps": [{"argv": [], "stdin": "/skills\n/secret list\n/schedules\n/tags\n/records zebra\n", "timeout": 60}], + "normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]}, + {"id": "persist.restart_history", "doc": "RESTART PERSISTENCE. Run 1 tells the agent a codeword (real LLM call), run 2 starts a new process with the SAME TAI_PLEXER_PANE and the same TAI_HOME and must resume the history: the events rows of run 1 survive and the model can answer from them. Expected to fail on detz until the foundation agent (store, history resume, plexer pane keyed session) lands. Probabilistic: real free model, 3 attempts, 1 must pass.", + "llm": true, "attempts": 3, "min_pass": 1, + "expected_fail": "needs the foundation agent: vault events persistence and history resume across process restarts; detz has no REPL/agent loop yet (2026-10-25)", + "symbols": ["function:main#restart", "class:Store#events"], + "steps": [ + {"argv": [], "stdin": "My secret codeword is heliotrope-7314. Just acknowledge it in one short sentence.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180}, + {"argv": [], "stdin": "What is my secret codeword? Answer with the codeword only.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180} + ], + "checks": [ + {"kind": "vault_sql", "sql": "select count(*) from events where text like '%heliotrope-7314%'", "op": ">=", "value": 2}, + {"kind": "stdout_regex", "step": 1, "pattern": "heliotrope-7314"} + ]} +]} diff --git a/taicpp/tools/parity.sh b/taicpp/tools/parity.sh new file mode 100755 index 0000000..025a95c --- /dev/null +++ b/taicpp/tools/parity.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Parity pipeline. Usage: tools/parity.sh run|check [extra parity_diff.py args] +# run regenerate cases, run the differential harness, mine test_tai.py, rebuild the ledger +# and the committed PARITY.md. Exit 0 unless a non-expected case failed. +# check same as run, then exit 1 while any symbol is missing or unverified, a case names an +# unknown symbol, or the evidence is stale. This is `make parity-check`. +# Outputs go to build/parity/ (PARITY.md is published to taicpp/PARITY.md by --publish). +set -u +cd "$(dirname "$0")/.." || exit 2 +mode="${1:-run}" +shift || true +rc=0 +python3 tools/parity_gen_cases.py --check || { echo "regenerating cli_generated.json"; python3 tools/parity_gen_cases.py || exit 2; } +python3 tools/parity_diff.py "$@" || rc=1 +python3 tools/parity_tests.py || rc=1 +if [ "$mode" = "check" ]; then + python3 tools/parity_ledger.py --publish --check || rc=1 +else + python3 tools/parity_ledger.py --publish || rc=1 +fi +exit $rc diff --git a/taicpp/tools/parity_diff.py b/taicpp/tools/parity_diff.py new file mode 100755 index 0000000..1994ee3 --- /dev/null +++ b/taicpp/tools/parity_diff.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +"""Differential harness: run each case against tai.py and detz, compare. + +Usage: + tools/parity_diff.py [--detz PATH] [--py PATH] [--filter TEXT] [--no-llm] + [--cases DIR] [--jobs N] [--list] + +Every case runs twice, once per side, in its own throwaway TAI_HOME and cwd +(never ~/.tai), with the real binaries and a real vault. Compared channels: +stdout, stderr, exit code, vault state (sqlite dump of every *.db in the home), +and declared files. Volatile data is normalized ONLY by rules declared in the +case itself (each with a `why`); there is no blanket scrubbing. A rule that +matched nothing on either side is reported as unused. + +LLM cases (`"llm": true`) make a real model call on both sides (tests/README.md +rule, no mocks). Their outputs are not equal-compared; both sides must each +satisfy the case's `checks`. They are probabilistic: `attempts` / `min_pass` +report a ratio, and the result carries it. + +Writes build/parity/diff-results.json and diff-report.md. Exit 1 if any case +that is not marked expected_fail failed, or an expected_fail case passed. +See docs/parity.md for the case format. +""" +import argparse +import concurrent.futures +import difflib +import glob as globmod +import hashlib +import json +import os +import re +import shutil +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +REPO = os.path.dirname(ROOT) +OUT = os.path.join(ROOT, "build", "parity") +CASES_DIR = os.path.join(ROOT, "tests", "diff", "cases") + +# Hard caps (nothing unbounded). +MAX_JOBS = 6 +MAX_STEPS = 8 +MAX_STEP_TIMEOUT = 300 +MAX_DIFF_LINES = 40 +MAX_CAPTURE = 2_000_000 +MAX_ATTEMPTS = 5 +BASE_ENV = {"COLUMNS": "80", "LINES": "24", "TAI_PASSPHRASE": "", "LC_ALL": "C.UTF-8", "TERM": "dumb"} +VAULT_SKIP_PREFIXES = ("sqlite_", "fts_docs_") # sqlite internals and fts shadow tables +CHANNELS = ("stdout", "stderr", "exit", "vault", "files") + + +class CaseError(Exception): + pass + + +def sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +class Rule: + """One declared normalization: regex -> replacement on named channels.""" + + def __init__(self, spec, case_id): + for key in ("on", "pattern", "replace", "why"): + if key not in spec: + raise CaseError("%s: normalize rule missing %r" % (case_id, key)) + if not str(spec["why"]).strip(): + raise CaseError("%s: normalize rule needs a non-empty why" % case_id) + self.on = list(spec["on"]) + for ch in self.on: + if ch not in CHANNELS: + raise CaseError("%s: normalize rule on unknown channel %r" % (case_id, ch)) + self.regex = re.compile(spec["pattern"], re.M | (re.S if spec.get("dotall") else 0)) + self.replace = spec["replace"] + self.why = spec["why"] + self.hits = {"ref": 0, "cpp": 0} + + def apply(self, channel, text, side): + if channel not in self.on: + return text + text, count = self.regex.subn(self.replace, text) + self.hits[side] += count + return text + + +class Step: + def __init__(self, spec, case_id): + self.argv = [str(a) for a in spec.get("argv", [])] + self.stdin = spec.get("stdin", "") + self.env = {k: str(v) for k, v in spec.get("env", {}).items()} + self.files = dict(spec.get("files", {})) + self.home_files = dict(spec.get("home_files", {})) + self.mode = spec.get("mode", "pipe") + self.timeout = min(float(spec.get("timeout", 60)), MAX_STEP_TIMEOUT) + if self.mode not in ("pipe", "pty"): + raise CaseError("%s: unknown step mode %r" % (case_id, self.mode)) + + +class Case: + def __init__(self, spec, source): + self.id = spec["id"] + self.source = source + self.doc = spec.get("doc", "") + self.symbols = list(spec.get("symbols", [])) + self.tests = list(spec.get("tests", [])) + self.llm = bool(spec.get("llm", False)) + self.expected_fail = spec.get("expected_fail") + self.steps = [Step(s, self.id) for s in spec["steps"]] + if not self.steps or len(self.steps) > MAX_STEPS: + raise CaseError("%s: 1..%d steps required" % (self.id, MAX_STEPS)) + compare = spec.get("compare", {}) + self.compare = { + "stdout": compare.get("stdout", True), + "stderr": compare.get("stderr", True), + "exit": compare.get("exit", True), + "vault": compare.get("vault", False), + "files": compare.get("files", []), + } + self.vault_ignore = list(compare.get("vault_ignore_tables", [])) + if self.vault_ignore and not str(compare.get("vault_ignore_why", "")).strip(): + raise CaseError("%s: vault_ignore_tables needs vault_ignore_why" % self.id) + self.rule_specs = list(spec.get("normalize", [])) + for rs in self.rule_specs: + Rule(rs, self.id) # validate at load + self.checks = list(spec.get("checks", [])) + # Symbols named by the reference's own output (e.g. every tool /tools lists), claimed only + # when the case passes, i.e. when detz printed the same thing. + self.symbols_from_stdout = list(spec.get("symbols_from_stdout", [])) + self.attempts = min(int(spec.get("attempts", 1)), MAX_ATTEMPTS) + self.min_pass = int(spec.get("min_pass", 1)) + if self.llm and not self.checks: + raise CaseError("%s: llm case needs checks (outputs are not equal-compared)" % self.id) + + def rules(self): + return [Rule(rs, self.id) for rs in self.rule_specs] + + +def load_cases(directory): + cases, seen = [], {} + for path in sorted(globmod.glob(os.path.join(directory, "*.json"))): + with open(path, encoding="utf-8") as fh: + doc = json.load(fh) + for spec in doc["cases"]: + case = Case(spec, os.path.basename(path)) + if case.id in seen: + raise CaseError("duplicate case id %s (%s, %s)" % (case.id, seen[case.id], path)) + seen[case.id] = path + cases.append(case) + return cases + + +class Side: + """One implementation under test.""" + + def __init__(self, name, command): + self.name = name + self.command = list(command) + + +class Capture: + """Everything observed from running one case once on one side.""" + + def __init__(self): + self.steps = [] # list of dict(argv, rc, stdout, stderr, timed_out) + self.vault = "" + self.files = "" + self.sql = None # open read-only vault path for checks + self.home = None + self.cwd = None + + +class Runner: + def __init__(self, work_root): + self.work_root = work_root + + def run_step(self, side, step, home, cwd): + for rel, content in step.files.items(): + self._write(os.path.join(cwd, rel), content) + for rel, content in step.home_files.items(): + self._write(os.path.join(home, rel), content) + # Hermetic environment: inherited TAI_*, CAP_*, TMUX* leak the caller's session into the + # run (parent shell session, pane resolution), so they are dropped. Declared per step env wins. + env = {k: v for k, v in os.environ.items() if not k.startswith(("TAI_", "CAP_", "TMUX"))} + env.update(BASE_ENV) + user_home = os.path.join(os.path.dirname(home), "user") + os.makedirs(user_home, exist_ok=True) + env.update({"TAI_HOME": home, "HOME": user_home}) + env.update(step.env) + cmd = side.command + step.argv + start = time.monotonic() + if step.mode == "pty": + return self._run_pty(cmd, step, env, cwd, start) + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, env=env, start_new_session=True) + timed_out = False + try: + out, err = proc.communicate(step.stdin.encode("utf-8"), timeout=step.timeout) + except subprocess.TimeoutExpired: + timed_out = True + self._kill(proc) + out, err = proc.communicate() + return {"argv": step.argv, "rc": proc.returncode, "timed_out": timed_out, + "stdout": out[:MAX_CAPTURE].decode("utf-8", "replace"), + "stderr": err[:MAX_CAPTURE].decode("utf-8", "replace"), + "secs": round(time.monotonic() - start, 2)} + + def _run_pty(self, cmd, step, env, cwd, start): + import asyncio + sys.path.insert(0, REPO) + import tai_test # the one base harness (tests/README.md); pty merges stderr into stdout + + async def go(): + proc = tai_test.ManagedProcess(cmd, mode="pty", cwd=cwd, env=env) + await proc.start() + timed_out = False + text = "" + try: + for line in step.stdin.splitlines(): + await proc.send_line(line) + text = await proc.drain_until_eof(timeout=step.timeout) + except tai_test.TestTimeout: + timed_out = True + text = "".join(l.text for l in proc.transcript if l.stream == "stdout") + rc = await proc.terminate() if timed_out else await proc.wait(5) + await proc.terminate() + return text, rc, timed_out + text, rc, timed_out = asyncio.run(go()) + return {"argv": step.argv, "rc": rc, "timed_out": timed_out, "stdout": tai_test.strip_ansi(text)[:MAX_CAPTURE], + "stderr": "", "secs": round(time.monotonic() - start, 2)} + + @staticmethod + def _write(path, content): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(content) + + @staticmethod + def _kill(proc): + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(proc.pid, sig) + except (ProcessLookupError, PermissionError): + return + try: + proc.wait(3) + return + except subprocess.TimeoutExpired: + continue + + def capture(self, case, side, tag): + base = os.path.join(self.work_root, case.id.replace("/", "_"), "%s-%s" % (side.name, tag)) + shutil.rmtree(base, ignore_errors=True) + home, cwd = os.path.join(base, "home"), os.path.join(base, "cwd") + os.makedirs(home) + os.makedirs(cwd) + cap = Capture() + cap.home, cap.cwd = home, cwd + for step in case.steps: + cap.steps.append(self.run_step(side, step, home, cwd)) + if case.compare["vault"]: + cap.vault = VaultDump(home, case.vault_ignore).text() + if case.compare["files"]: + cap.files = FilesDump(home, cwd, case.compare["files"]).text() + return cap + + +class VaultDump: + """Deterministic text dump of every sqlite db under the home (read-only copy).""" + + def __init__(self, home, ignore): + self.home = home + self.ignore = set(ignore) + + def text(self): + lines = [] + for base, _dirs, names in sorted(os.walk(self.home)): + for name in sorted(names): + if name.endswith(".db"): + path = os.path.join(base, name) + lines.append("# db %s" % os.path.relpath(path, self.home)) + lines.extend(self._dump(path)) + return "\n".join(lines) + ("\n" if lines else "") + + def _dump(self, path): + out = [] + tmp = tempfile.mkdtemp() + try: + copy = os.path.join(tmp, "v.db") + shutil.copy(path, copy) + for suffix in ("-wal", "-shm"): + if os.path.exists(path + suffix): + shutil.copy(path + suffix, copy + suffix) + db = sqlite3.connect(copy) + db.text_factory = lambda b: b.decode("utf-8", "replace") + tables = [r for r in db.execute("SELECT name, sql FROM sqlite_master WHERE type='table' ORDER BY name")] + for name, sql in tables: + if name.startswith(VAULT_SKIP_PREFIXES) or name in self.ignore: + continue + out.append("## table %s :: %s" % (name, " ".join((sql or "").split()))) + rows = [json.dumps(list(r), ensure_ascii=False, default=repr) for r in db.execute('SELECT * FROM "%s"' % name)] + out.extend(" " + r for r in sorted(rows)) + idx = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name")] + out.append("## indexes %s" % ",".join(idx)) + db.close() + except sqlite3.Error as exc: + out.append("## unreadable: %s" % exc) + finally: + shutil.rmtree(tmp, ignore_errors=True) + return out + + +class FilesDump: + """Path + content hash listing for declared globs, rooted at home or cwd.""" + + def __init__(self, home, cwd, specs): + self.roots = {"home": home, "cwd": cwd} + self.specs = specs + + def text(self): + lines = [] + for spec in self.specs: + root = self.roots[spec["root"]] + for path in sorted(globmod.glob(os.path.join(root, spec["glob"]), recursive=True)): + if os.path.isfile(path): + rel = "%s:%s" % (spec["root"], os.path.relpath(path, root)) + with open(path, "rb") as fh: + data = fh.read(MAX_CAPTURE) + try: + body = data.decode("utf-8") + lines.append("%s\t%s" % (rel, json.dumps(body, ensure_ascii=False))) + except UnicodeDecodeError: + lines.append("%s\tsha256:%s" % (rel, hashlib.sha256(data).hexdigest())) + return "\n".join(sorted(lines)) + ("\n" if lines else "") + + +class Normalizer: + def __init__(self, case): + self.rules = case.rules() + + def apply(self, channel, text, side): + for rule in self.rules: + text = rule.apply(channel, text, side) + return text + + def unused(self): + return [r.why for r in self.rules if r.hits["ref"] == 0 and r.hits["cpp"] == 0] + + +class Checker: + """Evaluates `checks` of an llm case against one side's capture.""" + + def __init__(self, case): + self.case = case + + def evaluate(self, cap, side): + fails = [] + for chk in self.case.checks: + if chk.get("side", "both") not in ("both", side): + continue + kind = chk["kind"] + if kind == "exit": + got = cap.steps[chk["step"]]["rc"] + if got != chk["value"]: + fails.append("exit of step %d is %r, want %r" % (chk["step"], got, chk["value"])) + elif kind in ("stdout_regex", "stderr_regex"): + text = cap.steps[chk["step"]]["stdout" if kind == "stdout_regex" else "stderr"] + if not re.search(chk["pattern"], text, re.S | re.I): + fails.append("%s step %d: /%s/ not found" % (kind, chk["step"], chk["pattern"])) + elif kind == "vault_sql": + got = self._sql(cap, chk["sql"]) + if not self._cmp(got, chk["op"], chk["value"]): + fails.append("vault_sql %r gave %r, want %s %r" % (chk["sql"], got, chk["op"], chk["value"])) + else: + fails.append("unknown check kind %r" % kind) + return fails + + @staticmethod + def _cmp(got, op, want): + try: + return {"==": got == want, ">=": got >= want, "<=": got <= want, ">": got > want, "<": got < want}[op] + except TypeError: + return False + + @staticmethod + def _sql(cap, sql): + path = os.path.join(cap.home, "memory.db") + if not os.path.exists(path): + return None + tmp = tempfile.mkdtemp() + try: + copy = os.path.join(tmp, "v.db") + shutil.copy(path, copy) + for suffix in ("-wal", "-shm"): + if os.path.exists(path + suffix): + shutil.copy(path + suffix, copy + suffix) + db = sqlite3.connect(copy) + row = db.execute(sql).fetchone() + db.close() + return row[0] if row else None + except sqlite3.Error as exc: + return "sqlerror: %s" % exc + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +class Verdict: + def __init__(self, case): + self.case = case + self.status = "fail" # pass | fail | xfail | xpass | skipped + self.problems = [] # human lines, capped + self.ratio = None # llm: "ok/attempts" per side + self.unused_rules = [] + self.extra_symbols = [] + self.secs = 0.0 + + def as_dict(self): + return {"id": self.case.id, "status": self.status, "llm": self.case.llm, "symbols": self.case.symbols + self.extra_symbols, + "tests": self.case.tests, "expected_fail": self.case.expected_fail, "problems": self.problems, + "ratio": self.ratio, "unused_rules": self.unused_rules, "secs": round(self.secs, 1), + "source": self.case.source} + + +class CaseExecutor: + def __init__(self, runner, ref, cpp): + self.runner, self.ref, self.cpp = runner, ref, cpp + + def execute(self, case): + start = time.monotonic() + verdict = Verdict(case) + try: + if case.llm: + self._llm(case, verdict) + else: + self._equal(case, verdict) + except Exception as exc: # harness fault is a failure, loudly + verdict.problems.append("harness error: %r" % exc) + passed = not verdict.problems + if case.expected_fail: + verdict.status = "xpass" if passed else "xfail" + else: + verdict.status = "pass" if passed else "fail" + verdict.secs = time.monotonic() - start + return verdict + + def _equal(self, case, verdict): + norm = Normalizer(case) + caps = {"ref": self.runner.capture(case, self.ref, "a"), "cpp": self.runner.capture(case, self.cpp, "a")} + for i, step in enumerate(case.steps): + for side, cap in caps.items(): + if cap.steps[i]["timed_out"]: + verdict.problems.append("step %d timed out on %s (%.0fs)" % (i, side, step.timeout)) + views = {} + for side, cap in caps.items(): + view = {"exit": "\n".join(str(s["rc"]) for s in cap.steps) + "\n"} + view["stdout"] = "".join("--- step %d\n%s" % (i, s["stdout"]) for i, s in enumerate(cap.steps)) + view["stderr"] = "".join("--- step %d\n%s" % (i, s["stderr"]) for i, s in enumerate(cap.steps)) + view["vault"] = cap.vault + view["files"] = cap.files + views[side] = {ch: norm.apply(ch, txt, side) for ch, txt in view.items()} + for ch in CHANNELS: + if not case.compare[ch]: + continue + if views["ref"][ch] != views["cpp"][ch]: + verdict.problems.extend(self._diff(ch, views["ref"][ch], views["cpp"][ch])) + verdict.unused_rules = norm.unused() + for spec in case.symbols_from_stdout: + text = views["ref"]["stdout"].split("--- step %d\n" % spec["step"], 1)[-1] + names = re.findall(spec["pattern"], text, re.M) + if not names: + verdict.problems.append("symbols_from_stdout matched nothing for %s" % spec["prefix"]) + verdict.extra_symbols += ["%s%s#%s" % (spec["prefix"], n, spec["facet"]) for n in names] + + @staticmethod + def _diff(channel, ref, cpp): + lines = list(difflib.unified_diff(ref.splitlines(), cpp.splitlines(), "tai.py:" + channel, "detz:" + channel, lineterm="", n=1)) + out = ["%s differs (%d diff lines)" % (channel, len(lines))] + out.extend(lines[:MAX_DIFF_LINES]) + if len(lines) > MAX_DIFF_LINES: + out.append("... %d more diff lines" % (len(lines) - MAX_DIFF_LINES)) + return out + + def _llm(self, case, verdict): + checker = Checker(case) + ratios = {} + for name, side in (("ref", self.ref), ("cpp", self.cpp)): + ok, last = 0, [] + for attempt in range(case.attempts): + cap = self.runner.capture(case, side, "t%d" % attempt) + fails = checker.evaluate(cap, name) + for i, s in enumerate(cap.steps): + if s["timed_out"]: + fails.append("step %d timed out" % i) + if fails: + last = fails + else: + ok += 1 + ratios[name] = "%d/%d" % (ok, case.attempts) + if ok < case.min_pass: + verdict.problems.append("%s: %d/%d attempts passed, need %d" % (name, ok, case.attempts, case.min_pass)) + verdict.problems.extend(" " + f for f in last[:MAX_DIFF_LINES]) + verdict.ratio = ratios + + +class Report: + def __init__(self, verdicts, meta): + self.verdicts, self.meta = verdicts, meta + + def write(self): + os.makedirs(OUT, exist_ok=True) + suffix = "-partial" if self.meta["filtered"] else "" # a filtered run never feeds the ledger + with open(os.path.join(OUT, "diff-results%s.json" % suffix), "w") as fh: + json.dump({"meta": self.meta, "results": [v.as_dict() for v in self.verdicts]}, fh, indent=1) + counts = {} + for v in self.verdicts: + counts[v.status] = counts.get(v.status, 0) + 1 + lines = ["# Differential report (generated)", "", + "tai.py sha256 `%s`, detz sha256 `%s`" % (self.meta["tai_sha"][:16], self.meta["detz_sha"][:16]), "", + "Counts: " + ", ".join("%s=%d" % kv for kv in sorted(counts.items())), "", + "| case | status | llm | secs | note |", "|---|---|---|---|---|"] + for v in sorted(self.verdicts, key=lambda x: x.case.id): + note = (v.case.expected_fail or "") if v.status in ("xfail", "xpass") else "" + if v.ratio: + note += " ratio %s" % json.dumps(v.ratio) + lines.append("| %s | %s | %s | %.1f | %s |" % (v.case.id, v.status, "yes" if v.case.llm else "", v.secs, note.strip())) + lines += ["", "## Details of non-passing cases", ""] + for v in sorted(self.verdicts, key=lambda x: x.case.id): + if v.status in ("pass", "skipped"): + continue + lines += ["### %s (%s)" % (v.case.id, v.status), ""] + if v.case.expected_fail: + lines += ["expected_fail: " + v.case.expected_fail, ""] + lines += ["```"] + v.problems[:MAX_DIFF_LINES * 3] + ["```", ""] + unused = [(v.case.id, w) for v in self.verdicts for w in v.unused_rules] + if unused: + lines += ["## Unused normalization rules (suspect)", ""] + ["- %s: %s" % u for u in unused] + [""] + with open(os.path.join(OUT, "diff-report%s.md" % suffix), "w") as fh: + fh.write("\n".join(lines)) + return counts + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--detz", default=os.environ.get("DETZ_BIN", os.path.join(ROOT, "build", "detz"))) + ap.add_argument("--py", default=os.environ.get("TAI_PY", os.path.join(REPO, "tai.py"))) + ap.add_argument("--cases", default=CASES_DIR) + ap.add_argument("--filter", default=None) + ap.add_argument("--no-llm", action="store_true") + ap.add_argument("--jobs", type=int, default=4) + ap.add_argument("--list", action="store_true") + args = ap.parse_args() + cases = load_cases(args.cases) + if args.filter: + cases = [c for c in cases if args.filter in c.id] + if args.list: + for c in cases: + print(c.id, "llm" if c.llm else "", c.expected_fail or "") + return 0 + if not os.path.isfile(args.py): + print("tai.py not found: %s" % args.py, file=sys.stderr) + return 2 + detz_ok = os.path.isfile(args.detz) and os.access(args.detz, os.X_OK) + if not detz_ok: + print("detz binary missing: %s (every case will fail; build it with `make`)" % args.detz, file=sys.stderr) + work = os.path.join(OUT, "work") + shutil.rmtree(work, ignore_errors=True) + runner = Runner(work) + ref = Side("ref", [sys.executable, args.py]) + cpp = Side("cpp", [args.detz] if detz_ok else ["/nonexistent/detz"]) + executor = CaseExecutor(runner, ref, cpp) + todo, verdicts = [], [] + for c in cases: + if c.llm and args.no_llm: + v = Verdict(c) + v.status = "skipped" + verdicts.append(v) + else: + todo.append(c) + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(args.jobs, MAX_JOBS))) as pool: + for v in pool.map(executor.execute, todo): + print("%-7s %s" % (v.status.upper(), v.case.id), flush=True) + verdicts.append(v) + meta = {"tai_sha": sha256_file(args.py), "detz_sha": sha256_file(args.detz) if detz_ok else "missing", + "detz_path": args.detz, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "filtered": bool(args.filter), + "no_llm": args.no_llm} + counts = Report(verdicts, meta).write() + print("counts:", counts, "(report dir: %s)" % OUT) + bad = counts.get("fail", 0) + counts.get("xpass", 0) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tools/parity_gen_cases.py b/taicpp/tools/parity_gen_cases.py new file mode 100755 index 0000000..8d455d8 --- /dev/null +++ b/taicpp/tools/parity_gen_cases.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Generate tests/diff/cases/cli_generated.json from tai.py's build_parser(). + +Every argparse error and choice message is exercised from the parser's own +declaration (read with ast, tai.py untouched), so the case list cannot drift +from the flags. Usage: tools/parity_gen_cases.py [--check] +--check exits 1 if the committed file differs from what the parser implies. + +Facet note: these cases verify argument PARSING only (messages, exit code), +so they name symbols as `flag:--x#parse`, never the flag's behavior. +""" +import ast +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +PY = os.path.join(os.path.dirname(ROOT), "tai.py") +TARGET = os.path.join(ROOT, "tests", "diff", "cases", "cli_generated.json") + + +class Opt: + def __init__(self, call): + self.flag = next(a.value for a in call.args if isinstance(a, ast.Constant) and isinstance(a.value, str)) + kw = {k.arg: k.value for k in call.keywords} + self.action = ast.literal_eval(kw["action"]) if "action" in kw and isinstance(kw["action"], ast.Constant) else None + self.type = kw["type"].id if "type" in kw and isinstance(kw["type"], ast.Name) else None + self.choices = list(ast.literal_eval(kw["choices"])) if "choices" in kw else None + nargs = kw.get("nargs") + self.nargs = None + if isinstance(nargs, ast.Constant): + self.nargs = nargs.value + elif isinstance(nargs, ast.Attribute): + self.nargs = nargs.attr # REMAINDER + + @property + def takes_value(self): + return self.action != "store_true" and self.nargs != "REMAINDER" + + @property + def optional_value(self): + return self.nargs == "?" + + +class CaseBuilder: + def __init__(self): + self.cases = [] + + def add(self, cid, doc, argv, flag, facet="parse"): + self.cases.append({ + "id": cid, "doc": doc, "symbols": ["flag:%s#%s" % (flag, facet)] if flag else [], + "steps": [{"argv": argv, "timeout": 30}], + "compare": {"stdout": True, "stderr": True, "exit": True}, + }) + + +def options(): + with open(PY, encoding="utf-8") as fh: + tree = ast.parse(fh.read()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "build_parser") + out = [] + for node in ast.walk(fn): + if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument": + if any(isinstance(a, ast.Constant) and isinstance(a.value, str) and a.value.startswith("--") for a in node.args): + out.append(Opt(node)) + return out + + +def build(): + b = CaseBuilder() + opts = options() + for o in opts: + slug = o.flag[2:] + if o.takes_value and not o.optional_value: + b.add("cli.err.%s.missing" % slug, "%s with no value" % o.flag, [o.flag], o.flag) + if o.choices: + b.add("cli.err.%s.choice" % slug, "%s with a value outside its choices" % o.flag, [o.flag, "zzz-not-a-choice"], o.flag) + if o.type == "int": + b.add("cli.err.%s.int" % slug, "%s with a non-integer" % o.flag, [o.flag, "abc"], o.flag) + if o.action == "store_true": + b.add("cli.err.%s.explicit" % slug, "%s given an explicit value" % o.flag, ["%s=1" % o.flag], o.flag) + # Prefix abbreviation: python argparse allows unique prefixes and rejects ambiguous ones. + b.add("cli.err.unknown.long", "unknown long flag", ["--bogus"], None) + b.add("cli.err.unknown.short", "unknown short flag", ["-z"], None) + b.add("cli.err.ambiguous.debug", "--deb matches --debug and --debug-format", ["--deb"], "--debug") + b.add("cli.err.ambiguous.cap", "--cap-m matches --cap-model and --cap-max-turns", ["--cap-m"], "--cap-model") + b.add("cli.err.ambiguous.install", "--ins matches --install, --install-telegram", ["--ins"], "--install") + b.add("cli.err.ambiguous.tui", "--tu matches --tui and --tui-local", ["--tu"], "--tui") + b.add("cli.prefix.version", "unique prefix --vers resolves to --version", ["--vers"], "--version") + b.add("cli.version.long", "--version", ["--version"], "--version", "output") + b.add("cli.version.with_choice", "--version alongside a valid choice flag", ["--version", "--cap-permission-mode", "none"], "--cap-permission-mode") + b.add("cli.help.long", "--help text, exact", ["--help"], None) + b.add("cli.help.short", "-h text, exact", ["-h"], None) + b.add("cli.help.wins_over_error", "-h before a bad flag prints help, exit 0", ["-h", "--bogus"], None) + b.add("cli.err.error_before_help", "bad choice before -h is an error, exit 2", ["--cap-permission-mode", "zzz", "-h"], "--cap-permission-mode") + b.add("cli.cap_manifest", "--cap-manifest text, exact", ["--cap-manifest"], "--cap-manifest", "output") + b.cases[-1]["tests"] = ["CapCliTests.test_cap_manifest_flag_prints_valid_toml"] + # Choice success path for each choices option, parse-only via --version short-circuit is not + # reliable for all flags, so only the flags whose success path is side-effect free are listed. + for o in opts: + if o.choices: + for ch in o.choices: + if o.flag == "--debug": + continue # --debug starts logging; success behavior is a behavior case + b.add("cli.ok.%s.%s" % (o.flag[2:], ch), "%s %s accepted, then --version" % (o.flag, ch), [o.flag, ch, "--version"], o.flag) + return {"cases": b.cases} + + +def main(): + text = json.dumps(build(), indent=1) + "\n" + if "--check" in sys.argv: + have = open(TARGET).read() if os.path.exists(TARGET) else "" + if have != text: + print("cli_generated.json is stale: run tools/parity_gen_cases.py") + return 1 + print("cli_generated.json current") + return 0 + os.makedirs(os.path.dirname(TARGET), exist_ok=True) + with open(TARGET, "w") as fh: + fh.write(text) + print("wrote %s (%d cases)" % (TARGET, len(json.loads(text)["cases"]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taicpp/tools/parity_ledger.py b/taicpp/tools/parity_ledger.py old mode 100644 new mode 100755 index d987fa6..82dcab4 --- a/taicpp/tools/parity_ledger.py +++ b/taicpp/tools/parity_ledger.py @@ -1,15 +1,27 @@ #!/usr/bin/env python3 -"""Parity ledger: every tai.py symbol against the C++ tree. +"""Parity ledger: every tai.py symbol against the C++ tree and the differential results. -Usage: tools/parity_ledger.py [--check] (writes build/parity/ledger.json and ledger.md) +Usage: tools/parity_ledger.py [--check] [--publish] -Status per symbol: - missing no identifier in src/ or main.cpp matches the normalized name - name-only an identifier matches; behavior is NOT verified by this tool -Verification (diff of real outputs against tai.py) is a separate, later status -and is never inferred here. --check exits 1 while any symbol is missing. +Writes build/parity/ledger.json, ledger.md and PARITY.md (all under build/). +--publish also copies the generated PARITY.md over taicpp/PARITY.md (the committed copy). +--check exits 1 while ANY symbol is missing or unverified, a case names a symbol the +ledger does not know, or the differential results are absent or stale. + +Status per symbol (strongest first): + verified a passing differential case (tools/parity_diff.py, unfiltered run, tai.py and + detz hashes equal to the current files) names the symbol. Facets list WHAT the + case verified (`parse`, `output`, ...); verified never means "all behavior". + name-only an identifier in src/ or main.cpp matches the normalized name. Behavior unproven. + missing no identifier matches. +Only a differential case sets `verified`; nothing here infers it. + +Symbol ids are `kind:name`, optionally `#facet` in case files: + class:Store method:Store.load function:main tool:shell flag:--version + setting:model command:/help """ import ast +import hashlib import json import os import re @@ -17,93 +29,294 @@ import sys HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) -PY = os.path.join(os.path.dirname(ROOT), "tai.py") +REPO = os.path.dirname(ROOT) +PY = os.path.join(REPO, "tai.py") OUT = os.path.join(ROOT, "build", "parity") +DETZ = os.environ.get("DETZ_BIN", os.path.join(ROOT, "build", "detz")) +CASES_DIR = os.path.join(ROOT, "tests", "diff", "cases") +PARITY_MD = os.path.join(ROOT, "PARITY.md") +KINDS = ("class", "method", "function", "tool", "flag", "setting", "command") def norm(name): return re.sub(r"[^a-z0-9]", "", name.lower()) -def cpp_idents(): - idents = set() - files = [os.path.join(ROOT, "main.cpp")] - for base, _dirs, names in os.walk(os.path.join(ROOT, "src")): - files += [os.path.join(base, n) for n in names if n.endswith((".cpp", ".h", ".hpp"))] - for path in files: - with open(path, encoding="utf-8", errors="replace") as fh: - idents.update(norm(t) for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", fh.read())) - return idents +def sha256_file(path): + if not os.path.isfile(path): + return "missing" + digest = hashlib.sha256() + with open(path, "rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() -def python_symbols(tree): - out = [] - for node in tree.body: - if isinstance(node, ast.ClassDef): - out.append(("class", node.name, node.lineno)) - for sub in node.body: - if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)): - out.append(("method", "%s.%s" % (node.name, sub.name), sub.lineno)) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - out.append(("function", node.name, node.lineno)) - return out +class PySymbols: + """Symbols extracted from tai.py with ast.""" + + def __init__(self, path): + with open(path, encoding="utf-8") as fh: + self.tree = ast.parse(fh.read()) + + def code(self): + out = [] + for node in self.tree.body: + if isinstance(node, ast.ClassDef): + out.append(("class", node.name, node.lineno)) + for sub in node.body: + if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)): + out.append(("method", "%s.%s" % (node.name, sub.name), sub.lineno)) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + out.append(("function", node.name, node.lineno)) + return out + + def tools(self): + out = [] + for node in ast.walk(self.tree): + if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "tool_schema" and node.args: + arg = node.args[0] + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + out.append(("tool", arg.value, node.lineno)) + return out + + def flags(self): + out = [] + for node in ast.walk(self.tree): + if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument": + for arg in node.args: + if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and arg.value.startswith("--"): + out.append(("flag", arg.value, node.lineno)) + return out + + def settings(self): + out = [] + for node in self.tree.body: + if isinstance(node, ast.Assign) and any(getattr(t, "id", "") == "SETTINGS_CATALOG" for t in node.targets): + for sub in ast.walk(node): + if isinstance(sub, ast.Call) and getattr(sub.func, "id", "") == "setting_spec" and sub.args: + arg = sub.args[0] + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + out.append(("setting", arg.value, sub.lineno)) + return out + + def commands(self): + out = [] + for node in self.tree.body: + if isinstance(node, ast.Assign) and any(getattr(t, "id", "") == "HELP_TEXT" for t in node.targets): + text = "".join(c.value for c in ast.walk(node.value) if isinstance(c, ast.Constant) and isinstance(c.value, str)) + seen = set() + for line in text.splitlines(): + m = re.match(r"\s+(/[a-z]+)", line) + if m and m.group(1) not in seen: + seen.add(m.group(1)) + out.append(("command", m.group(1), node.lineno)) + return out + + def all(self): + return self.code() + self.tools() + self.flags() + self.settings() + self.commands() -def tool_names(tree): - names = [] - for node in ast.walk(tree): - if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "tool_schema" and node.args: - arg = node.args[0] - if isinstance(arg, ast.Constant) and isinstance(arg.value, str): - names.append((arg.value, node.lineno)) - return names +class CppIndex: + """Normalized identifiers found in the C++ tree.""" + def __init__(self): + self.idents = set() + files = [os.path.join(ROOT, "main.cpp")] + for base, _dirs, names in os.walk(os.path.join(ROOT, "src")): + files += [os.path.join(base, n) for n in names if n.endswith((".cpp", ".h", ".hpp"))] + for path in files: + if os.path.isfile(path): + with open(path, encoding="utf-8", errors="replace") as fh: + self.idents.update(norm(t) for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", fh.read())) -def flag_names(tree): - flags = [] - for node in ast.walk(tree): - if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument": - for arg in node.args: - if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and arg.value.startswith("--"): - flags.append((arg.value, node.lineno)) - return flags - - -def build(): - with open(PY, encoding="utf-8") as fh: - tree = ast.parse(fh.read()) - idents = cpp_idents() - rows = [] - for kind, name, line in python_symbols(tree): + def has(self, kind, name): + if kind in ("tool", "setting"): + return norm(name) in self.idents + if kind == "flag": + return norm(name) in self.idents + if kind == "command": + return norm(name.lstrip("/")) in self.idents and len(norm(name)) > 3 leaf = name.split(".")[-1] - hit = norm(name.replace(".", "")) in idents or (kind != "class" and len(norm(leaf)) > 5 and norm(leaf) in idents) - rows.append({"kind": kind, "name": name, "line": line, "status": "name-only" if hit else "missing"}) - for name, line in tool_names(tree): - rows.append({"kind": "tool", "name": name, "line": line, "status": "name-only" if norm(name) in idents else "missing"}) - for name, line in flag_names(tree): - rows.append({"kind": "flag", "name": name, "line": line, "status": "name-only" if norm(name) in idents else "missing"}) - return rows + if norm(name.replace(".", "")) in self.idents: + return True + return kind != "class" and len(norm(leaf)) > 5 and norm(leaf) in self.idents + + +class Evidence: + """Passing differential cases, and whether they may be trusted for the current files.""" + + def __init__(self): + self.path = os.path.join(OUT, "diff-results.json") + self.problems = [] + self.by_symbol = {} # "kind:name" -> {"cases": [ids], "facets": set} + self.cases = {} # id -> result dict + self.meta = None + if not os.path.isfile(self.path): + self.problems.append("no differential results (run tools/parity_diff.py)") + return + with open(self.path) as fh: + doc = json.load(fh) + self.meta = doc["meta"] + now_py, now_detz = sha256_file(PY), sha256_file(self.meta.get("detz_path", DETZ)) + if self.meta["tai_sha"] != now_py: + self.problems.append("differential results are stale: tai.py changed since the run") + if self.meta["detz_sha"] != now_detz: + self.problems.append("differential results are stale: detz binary changed or missing since the run") + if self.meta.get("no_llm"): + self.problems.append("last differential run skipped LLM cases (--no-llm)") + trusted = not any("stale" in p for p in self.problems) + for res in doc["results"]: + self.cases[res["id"]] = res + if res["status"] != "pass" or not trusted: + continue + for sym in res["symbols"]: + base, _, facet = sym.partition("#") + slot = self.by_symbol.setdefault(base, {"cases": [], "facets": set()}) + slot["cases"].append(res["id"]) + slot["facets"].add(facet or "all") + + +class Ledger: + def __init__(self): + self.py = PySymbols(PY) + self.cpp = CppIndex() + self.evidence = Evidence() + self.rows = [] + self.case_symbol_errors = [] + self._build() + + def _build(self): + known = set() + for kind, name, line in self.py.all(): + sid = "%s:%s" % (kind, name) + if sid in known: + continue + known.add(sid) + ev = self.evidence.by_symbol.get(sid) + if ev: + status, facets, cases = "verified", sorted(ev["facets"]), ev["cases"] + else: + status, facets, cases = ("name-only" if self.cpp.has(kind, name) else "missing"), [], [] + self.rows.append({"id": sid, "kind": kind, "name": name, "line": line, "status": status, "facets": facets, "cases": cases}) + for cid, res in self.evidence.cases.items(): + for sym in res["symbols"]: + if sym.partition("#")[0] not in known: + self.case_symbol_errors.append("case %s names unknown symbol %s" % (cid, sym)) + + def count(self, kind=None): + rows = [r for r in self.rows if kind is None or r["kind"] == kind] + tally = {"total": len(rows), "verified": 0, "name-only": 0, "missing": 0} + for r in rows: + tally[r["status"]] += 1 + return tally + + def problems(self): + out = list(self.evidence.problems) + self.case_symbol_errors + t = self.count() + if t["missing"]: + out.append("%d symbols missing" % t["missing"]) + if t["name-only"]: + out.append("%d symbols name-only (unverified)" % t["name-only"]) + return out + + +class Writer: + def __init__(self, ledger): + self.ledger = ledger + + def tally_table(self): + lines = ["| kind | total | verified | name-only | missing |", "|---|---|---|---|---|"] + for kind in KINDS + (None,): + t = self.ledger.count(kind) + lines.append("| %s | %d | %d | %d | %d |" % (kind or "**all**", t["total"], t["verified"], t["name-only"], t["missing"])) + return lines + + def parity_md(self): + led, ev = self.ledger, self.ledger.evidence + meta = ev.meta or {} + lines = ["# Parity detz vs tai.py (generated, do not edit)", "", + "Generated by `tools/parity_ledger.py` from the symbol ledger and the last differential run.", + "Only a passing differential case (`tools/parity_diff.py`) makes a symbol `verified`; a facet", + "after the symbol says what that case proved (for example `parse` means argument parsing only).", + "`name-only` means an identifier matches in the C++ tree and NOTHING about behavior is known.", "", + "- tai.py sha256: `%s`" % sha256_file(PY)[:16], + "- detz sha256: `%s`" % sha256_file(meta.get("detz_path", DETZ))[:16], + "- differential run: %s (tai.py `%s`, detz `%s`)" % (meta.get("ts", "none"), meta.get("tai_sha", "-")[:16], meta.get("detz_sha", "-")[:16]), ""] + if ev.problems: + lines += ["## Evidence problems", ""] + ["- " + p for p in ev.problems] + [""] + lines += ["## Summary", ""] + self.tally_table() + [""] + results = list(ev.cases.values()) + counts = {} + for r in results: + counts[r["status"]] = counts.get(r["status"], 0) + 1 + lines += ["## Differential cases", "", "Counts: " + (", ".join("%s=%d" % kv for kv in sorted(counts.items())) or "none"), ""] + failing = [r for r in results if r["status"] in ("fail", "xfail", "xpass")] + if failing: + lines += ["| case | status | reason |", "|---|---|---|"] + for r in sorted(failing, key=lambda x: x["id"]): + reason = r["expected_fail"] or (r["problems"][0] if r["problems"] else "") + lines.append("| %s | %s | %s |" % (r["id"], r["status"], reason.replace("|", "/")[:120])) + lines.append("") + tm = os.path.join(OUT, "tests-map.json") + if os.path.isfile(tm): + with open(tm) as fh: + cov = json.load(fh)["summary"] + lines += ["## Python test-suite coverage", "", + "%d test classes, %d tests; tests reproduced by a passing diff case: %d (in %d classes); classes with no covered test: %d." + % (cov["classes"], cov["tests"], cov["tests_covered"], cov["classes_covered"], cov["classes_uncovered"]), + "Full table: `build/parity/tests-map.md`.", ""] + for kind in ("flag", "tool", "setting", "command"): + rows = [r for r in led.rows if r["kind"] == kind] + lines += ["## %ss (%d)" % (kind, len(rows)), "", "| name | status | facets | cases |", "|---|---|---|---|"] + for r in sorted(rows, key=lambda x: x["name"]): + lines.append("| `%s` | %s | %s | %s |" % (r["name"], r["status"], ",".join(r["facets"]), ",".join(r["cases"][:3]))) + lines.append("") + miss = {} + for r in led.rows: + if r["kind"] == "method" and r["status"] == "missing": + miss.setdefault(r["name"].split(".")[0], 0) + miss[r["name"].split(".")[0]] += 1 + top = sorted(miss.items(), key=lambda kv: -kv[1])[:25] + lines += ["## Classes with the most missing methods (top 25 of %d)" % len(miss), "", "| class | missing methods |", "|---|---|"] + lines += ["| %s | %d |" % kv for kv in top] + lines.append("") + return "\n".join(lines) + + def ledger_md(self): + lines = ["# Parity ledger (generated, do not edit)", ""] + self.tally_table() + [""] + lines += ["| id | status | facets | cases |", "|---|---|---|---|"] + for r in self.ledger.rows: + lines.append("| `%s` | %s | %s | %s |" % (r["id"], r["status"], ",".join(r["facets"]), ",".join(r["cases"][:3]))) + return "\n".join(lines) + "\n" + + def write(self, publish): + os.makedirs(OUT, exist_ok=True) + with open(os.path.join(OUT, "ledger.json"), "w") as fh: + json.dump(self.ledger.rows, fh, indent=1) + with open(os.path.join(OUT, "ledger.md"), "w") as fh: + fh.write(self.ledger_md()) + text = self.parity_md() + with open(os.path.join(OUT, "PARITY.md"), "w") as fh: + fh.write(text) + if publish: + with open(PARITY_MD, "w") as fh: + fh.write(text) def main(): - rows = build() - os.makedirs(OUT, exist_ok=True) - with open(os.path.join(OUT, "ledger.json"), "w") as fh: - json.dump(rows, fh, indent=1) - kinds = sorted({r["kind"] for r in rows}) - lines = ["# Parity ledger (generated, do not edit)", "", - "Source: tai.py. `name-only` means an identifier matches, NOT that behavior is verified.", "", - "| kind | total | name-only | missing |", "|---|---|---|---|"] - for kind in kinds: - sub = [r for r in rows if r["kind"] == kind] - have = sum(1 for r in sub if r["status"] == "name-only") - lines.append("| %s | %d | %d | %d |" % (kind, len(sub), have, len(sub) - have)) - total_missing = sum(1 for r in rows if r["status"] == "missing") - lines += ["", "Total symbols: %d, missing: %d, verified: 0" % (len(rows), total_missing)] - with open(os.path.join(OUT, "ledger.md"), "w") as fh: - fh.write("\n".join(lines) + "\n") - print("\n".join(lines)) - return 1 if "--check" in sys.argv and total_missing else 0 + ledger = Ledger() + Writer(ledger).write("--publish" in sys.argv) + print("\n".join(Writer(ledger).tally_table())) + problems = ledger.problems() + for p in problems: + print("PROBLEM: %s" % p) + if "--check" in sys.argv: + if problems: + print("parity-check: FAIL (%d problems)" % len(problems)) + return 1 + print("parity-check: OK") + return 0 if __name__ == "__main__": diff --git a/taicpp/tools/parity_tests.py b/taicpp/tools/parity_tests.py new file mode 100755 index 0000000..5ce0057 --- /dev/null +++ b/taicpp/tools/parity_tests.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Mine ../test_tai.py into a coverage table: test class -> tai.py symbols -> diff case or `uncovered`. + +Usage: tools/parity_tests.py (writes build/parity/tests-map.json and tests-map.md) + +For every unittest class in test_tai.py (ast, never imported): the number of tests, and the +tai.py symbols it touches (`tai.X`, `tai.X.y`, and `"tai.X.y"` mock.patch strings), resolved +against the ledger ids (class:X, function:X, method:X.y; module data shows as data:X). + +A test (or whole class) maps to a differential case ONLY when the case lists `Class` or `Class.test_name` in its `tests` field +AND that case passed in the last unfiltered run. Everything else is `uncovered`. Listing a +class in a case means the case reproduces what those tests assert through the real binaries; +it is a claim the reviewer can read in the case file, not something inferred here. +""" +import ast +import json +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import parity_ledger as pl # noqa: E402 + +TEST_FILE = os.path.join(pl.REPO, "test_tai.py") + + +class TestMiner: + def __init__(self, path, symbol_ids): + with open(path, encoding="utf-8") as fh: + self.tree = ast.parse(fh.read()) + self.ids = symbol_ids + + def classes(self): + out = [] + for node in self.tree.body: + if isinstance(node, ast.ClassDef): + tests = [n for n in node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name.startswith("test")] + if tests: + out.append({"class": node.name, "line": node.lineno, "tests": len(tests), "names": [t.name for t in tests], "symbols": self.symbols(node)}) + return out + + def symbols(self, cls): + found = set() + for node in ast.walk(cls): + if isinstance(node, ast.Attribute): + inner = node.value + if isinstance(inner, ast.Name) and inner.id == "tai": + found.add(node.attr) + elif isinstance(inner, ast.Attribute) and isinstance(inner.value, ast.Name) and inner.value.id == "tai": + found.add("%s.%s" % (inner.attr, node.attr)) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + for m in re.finditer(r"\btai\.([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?)\b", node.value): + found.add(m.group(1)) + return sorted({self.resolve(n) for n in found}) + + def resolve(self, name): + for kind in ("method", "class", "function"): + sid = "%s:%s" % (kind, name) + if sid in self.ids: + return sid + head = name.split(".")[0] + if "." in name and "class:%s" % head in self.ids: + return "class:%s" % head # attribute or nested name on a known class + return "data:%s" % name + + +def main(): + ledger = pl.Ledger() + ids = {r["id"] for r in ledger.rows} + classes = TestMiner(TEST_FILE, ids).classes() + ev = ledger.evidence + by_test = {} # "Class" or "Class.test" -> passing case ids + for cid, res in ev.cases.items(): + if res["status"] == "pass": + for tc in res["tests"]: + by_test.setdefault(tc, []).append(cid) + errors = [] + names = {c["class"]: set(c["names"]) for c in classes} + for cid, res in ev.cases.items(): + for tc in res["tests"]: + cls, _, test = tc.partition(".") + if cls not in names or (test and test not in names[cls]): + errors.append("case %s names unknown test %s" % (cid, tc)) + for c in classes: + whole = by_test.get(c["class"], []) + per = {n: sorted(set(whole + by_test.get("%s.%s" % (c["class"], n), []))) for n in c["names"]} + c["covered_tests"] = sum(1 for v in per.values() if v) + c["cases"] = sorted({cid for v in per.values() for cid in v}) + c["status"] = "uncovered" if not c["covered_tests"] else ("covered" if c["covered_tests"] == c["tests"] else "partial") + total_tests = sum(c["tests"] for c in classes) + cov = [c for c in classes if c["covered_tests"]] + summary = {"classes": len(classes), "tests": total_tests, "classes_covered": len(cov), + "tests_covered": sum(c["covered_tests"] for c in classes), "classes_uncovered": len(classes) - len(cov)} + os.makedirs(pl.OUT, exist_ok=True) + with open(os.path.join(pl.OUT, "tests-map.json"), "w") as fh: + json.dump({"summary": summary, "classes": classes, "errors": sorted(set(errors))}, fh, indent=1) + lines = ["# test_tai.py coverage map (generated)", "", + "%(classes)d classes, %(tests)d tests. Tests reproduced by a passing diff case: %(tests_covered)d, in %(classes_covered)d classes. Fully uncovered classes: %(classes_uncovered)d." % summary, "", + "| test class | tests | covered | status | diff cases | tai.py symbols |", "|---|---|---|---|---|---|"] + for c in sorted(classes, key=lambda x: (x["status"], -x["tests"], x["class"])): + syms = ", ".join(c["symbols"][:8]) + (" ... +%d" % (len(c["symbols"]) - 8) if len(c["symbols"]) > 8 else "") + lines.append("| %s | %d | %d | %s | %s | %s |" % (c["class"], c["tests"], c["covered_tests"], c["status"], ",".join(c["cases"]) or "uncovered", syms)) + if errors: + lines += ["", "## Errors", ""] + ["- " + e for e in sorted(set(errors))] + with open(os.path.join(pl.OUT, "tests-map.md"), "w") as fh: + fh.write("\n".join(lines) + "\n") + print("test classes %(classes)d, tests %(tests)d, covered classes %(classes_covered)d (tests %(tests_covered)d), uncovered classes %(classes_uncovered)d" % summary) + for e in sorted(set(errors)): + print("ERROR:", e) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main())