taicpp parity: differential harness, ledger-generated PARITY.md with verified status, test_tai.py coverage map, seed cases
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,305 @@
|
||||
# Parity detz vs tai.py (generated, do not edit)
|
||||
|
||||
Generated by `tools/parity_ledger.py` from the symbol ledger and the last differential run.
|
||||
Only a passing differential case (`tools/parity_diff.py`) makes a symbol `verified`; a facet
|
||||
after the symbol says what that case proved (for example `parse` means argument parsing only).
|
||||
`name-only` means an identifier matches in the C++ tree and NOTHING about behavior is known.
|
||||
|
||||
- tai.py sha256: `7e817bba8d48f4ab`
|
||||
- detz sha256: `a7a705c9b8bd5ed0`
|
||||
- differential run: 2026-10-25T01:20:14 (tai.py `7e817bba8d48f4ab`, detz `a7a705c9b8bd5ed0`)
|
||||
|
||||
## Summary
|
||||
|
||||
| kind | total | verified | name-only | missing |
|
||||
|---|---|---|---|---|
|
||||
| class | 264 | 0 | 28 | 236 |
|
||||
| method | 2140 | 0 | 109 | 2031 |
|
||||
| function | 413 | 0 | 25 | 388 |
|
||||
| tool | 61 | 0 | 61 | 0 |
|
||||
| flag | 45 | 3 | 42 | 0 |
|
||||
| setting | 28 | 0 | 13 | 15 |
|
||||
| command | 31 | 0 | 24 | 7 |
|
||||
| **all** | 2982 | 3 | 302 | 2677 |
|
||||
|
||||
## Differential cases
|
||||
|
||||
Counts: fail=54, pass=9, xfail=1
|
||||
|
||||
| case | status | reason |
|
||||
|---|---|---|
|
||||
| boot.vault | fail | vault differs (44 diff lines) |
|
||||
| cli.cap_manifest | fail | stdout differs (61 diff lines) |
|
||||
| cli.err.ambiguous.cap | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.ambiguous.debug | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.ambiguous.install | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.ambiguous.tui | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.auto.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.cap-cwd.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.cap-manifest.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.cap-max-turns.int | fail | stderr differs (19 diff lines) |
|
||||
| cli.err.cap-max-turns.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.cap-model.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.cap-permission-mode.choice | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.cap-permission-mode.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.cap-speak.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.continue.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.debug-format.choice | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.debug-format.missing | fail | stderr differs (17 diff lines) |
|
||||
| cli.err.debug.choice | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.demo-shot.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.demo.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.env.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.error_before_help | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.eval-retrieval.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.install-telegram.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.install.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.mcp-bind.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.mcp-port.int | fail | stderr differs (19 diff lines) |
|
||||
| cli.err.mcp-port.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.no-opencode.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.plexer-wrap.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.profile.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.scheduler.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.serve-mcp.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.stats-rag.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.telegram.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.theme.missing | fail | stdout differs (12 diff lines) |
|
||||
| cli.err.tui-local.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.tui.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.uninstall-telegram.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.unknown.long | fail | stderr differs (17 diff lines) |
|
||||
| cli.err.unknown.short | fail | stderr differs (17 diff lines) |
|
||||
| cli.err.update-models.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.version.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.yes.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.err.yolo.explicit | fail | stderr differs (18 diff lines) |
|
||||
| cli.help.wins_over_error | fail | stdout differs (87 diff lines) |
|
||||
| cli.prefix.version | fail | stdout differs (5 diff lines) |
|
||||
| persist.restart_history | xfail | needs the foundation agent: vault events persistence and history resume across process restarts; detz has no REPL/agent |
|
||||
| repl.help | fail | stdout differs (123 diff lines) |
|
||||
| repl.profiles | fail | stdout differs (94 diff lines) |
|
||||
| repl.skills_secret_schedules | fail | stdout differs (103 diff lines) |
|
||||
| repl.tools | fail | stdout differs (139 diff lines) |
|
||||
| repl.tools_named | fail | stdout differs (92 diff lines) |
|
||||
| repl.unknown_command | fail | stdout differs (89 diff lines) |
|
||||
|
||||
## Python test-suite coverage
|
||||
|
||||
150 test classes, 1233 tests; tests reproduced by a passing diff case: 0 (in 0 classes); classes with no covered test: 150.
|
||||
Full table: `build/parity/tests-map.md`.
|
||||
|
||||
## flags (45)
|
||||
|
||||
| name | status | facets | cases |
|
||||
|---|---|---|---|
|
||||
| `--allow-tool` | name-only | | |
|
||||
| `--argv` | name-only | | |
|
||||
| `--auto` | name-only | | |
|
||||
| `--auto-answer` | name-only | | |
|
||||
| `--binary` | name-only | | |
|
||||
| `--cap` | name-only | | |
|
||||
| `--cap-cwd` | name-only | | |
|
||||
| `--cap-manifest` | name-only | | |
|
||||
| `--cap-max-turns` | name-only | | |
|
||||
| `--cap-model` | name-only | | |
|
||||
| `--cap-permission-mode` | verified | parse | cli.version.with_choice,cli.ok.cap-permission-mode.none,cli.ok.cap-permission-mode.confirm |
|
||||
| `--cap-speak` | name-only | | |
|
||||
| `--continue` | name-only | | |
|
||||
| `--cwd` | name-only | | |
|
||||
| `--debug` | name-only | | |
|
||||
| `--debug-format` | verified | parse | cli.ok.debug-format.sentence,cli.ok.debug-format.json |
|
||||
| `--demo` | name-only | | |
|
||||
| `--demo-shot` | name-only | | |
|
||||
| `--deny-tool` | name-only | | |
|
||||
| `--driver` | name-only | | |
|
||||
| `--env` | name-only | | |
|
||||
| `--eval-retrieval` | name-only | | |
|
||||
| `--install` | name-only | | |
|
||||
| `--install-telegram` | name-only | | |
|
||||
| `--max-risk` | name-only | | |
|
||||
| `--mcp-bind` | name-only | | |
|
||||
| `--mcp-port` | name-only | | |
|
||||
| `--model` | name-only | | |
|
||||
| `--mplex` | name-only | | |
|
||||
| `--no-opencode` | name-only | | |
|
||||
| `--plexer-wrap` | name-only | | |
|
||||
| `--profile` | name-only | | |
|
||||
| `--scheduler` | name-only | | |
|
||||
| `--serve-mcp` | name-only | | |
|
||||
| `--stats-rag` | name-only | | |
|
||||
| `--task` | name-only | | |
|
||||
| `--telegram` | name-only | | |
|
||||
| `--theme` | name-only | | |
|
||||
| `--tui` | name-only | | |
|
||||
| `--tui-local` | name-only | | |
|
||||
| `--uninstall-telegram` | name-only | | |
|
||||
| `--update-models` | name-only | | |
|
||||
| `--version` | verified | output | cli.version.long |
|
||||
| `--yes` | name-only | | |
|
||||
| `--yolo` | name-only | | |
|
||||
|
||||
## tools (61)
|
||||
|
||||
| name | status | facets | cases |
|
||||
|---|---|---|---|
|
||||
| `ask_user` | name-only | | |
|
||||
| `audit` | name-only | | |
|
||||
| `create_bot` | name-only | | |
|
||||
| `create_skill` | name-only | | |
|
||||
| `delete_file` | name-only | | |
|
||||
| `delete_secret` | name-only | | |
|
||||
| `describe_image` | name-only | | |
|
||||
| `edit_file` | name-only | | |
|
||||
| `export_terminal_log` | name-only | | |
|
||||
| `fork` | name-only | | |
|
||||
| `get_current_terminal_content` | name-only | | |
|
||||
| `graph_link` | name-only | | |
|
||||
| `graph_query` | name-only | | |
|
||||
| `install` | name-only | | |
|
||||
| `install_skill` | name-only | | |
|
||||
| `list_secrets` | name-only | | |
|
||||
| `list_terminals` | name-only | | |
|
||||
| `listen` | name-only | | |
|
||||
| `load_skill` | name-only | | |
|
||||
| `mcp_add` | name-only | | |
|
||||
| `mcp_call` | name-only | | |
|
||||
| `mcp_disable` | name-only | | |
|
||||
| `mcp_enable` | name-only | | |
|
||||
| `mcp_list` | name-only | | |
|
||||
| `mcp_login` | name-only | | |
|
||||
| `mcp_remove` | name-only | | |
|
||||
| `mcp_tools` | name-only | | |
|
||||
| `plexer_read` | name-only | | |
|
||||
| `plexer_send` | name-only | | |
|
||||
| `plexer_show` | name-only | | |
|
||||
| `plexer_status` | name-only | | |
|
||||
| `plexer_type` | name-only | | |
|
||||
| `poll` | name-only | | |
|
||||
| `read_file` | name-only | | |
|
||||
| `recall` | name-only | | |
|
||||
| `record_delete` | name-only | | |
|
||||
| `record_read` | name-only | | |
|
||||
| `record_save` | name-only | | |
|
||||
| `record_search` | name-only | | |
|
||||
| `release` | name-only | | |
|
||||
| `remember` | name-only | | |
|
||||
| `restore` | name-only | | |
|
||||
| `schedule` | name-only | | |
|
||||
| `schedules` | name-only | | |
|
||||
| `search` | name-only | | |
|
||||
| `shell` | name-only | | |
|
||||
| `speak` | name-only | | |
|
||||
| `store_secret` | name-only | | |
|
||||
| `sysinfo` | name-only | | |
|
||||
| `tags` | name-only | | |
|
||||
| `tail_terminals` | name-only | | |
|
||||
| `tui_click` | name-only | | |
|
||||
| `tui_goto` | name-only | | |
|
||||
| `tui_keys` | name-only | | |
|
||||
| `tui_result` | name-only | | |
|
||||
| `tui_screen` | name-only | | |
|
||||
| `tui_type` | name-only | | |
|
||||
| `unschedule` | name-only | | |
|
||||
| `web_fetch` | name-only | | |
|
||||
| `web_search` | name-only | | |
|
||||
| `write_file` | name-only | | |
|
||||
|
||||
## settings (28)
|
||||
|
||||
| name | status | facets | cases |
|
||||
|---|---|---|---|
|
||||
| `backends` | name-only | | |
|
||||
| `box_caps` | missing | | |
|
||||
| `claude_bin` | missing | | |
|
||||
| `codex_bin` | missing | | |
|
||||
| `codex_sandbox` | missing | | |
|
||||
| `devplace_base` | name-only | | |
|
||||
| `devplace_key` | name-only | | |
|
||||
| `devplace_models` | name-only | | |
|
||||
| `gemini_bin` | missing | | |
|
||||
| `gemini_plan` | missing | | |
|
||||
| `grok_bin` | missing | | |
|
||||
| `model` | name-only | | |
|
||||
| `muse_bin` | missing | | |
|
||||
| `opencode_bin` | missing | | |
|
||||
| `opencode_client_name` | missing | | |
|
||||
| `opencode_client_ua` | missing | | |
|
||||
| `opencode_enabled` | missing | | |
|
||||
| `opencode_models` | name-only | | |
|
||||
| `openrouter_enabled` | missing | | |
|
||||
| `openrouter_key` | name-only | | |
|
||||
| `openrouter_models` | name-only | | |
|
||||
| `pollinations_enabled` | missing | | |
|
||||
| `pollinations_models` | name-only | | |
|
||||
| `retry_seconds` | name-only | | |
|
||||
| `theme` | name-only | | |
|
||||
| `voice` | name-only | | |
|
||||
| `zen_enabled` | missing | | |
|
||||
| `zen_models` | name-only | | |
|
||||
|
||||
## commands (31)
|
||||
|
||||
| name | status | facets | cases |
|
||||
|---|---|---|---|
|
||||
| `/agent` | name-only | | |
|
||||
| `/agents` | missing | | |
|
||||
| `/audit` | name-only | | |
|
||||
| `/bot` | missing | | |
|
||||
| `/bots` | name-only | | |
|
||||
| `/clear` | name-only | | |
|
||||
| `/compact` | missing | | |
|
||||
| `/ctx` | missing | | |
|
||||
| `/demo` | name-only | | |
|
||||
| `/env` | missing | | |
|
||||
| `/fork` | name-only | | |
|
||||
| `/graph` | name-only | | |
|
||||
| `/help` | name-only | | |
|
||||
| `/install` | name-only | | |
|
||||
| `/models` | name-only | | |
|
||||
| `/profile` | name-only | | |
|
||||
| `/profiles` | missing | | |
|
||||
| `/quit` | missing | | |
|
||||
| `/record` | name-only | | |
|
||||
| `/records` | name-only | | |
|
||||
| `/release` | name-only | | |
|
||||
| `/restore` | name-only | | |
|
||||
| `/schedule` | name-only | | |
|
||||
| `/schedules` | name-only | | |
|
||||
| `/search` | name-only | | |
|
||||
| `/secret` | name-only | | |
|
||||
| `/skills` | name-only | | |
|
||||
| `/sysinfo` | name-only | | |
|
||||
| `/tags` | name-only | | |
|
||||
| `/tools` | name-only | | |
|
||||
| `/unschedule` | name-only | | |
|
||||
|
||||
## Classes with the most missing methods (top 25 of 241)
|
||||
|
||||
| class | missing methods |
|
||||
|---|---|
|
||||
| MplexCopyMode | 128 |
|
||||
| MplexCommands | 92 |
|
||||
| Store | 90 |
|
||||
| MplexServer | 87 |
|
||||
| Tools | 85 |
|
||||
| TuiApp | 62 |
|
||||
| GrrChatView | 55 |
|
||||
| TuiPanel | 50 |
|
||||
| GrrShell | 50 |
|
||||
| CapAcpDriver | 44 |
|
||||
| TuiVtScreen | 35 |
|
||||
| ChatClient | 28 |
|
||||
| Agent | 26 |
|
||||
| CapA2aDriver | 24 |
|
||||
| McpOAuth | 21 |
|
||||
| MplexPane | 20 |
|
||||
| TuiCompositor | 20 |
|
||||
| TuiTextBox | 19 |
|
||||
| CapPtyDriver | 19 |
|
||||
| CapFleet | 19 |
|
||||
| GrrPage | 18 |
|
||||
| CapDriver | 17 |
|
||||
| Glow | 16 |
|
||||
| DebugLog | 16 |
|
||||
| TuiTerminal | 16 |
|
||||
@@ -0,0 +1,126 @@
|
||||
# parity: how detz is proven equal to tai.py
|
||||
|
||||
Goal: detz behaves exactly like `../tai.py`. This document is the spec of the proof
|
||||
machinery. It states what exists today; gaps are listed at the end.
|
||||
|
||||
## Pipeline
|
||||
|
||||
`tools/parity.sh run|check` runs, in order:
|
||||
|
||||
1. `tools/parity_gen_cases.py` - regenerates `tests/diff/cases/cli_generated.json` from
|
||||
`build_parser()` in tai.py (every argparse error and choice message). `--check` fails if stale.
|
||||
2. `tools/parity_diff.py` - the differential harness (below).
|
||||
3. `tools/parity_tests.py` - mines `../test_tai.py` into `build/parity/tests-map.{json,md}`.
|
||||
4. `tools/parity_ledger.py --publish` - builds the symbol ledger and writes `PARITY.md`
|
||||
(generated, never hand-edited). `--check` makes the pipeline exit 1 while anything is
|
||||
missing or unverified.
|
||||
|
||||
All output goes to `build/parity/`; only `PARITY.md` is published to the tree.
|
||||
|
||||
## Symbol statuses
|
||||
|
||||
| status | meaning | who sets it |
|
||||
|---|---|---|
|
||||
| missing | no identifier in `src/` or `main.cpp` matches the normalized name | ledger scan |
|
||||
| name-only | an identifier matches; behavior unknown | ledger scan |
|
||||
| verified | a passing differential case names the symbol | harness result only |
|
||||
|
||||
`verified` is never inferred. A case names a symbol as `kind:name` or `kind:name#facet`
|
||||
(`#parse`, `#output`, `#listed`, ...). The facet is printed in PARITY.md, so "verified" always
|
||||
says what was proven (`flag:--debug-format#parse` is argument parsing, not the flag's effect).
|
||||
Evidence is rejected as stale when the sha256 of tai.py or of the detz binary differs from
|
||||
the one recorded in `diff-results.json`, and a filtered run (`--filter`) writes
|
||||
`diff-results-partial.json`, which the ledger ignores. A case naming a symbol the ledger does not
|
||||
know is an error. Kinds: class, method, function, tool, flag, setting, command (a `/command` of
|
||||
`HELP_TEXT`).
|
||||
|
||||
## Case format (`tests/diff/cases/*.json`)
|
||||
|
||||
```json
|
||||
{"cases": [{
|
||||
"id": "repl.help",
|
||||
"doc": "what this proves",
|
||||
"symbols": ["command:/help"],
|
||||
"symbols_from_stdout": [{"prefix": "tool:", "pattern": "^ ([a-z_]+)$", "facet": "listed", "step": 0}],
|
||||
"tests": ["CapCliTests.test_cap_manifest_flag_prints_valid_toml"],
|
||||
"steps": [{"argv": [], "stdin": "", "env": {}, "files": {"rel/in/cwd": "text"},
|
||||
"home_files": {"rel/in/home": "text"}, "mode": "pipe|pty", "timeout": 60}],
|
||||
"compare": {"stdout": true, "stderr": true, "exit": true, "vault": false,
|
||||
"files": [{"root": "home|cwd", "glob": "**/*"}],
|
||||
"vault_ignore_tables": [], "vault_ignore_why": ""},
|
||||
"normalize": [{"on": ["stdout"], "pattern": "regex", "replace": "<X>", "dotall": false, "why": "required"}],
|
||||
"llm": false, "attempts": 1, "min_pass": 1, "checks": [], "expected_fail": "reason"
|
||||
}]}
|
||||
```
|
||||
|
||||
Semantics:
|
||||
|
||||
- Each side (`python3 ../tai.py`, `build/detz`) runs every step in order, sharing one throwaway
|
||||
`TAI_HOME` and cwd per side (`build/parity/work/<case>/<side>-<n>/`). `HOME` is a third
|
||||
throwaway dir so provider CLIs never touch the real one. Inherited `TAI_*`, `CAP_*`, `TMUX*`
|
||||
variables are dropped (they leak the caller's session); `COLUMNS=80`, `LINES=24`,
|
||||
`TERM=dumb`, `TAI_PASSPHRASE=` are set. A step's `env` overrides.
|
||||
- Compared channels: stdout, stderr, exit code per step, vault, declared files. `mode: pty` uses
|
||||
`tai_test.ManagedProcess` (stderr is merged into stdout there).
|
||||
- Vault = a deterministic dump of every `*.db` under the home: table DDL, rows sorted as JSON,
|
||||
index names. sqlite internals and `fts_docs_*` shadow tables are skipped globally. Anything
|
||||
else skipped must be listed in `vault_ignore_tables` with `vault_ignore_why`.
|
||||
- Normalization is per case, per channel, regex only, and every rule needs a `why`. There is
|
||||
no global scrubbing. Rules that matched nothing on either side are listed in the report
|
||||
as suspect.
|
||||
- `symbols_from_stdout` adds symbols read out of the reference's own stdout (for example every
|
||||
tool `/tools` prints); they count only if the case passes, so detz had to print the same names.
|
||||
- `tests` lists test_tai.py tests (`Class` or `Class.test_name`) whose assertions the case
|
||||
reproduces through the real binaries. This is a claim for a reviewer to read, not inferred.
|
||||
- `expected_fail` marks a known gap with its reason. It still counts as not verified and as
|
||||
unverified in `parity-check`; it only keeps the harness exit code at 0. If such a case
|
||||
passes it is reported `xpass` and fails the run, forcing the marker to be removed.
|
||||
|
||||
### LLM cases
|
||||
|
||||
`"llm": true` follows tests/README.md: a real model call on both sides, no mocks, no database
|
||||
writes standing in for the agent. Outputs are not equal-compared. Instead `checks` must hold
|
||||
on each side independently: `{"kind":"exit","step":0,"value":0}`,
|
||||
`{"kind":"stdout_regex","step":1,"pattern":"..."}` (also `stderr_regex`),
|
||||
`{"kind":"vault_sql","sql":"select count(*) ...","op":">=","value":2}`; add `"side":"ref"|"cpp"`
|
||||
to scope one. The free rotating backend is flaky by design: `attempts` (cap 5) and `min_pass`
|
||||
report a ratio per side (`2/3`) in the result instead of demanding 100 percent. `--no-llm`
|
||||
skips them; skipped cases are never verified.
|
||||
|
||||
## Caps
|
||||
|
||||
jobs 6, steps per case 8, step timeout 300 s, captured output 2 MB per stream, diff lines shown
|
||||
40 per channel, LLM attempts 5. A timed-out step kills the whole process group.
|
||||
|
||||
## Test-suite map
|
||||
|
||||
`build/parity/tests-map.md` lists every test class of `../test_tai.py` (150 classes, 1233
|
||||
tests when written), the tai.py symbols it touches (ast, from `tai.X`, `tai.X.y` and
|
||||
`"tai.X.y"` patch strings; names that are not code symbols show as `data:`), the number of its
|
||||
tests reproduced by a passing case, and the case ids or `uncovered`.
|
||||
|
||||
## Seed cases (what exists today)
|
||||
|
||||
- 56 generated cli cases: every missing-value, invalid-choice, invalid-int, explicit-value
|
||||
error, unknown flags, ambiguous prefixes, valid prefix, `--version`, `-h`, `--help`,
|
||||
`--cap-manifest`, each choice accepted.
|
||||
- REPL cases: `/help`, `/tools` (names), `/tools <name>`, unknown command, profiles and bots,
|
||||
empty listings, plus `boot.vault` (seeded settings catalog, schema, input_history).
|
||||
- `persist.restart_history`: LLM-driven restart persistence with the same `TAI_PLEXER_PANE`
|
||||
in both runs. Marked `expected_fail` until the foundation agent lands the store and history
|
||||
resume. On tai.py it passed 2 of 3 attempts on the first run (free model, flaky).
|
||||
|
||||
## Gaps (each with what closes it)
|
||||
|
||||
- Full tool schema JSON and `SETTINGS_CATALOG` entries are not exposed by any CLI flag, so only
|
||||
tool names (`/tools`) and the seeded settings rows (`boot.vault`) are diffable today. Closing
|
||||
it needs a read-only dump command in BOTH implementations, or a Python-side oracle (a script
|
||||
importing tai.py) plus a detz dump flag.
|
||||
- Startup banner lines (self-backup name, ctx id, live backend probes) are stripped by a
|
||||
declared per-case rule; they cannot be compared byte for byte.
|
||||
- tai.py snapshots itself into the vault on boot (audit, records, chunks, tags, fts_docs). detz
|
||||
has no such file; `boot.vault` excludes those tables on purpose and says so. Needs an owner
|
||||
decision, not a harness change.
|
||||
- Module constants, TUI rendering, mplex, grr need their own cases (pty replay); none exist.
|
||||
- Ledger "name-only" is a normalized-identifier match, weak by nature; it only separates
|
||||
"nothing written" from "something written".
|
||||
@@ -0,0 +1,24 @@
|
||||
# wiring: parity verification targets (for the integrator)
|
||||
|
||||
Add these to `taicpp/Makefile`. Nothing else in the tree needs to change; the tools live in
|
||||
`tools/` and write only under `build/parity/` (plus the generated `PARITY.md`).
|
||||
|
||||
```make
|
||||
parity: $(BIN)
|
||||
DETZ_BIN=$(abspath $(BIN)) tools/parity.sh run --detz $(abspath $(BIN))
|
||||
|
||||
parity-check: $(BIN)
|
||||
DETZ_BIN=$(abspath $(BIN)) tools/parity.sh check --detz $(abspath $(BIN))
|
||||
```
|
||||
|
||||
- `make parity` regenerates cases, runs the differential harness against `python3 ../tai.py`
|
||||
and `build/detz`, mines `../test_tai.py`, rebuilds the ledger and `PARITY.md`.
|
||||
Exit 1 if a non-expected case failed.
|
||||
- `make parity-check` does the same and exits non-zero while any symbol is missing or
|
||||
unverified, a case names an unknown symbol, or evidence is stale. Expect it to fail for a long time.
|
||||
- Extra arguments are passed to `tools/parity_diff.py`: `--no-llm`, `--filter TEXT`, `--jobs N`.
|
||||
Example: `tools/parity.sh run --no-llm --filter cli.err`. A filtered run
|
||||
never feeds the ledger (it writes `diff-results-partial.json`); a `--no-llm` run does feed it but parity-check reports it as a problem.
|
||||
- Requires python3 (stdlib only), network for LLM cases and for the REPL startup probes.
|
||||
Cases take about 4 s each because tai.py probes its backends on startup; `--jobs 6` is the cap.
|
||||
- Do not run `make parity-check` in a gate that must stay green until the symbol count says so.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,46 @@
|
||||
{"cases": [
|
||||
{"id": "boot.vault", "doc": "Fresh home, REPL started then EOF. Compares the seeded settings catalog rows, schema of every table, input_history. audit/records/chunks/tags/edges/shell_sessions are excluded because tai.py snapshots ITSELF into the vault on boot (path of tai.py, 50 KB source as a record); that is a deliberate per-binary difference, to be decided by the foundation agent, not hidden.",
|
||||
"symbols": ["function:ensure_schema#schema", "function:boot_settings#seed"],
|
||||
"steps": [{"argv": [], "stdin": "", "timeout": 60}],
|
||||
"compare": {"stdout": false, "stderr": true, "exit": true, "vault": true,
|
||||
"vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"],
|
||||
"vault_ignore_why": "tai.py self-snapshot (audit/records/chunks/tags/edges/fts_docs) and shell session tree (shell_sessions) are volatile per binary and host pid"},
|
||||
"normalize": [
|
||||
{"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"<TS>\"", "why": "created/updated/ts columns hold wall-clock ISO timestamps"}
|
||||
]},
|
||||
{"id": "repl.help", "doc": "/help text in the REPL", "symbols": ["command:/help"],
|
||||
"steps": [{"argv": [], "stdin": "/help\n", "timeout": 60}],
|
||||
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner holds self-backup file name with timestamp, ctx id with pid, and live backend probe results (model counts, latency ms)"}]},
|
||||
{"id": "repl.tools", "doc": "/tools lists core and lazy tool names", "symbols": ["command:/tools"],
|
||||
"symbols_from_stdout": [{"prefix": "tool:", "pattern": "^ ([a-z_]+)$", "facet": "listed", "step": 0}],
|
||||
"steps": [{"argv": [], "stdin": "/tools\n", "timeout": 60}],
|
||||
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
||||
{"id": "repl.tools_named", "doc": "/tools shell shows one tool", "symbols": ["command:/tools"],
|
||||
"steps": [{"argv": [], "stdin": "/tools shell\n/tools nosuchtool\n", "timeout": 60}],
|
||||
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
||||
{"id": "repl.unknown_command", "doc": "unknown slash command message", "symbols": [],
|
||||
"steps": [{"argv": [], "stdin": "/nonsense\n", "timeout": 60}],
|
||||
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
||||
{"id": "repl.profiles", "doc": "/profiles, /profile, /bots on a fresh home", "symbols": ["command:/profiles", "command:/profile", "command:/bots"],
|
||||
"steps": [{"argv": [], "stdin": "/profiles\n/profile\n/bots\n/profile work\n/profiles\n", "timeout": 60}],
|
||||
"compare": {"vault": true, "vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"], "vault_ignore_why": "tai.py self-snapshot and shell session tree, see boot.vault"},
|
||||
"normalize": [
|
||||
{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"},
|
||||
{"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"<TS>\"", "why": "wall-clock timestamps"}
|
||||
]},
|
||||
{"id": "repl.skills_secret_schedules", "doc": "listing commands on an empty vault", "symbols": ["command:/skills", "command:/secret", "command:/schedules", "command:/tags", "command:/records"],
|
||||
"steps": [{"argv": [], "stdin": "/skills\n/secret list\n/schedules\n/tags\n/records zebra\n", "timeout": 60}],
|
||||
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
||||
{"id": "persist.restart_history", "doc": "RESTART PERSISTENCE. Run 1 tells the agent a codeword (real LLM call), run 2 starts a new process with the SAME TAI_PLEXER_PANE and the same TAI_HOME and must resume the history: the events rows of run 1 survive and the model can answer from them. Expected to fail on detz until the foundation agent (store, history resume, plexer pane keyed session) lands. Probabilistic: real free model, 3 attempts, 1 must pass.",
|
||||
"llm": true, "attempts": 3, "min_pass": 1,
|
||||
"expected_fail": "needs the foundation agent: vault events persistence and history resume across process restarts; detz has no REPL/agent loop yet (2026-10-25)",
|
||||
"symbols": ["function:main#restart", "class:Store#events"],
|
||||
"steps": [
|
||||
{"argv": [], "stdin": "My secret codeword is heliotrope-7314. Just acknowledge it in one short sentence.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180},
|
||||
{"argv": [], "stdin": "What is my secret codeword? Answer with the codeword only.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180}
|
||||
],
|
||||
"checks": [
|
||||
{"kind": "vault_sql", "sql": "select count(*) from events where text like '%heliotrope-7314%'", "op": ">=", "value": 2},
|
||||
{"kind": "stdout_regex", "step": 1, "pattern": "heliotrope-7314"}
|
||||
]}
|
||||
]}
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# Parity pipeline. Usage: tools/parity.sh run|check [extra parity_diff.py args]
|
||||
# run regenerate cases, run the differential harness, mine test_tai.py, rebuild the ledger
|
||||
# and the committed PARITY.md. Exit 0 unless a non-expected case failed.
|
||||
# check same as run, then exit 1 while any symbol is missing or unverified, a case names an
|
||||
# unknown symbol, or the evidence is stale. This is `make parity-check`.
|
||||
# Outputs go to build/parity/ (PARITY.md is published to taicpp/PARITY.md by --publish).
|
||||
set -u
|
||||
cd "$(dirname "$0")/.." || exit 2
|
||||
mode="${1:-run}"
|
||||
shift || true
|
||||
rc=0
|
||||
python3 tools/parity_gen_cases.py --check || { echo "regenerating cli_generated.json"; python3 tools/parity_gen_cases.py || exit 2; }
|
||||
python3 tools/parity_diff.py "$@" || rc=1
|
||||
python3 tools/parity_tests.py || rc=1
|
||||
if [ "$mode" = "check" ]; then
|
||||
python3 tools/parity_ledger.py --publish --check || rc=1
|
||||
else
|
||||
python3 tools/parity_ledger.py --publish || rc=1
|
||||
fi
|
||||
exit $rc
|
||||
Executable
+602
@@ -0,0 +1,602 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Differential harness: run each case against tai.py and detz, compare.
|
||||
|
||||
Usage:
|
||||
tools/parity_diff.py [--detz PATH] [--py PATH] [--filter TEXT] [--no-llm]
|
||||
[--cases DIR] [--jobs N] [--list]
|
||||
|
||||
Every case runs twice, once per side, in its own throwaway TAI_HOME and cwd
|
||||
(never ~/.tai), with the real binaries and a real vault. Compared channels:
|
||||
stdout, stderr, exit code, vault state (sqlite dump of every *.db in the home),
|
||||
and declared files. Volatile data is normalized ONLY by rules declared in the
|
||||
case itself (each with a `why`); there is no blanket scrubbing. A rule that
|
||||
matched nothing on either side is reported as unused.
|
||||
|
||||
LLM cases (`"llm": true`) make a real model call on both sides (tests/README.md
|
||||
rule, no mocks). Their outputs are not equal-compared; both sides must each
|
||||
satisfy the case's `checks`. They are probabilistic: `attempts` / `min_pass`
|
||||
report a ratio, and the result carries it.
|
||||
|
||||
Writes build/parity/diff-results.json and diff-report.md. Exit 1 if any case
|
||||
that is not marked expected_fail failed, or an expected_fail case passed.
|
||||
See docs/parity.md for the case format.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import difflib
|
||||
import glob as globmod
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
REPO = os.path.dirname(ROOT)
|
||||
OUT = os.path.join(ROOT, "build", "parity")
|
||||
CASES_DIR = os.path.join(ROOT, "tests", "diff", "cases")
|
||||
|
||||
# Hard caps (nothing unbounded).
|
||||
MAX_JOBS = 6
|
||||
MAX_STEPS = 8
|
||||
MAX_STEP_TIMEOUT = 300
|
||||
MAX_DIFF_LINES = 40
|
||||
MAX_CAPTURE = 2_000_000
|
||||
MAX_ATTEMPTS = 5
|
||||
BASE_ENV = {"COLUMNS": "80", "LINES": "24", "TAI_PASSPHRASE": "", "LC_ALL": "C.UTF-8", "TERM": "dumb"}
|
||||
VAULT_SKIP_PREFIXES = ("sqlite_", "fts_docs_") # sqlite internals and fts shadow tables
|
||||
CHANNELS = ("stdout", "stderr", "exit", "vault", "files")
|
||||
|
||||
|
||||
class CaseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for block in iter(lambda: fh.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
class Rule:
|
||||
"""One declared normalization: regex -> replacement on named channels."""
|
||||
|
||||
def __init__(self, spec, case_id):
|
||||
for key in ("on", "pattern", "replace", "why"):
|
||||
if key not in spec:
|
||||
raise CaseError("%s: normalize rule missing %r" % (case_id, key))
|
||||
if not str(spec["why"]).strip():
|
||||
raise CaseError("%s: normalize rule needs a non-empty why" % case_id)
|
||||
self.on = list(spec["on"])
|
||||
for ch in self.on:
|
||||
if ch not in CHANNELS:
|
||||
raise CaseError("%s: normalize rule on unknown channel %r" % (case_id, ch))
|
||||
self.regex = re.compile(spec["pattern"], re.M | (re.S if spec.get("dotall") else 0))
|
||||
self.replace = spec["replace"]
|
||||
self.why = spec["why"]
|
||||
self.hits = {"ref": 0, "cpp": 0}
|
||||
|
||||
def apply(self, channel, text, side):
|
||||
if channel not in self.on:
|
||||
return text
|
||||
text, count = self.regex.subn(self.replace, text)
|
||||
self.hits[side] += count
|
||||
return text
|
||||
|
||||
|
||||
class Step:
|
||||
def __init__(self, spec, case_id):
|
||||
self.argv = [str(a) for a in spec.get("argv", [])]
|
||||
self.stdin = spec.get("stdin", "")
|
||||
self.env = {k: str(v) for k, v in spec.get("env", {}).items()}
|
||||
self.files = dict(spec.get("files", {}))
|
||||
self.home_files = dict(spec.get("home_files", {}))
|
||||
self.mode = spec.get("mode", "pipe")
|
||||
self.timeout = min(float(spec.get("timeout", 60)), MAX_STEP_TIMEOUT)
|
||||
if self.mode not in ("pipe", "pty"):
|
||||
raise CaseError("%s: unknown step mode %r" % (case_id, self.mode))
|
||||
|
||||
|
||||
class Case:
|
||||
def __init__(self, spec, source):
|
||||
self.id = spec["id"]
|
||||
self.source = source
|
||||
self.doc = spec.get("doc", "")
|
||||
self.symbols = list(spec.get("symbols", []))
|
||||
self.tests = list(spec.get("tests", []))
|
||||
self.llm = bool(spec.get("llm", False))
|
||||
self.expected_fail = spec.get("expected_fail")
|
||||
self.steps = [Step(s, self.id) for s in spec["steps"]]
|
||||
if not self.steps or len(self.steps) > MAX_STEPS:
|
||||
raise CaseError("%s: 1..%d steps required" % (self.id, MAX_STEPS))
|
||||
compare = spec.get("compare", {})
|
||||
self.compare = {
|
||||
"stdout": compare.get("stdout", True),
|
||||
"stderr": compare.get("stderr", True),
|
||||
"exit": compare.get("exit", True),
|
||||
"vault": compare.get("vault", False),
|
||||
"files": compare.get("files", []),
|
||||
}
|
||||
self.vault_ignore = list(compare.get("vault_ignore_tables", []))
|
||||
if self.vault_ignore and not str(compare.get("vault_ignore_why", "")).strip():
|
||||
raise CaseError("%s: vault_ignore_tables needs vault_ignore_why" % self.id)
|
||||
self.rule_specs = list(spec.get("normalize", []))
|
||||
for rs in self.rule_specs:
|
||||
Rule(rs, self.id) # validate at load
|
||||
self.checks = list(spec.get("checks", []))
|
||||
# Symbols named by the reference's own output (e.g. every tool /tools lists), claimed only
|
||||
# when the case passes, i.e. when detz printed the same thing.
|
||||
self.symbols_from_stdout = list(spec.get("symbols_from_stdout", []))
|
||||
self.attempts = min(int(spec.get("attempts", 1)), MAX_ATTEMPTS)
|
||||
self.min_pass = int(spec.get("min_pass", 1))
|
||||
if self.llm and not self.checks:
|
||||
raise CaseError("%s: llm case needs checks (outputs are not equal-compared)" % self.id)
|
||||
|
||||
def rules(self):
|
||||
return [Rule(rs, self.id) for rs in self.rule_specs]
|
||||
|
||||
|
||||
def load_cases(directory):
|
||||
cases, seen = [], {}
|
||||
for path in sorted(globmod.glob(os.path.join(directory, "*.json"))):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
doc = json.load(fh)
|
||||
for spec in doc["cases"]:
|
||||
case = Case(spec, os.path.basename(path))
|
||||
if case.id in seen:
|
||||
raise CaseError("duplicate case id %s (%s, %s)" % (case.id, seen[case.id], path))
|
||||
seen[case.id] = path
|
||||
cases.append(case)
|
||||
return cases
|
||||
|
||||
|
||||
class Side:
|
||||
"""One implementation under test."""
|
||||
|
||||
def __init__(self, name, command):
|
||||
self.name = name
|
||||
self.command = list(command)
|
||||
|
||||
|
||||
class Capture:
|
||||
"""Everything observed from running one case once on one side."""
|
||||
|
||||
def __init__(self):
|
||||
self.steps = [] # list of dict(argv, rc, stdout, stderr, timed_out)
|
||||
self.vault = ""
|
||||
self.files = ""
|
||||
self.sql = None # open read-only vault path for checks
|
||||
self.home = None
|
||||
self.cwd = None
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self, work_root):
|
||||
self.work_root = work_root
|
||||
|
||||
def run_step(self, side, step, home, cwd):
|
||||
for rel, content in step.files.items():
|
||||
self._write(os.path.join(cwd, rel), content)
|
||||
for rel, content in step.home_files.items():
|
||||
self._write(os.path.join(home, rel), content)
|
||||
# Hermetic environment: inherited TAI_*, CAP_*, TMUX* leak the caller's session into the
|
||||
# run (parent shell session, pane resolution), so they are dropped. Declared per step env wins.
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith(("TAI_", "CAP_", "TMUX"))}
|
||||
env.update(BASE_ENV)
|
||||
user_home = os.path.join(os.path.dirname(home), "user")
|
||||
os.makedirs(user_home, exist_ok=True)
|
||||
env.update({"TAI_HOME": home, "HOME": user_home})
|
||||
env.update(step.env)
|
||||
cmd = side.command + step.argv
|
||||
start = time.monotonic()
|
||||
if step.mode == "pty":
|
||||
return self._run_pty(cmd, step, env, cwd, start)
|
||||
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
cwd=cwd, env=env, start_new_session=True)
|
||||
timed_out = False
|
||||
try:
|
||||
out, err = proc.communicate(step.stdin.encode("utf-8"), timeout=step.timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
self._kill(proc)
|
||||
out, err = proc.communicate()
|
||||
return {"argv": step.argv, "rc": proc.returncode, "timed_out": timed_out,
|
||||
"stdout": out[:MAX_CAPTURE].decode("utf-8", "replace"),
|
||||
"stderr": err[:MAX_CAPTURE].decode("utf-8", "replace"),
|
||||
"secs": round(time.monotonic() - start, 2)}
|
||||
|
||||
def _run_pty(self, cmd, step, env, cwd, start):
|
||||
import asyncio
|
||||
sys.path.insert(0, REPO)
|
||||
import tai_test # the one base harness (tests/README.md); pty merges stderr into stdout
|
||||
|
||||
async def go():
|
||||
proc = tai_test.ManagedProcess(cmd, mode="pty", cwd=cwd, env=env)
|
||||
await proc.start()
|
||||
timed_out = False
|
||||
text = ""
|
||||
try:
|
||||
for line in step.stdin.splitlines():
|
||||
await proc.send_line(line)
|
||||
text = await proc.drain_until_eof(timeout=step.timeout)
|
||||
except tai_test.TestTimeout:
|
||||
timed_out = True
|
||||
text = "".join(l.text for l in proc.transcript if l.stream == "stdout")
|
||||
rc = await proc.terminate() if timed_out else await proc.wait(5)
|
||||
await proc.terminate()
|
||||
return text, rc, timed_out
|
||||
text, rc, timed_out = asyncio.run(go())
|
||||
return {"argv": step.argv, "rc": rc, "timed_out": timed_out, "stdout": tai_test.strip_ansi(text)[:MAX_CAPTURE],
|
||||
"stderr": "", "secs": round(time.monotonic() - start, 2)}
|
||||
|
||||
@staticmethod
|
||||
def _write(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
|
||||
@staticmethod
|
||||
def _kill(proc):
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
os.killpg(proc.pid, sig)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
return
|
||||
try:
|
||||
proc.wait(3)
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
|
||||
def capture(self, case, side, tag):
|
||||
base = os.path.join(self.work_root, case.id.replace("/", "_"), "%s-%s" % (side.name, tag))
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
home, cwd = os.path.join(base, "home"), os.path.join(base, "cwd")
|
||||
os.makedirs(home)
|
||||
os.makedirs(cwd)
|
||||
cap = Capture()
|
||||
cap.home, cap.cwd = home, cwd
|
||||
for step in case.steps:
|
||||
cap.steps.append(self.run_step(side, step, home, cwd))
|
||||
if case.compare["vault"]:
|
||||
cap.vault = VaultDump(home, case.vault_ignore).text()
|
||||
if case.compare["files"]:
|
||||
cap.files = FilesDump(home, cwd, case.compare["files"]).text()
|
||||
return cap
|
||||
|
||||
|
||||
class VaultDump:
|
||||
"""Deterministic text dump of every sqlite db under the home (read-only copy)."""
|
||||
|
||||
def __init__(self, home, ignore):
|
||||
self.home = home
|
||||
self.ignore = set(ignore)
|
||||
|
||||
def text(self):
|
||||
lines = []
|
||||
for base, _dirs, names in sorted(os.walk(self.home)):
|
||||
for name in sorted(names):
|
||||
if name.endswith(".db"):
|
||||
path = os.path.join(base, name)
|
||||
lines.append("# db %s" % os.path.relpath(path, self.home))
|
||||
lines.extend(self._dump(path))
|
||||
return "\n".join(lines) + ("\n" if lines else "")
|
||||
|
||||
def _dump(self, path):
|
||||
out = []
|
||||
tmp = tempfile.mkdtemp()
|
||||
try:
|
||||
copy = os.path.join(tmp, "v.db")
|
||||
shutil.copy(path, copy)
|
||||
for suffix in ("-wal", "-shm"):
|
||||
if os.path.exists(path + suffix):
|
||||
shutil.copy(path + suffix, copy + suffix)
|
||||
db = sqlite3.connect(copy)
|
||||
db.text_factory = lambda b: b.decode("utf-8", "replace")
|
||||
tables = [r for r in db.execute("SELECT name, sql FROM sqlite_master WHERE type='table' ORDER BY name")]
|
||||
for name, sql in tables:
|
||||
if name.startswith(VAULT_SKIP_PREFIXES) or name in self.ignore:
|
||||
continue
|
||||
out.append("## table %s :: %s" % (name, " ".join((sql or "").split())))
|
||||
rows = [json.dumps(list(r), ensure_ascii=False, default=repr) for r in db.execute('SELECT * FROM "%s"' % name)]
|
||||
out.extend(" " + r for r in sorted(rows))
|
||||
idx = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name")]
|
||||
out.append("## indexes %s" % ",".join(idx))
|
||||
db.close()
|
||||
except sqlite3.Error as exc:
|
||||
out.append("## unreadable: %s" % exc)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
return out
|
||||
|
||||
|
||||
class FilesDump:
|
||||
"""Path + content hash listing for declared globs, rooted at home or cwd."""
|
||||
|
||||
def __init__(self, home, cwd, specs):
|
||||
self.roots = {"home": home, "cwd": cwd}
|
||||
self.specs = specs
|
||||
|
||||
def text(self):
|
||||
lines = []
|
||||
for spec in self.specs:
|
||||
root = self.roots[spec["root"]]
|
||||
for path in sorted(globmod.glob(os.path.join(root, spec["glob"]), recursive=True)):
|
||||
if os.path.isfile(path):
|
||||
rel = "%s:%s" % (spec["root"], os.path.relpath(path, root))
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read(MAX_CAPTURE)
|
||||
try:
|
||||
body = data.decode("utf-8")
|
||||
lines.append("%s\t%s" % (rel, json.dumps(body, ensure_ascii=False)))
|
||||
except UnicodeDecodeError:
|
||||
lines.append("%s\tsha256:%s" % (rel, hashlib.sha256(data).hexdigest()))
|
||||
return "\n".join(sorted(lines)) + ("\n" if lines else "")
|
||||
|
||||
|
||||
class Normalizer:
|
||||
def __init__(self, case):
|
||||
self.rules = case.rules()
|
||||
|
||||
def apply(self, channel, text, side):
|
||||
for rule in self.rules:
|
||||
text = rule.apply(channel, text, side)
|
||||
return text
|
||||
|
||||
def unused(self):
|
||||
return [r.why for r in self.rules if r.hits["ref"] == 0 and r.hits["cpp"] == 0]
|
||||
|
||||
|
||||
class Checker:
|
||||
"""Evaluates `checks` of an llm case against one side's capture."""
|
||||
|
||||
def __init__(self, case):
|
||||
self.case = case
|
||||
|
||||
def evaluate(self, cap, side):
|
||||
fails = []
|
||||
for chk in self.case.checks:
|
||||
if chk.get("side", "both") not in ("both", side):
|
||||
continue
|
||||
kind = chk["kind"]
|
||||
if kind == "exit":
|
||||
got = cap.steps[chk["step"]]["rc"]
|
||||
if got != chk["value"]:
|
||||
fails.append("exit of step %d is %r, want %r" % (chk["step"], got, chk["value"]))
|
||||
elif kind in ("stdout_regex", "stderr_regex"):
|
||||
text = cap.steps[chk["step"]]["stdout" if kind == "stdout_regex" else "stderr"]
|
||||
if not re.search(chk["pattern"], text, re.S | re.I):
|
||||
fails.append("%s step %d: /%s/ not found" % (kind, chk["step"], chk["pattern"]))
|
||||
elif kind == "vault_sql":
|
||||
got = self._sql(cap, chk["sql"])
|
||||
if not self._cmp(got, chk["op"], chk["value"]):
|
||||
fails.append("vault_sql %r gave %r, want %s %r" % (chk["sql"], got, chk["op"], chk["value"]))
|
||||
else:
|
||||
fails.append("unknown check kind %r" % kind)
|
||||
return fails
|
||||
|
||||
@staticmethod
|
||||
def _cmp(got, op, want):
|
||||
try:
|
||||
return {"==": got == want, ">=": got >= want, "<=": got <= want, ">": got > want, "<": got < want}[op]
|
||||
except TypeError:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _sql(cap, sql):
|
||||
path = os.path.join(cap.home, "memory.db")
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
tmp = tempfile.mkdtemp()
|
||||
try:
|
||||
copy = os.path.join(tmp, "v.db")
|
||||
shutil.copy(path, copy)
|
||||
for suffix in ("-wal", "-shm"):
|
||||
if os.path.exists(path + suffix):
|
||||
shutil.copy(path + suffix, copy + suffix)
|
||||
db = sqlite3.connect(copy)
|
||||
row = db.execute(sql).fetchone()
|
||||
db.close()
|
||||
return row[0] if row else None
|
||||
except sqlite3.Error as exc:
|
||||
return "sqlerror: %s" % exc
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
class Verdict:
|
||||
def __init__(self, case):
|
||||
self.case = case
|
||||
self.status = "fail" # pass | fail | xfail | xpass | skipped
|
||||
self.problems = [] # human lines, capped
|
||||
self.ratio = None # llm: "ok/attempts" per side
|
||||
self.unused_rules = []
|
||||
self.extra_symbols = []
|
||||
self.secs = 0.0
|
||||
|
||||
def as_dict(self):
|
||||
return {"id": self.case.id, "status": self.status, "llm": self.case.llm, "symbols": self.case.symbols + self.extra_symbols,
|
||||
"tests": self.case.tests, "expected_fail": self.case.expected_fail, "problems": self.problems,
|
||||
"ratio": self.ratio, "unused_rules": self.unused_rules, "secs": round(self.secs, 1),
|
||||
"source": self.case.source}
|
||||
|
||||
|
||||
class CaseExecutor:
|
||||
def __init__(self, runner, ref, cpp):
|
||||
self.runner, self.ref, self.cpp = runner, ref, cpp
|
||||
|
||||
def execute(self, case):
|
||||
start = time.monotonic()
|
||||
verdict = Verdict(case)
|
||||
try:
|
||||
if case.llm:
|
||||
self._llm(case, verdict)
|
||||
else:
|
||||
self._equal(case, verdict)
|
||||
except Exception as exc: # harness fault is a failure, loudly
|
||||
verdict.problems.append("harness error: %r" % exc)
|
||||
passed = not verdict.problems
|
||||
if case.expected_fail:
|
||||
verdict.status = "xpass" if passed else "xfail"
|
||||
else:
|
||||
verdict.status = "pass" if passed else "fail"
|
||||
verdict.secs = time.monotonic() - start
|
||||
return verdict
|
||||
|
||||
def _equal(self, case, verdict):
|
||||
norm = Normalizer(case)
|
||||
caps = {"ref": self.runner.capture(case, self.ref, "a"), "cpp": self.runner.capture(case, self.cpp, "a")}
|
||||
for i, step in enumerate(case.steps):
|
||||
for side, cap in caps.items():
|
||||
if cap.steps[i]["timed_out"]:
|
||||
verdict.problems.append("step %d timed out on %s (%.0fs)" % (i, side, step.timeout))
|
||||
views = {}
|
||||
for side, cap in caps.items():
|
||||
view = {"exit": "\n".join(str(s["rc"]) for s in cap.steps) + "\n"}
|
||||
view["stdout"] = "".join("--- step %d\n%s" % (i, s["stdout"]) for i, s in enumerate(cap.steps))
|
||||
view["stderr"] = "".join("--- step %d\n%s" % (i, s["stderr"]) for i, s in enumerate(cap.steps))
|
||||
view["vault"] = cap.vault
|
||||
view["files"] = cap.files
|
||||
views[side] = {ch: norm.apply(ch, txt, side) for ch, txt in view.items()}
|
||||
for ch in CHANNELS:
|
||||
if not case.compare[ch]:
|
||||
continue
|
||||
if views["ref"][ch] != views["cpp"][ch]:
|
||||
verdict.problems.extend(self._diff(ch, views["ref"][ch], views["cpp"][ch]))
|
||||
verdict.unused_rules = norm.unused()
|
||||
for spec in case.symbols_from_stdout:
|
||||
text = views["ref"]["stdout"].split("--- step %d\n" % spec["step"], 1)[-1]
|
||||
names = re.findall(spec["pattern"], text, re.M)
|
||||
if not names:
|
||||
verdict.problems.append("symbols_from_stdout matched nothing for %s" % spec["prefix"])
|
||||
verdict.extra_symbols += ["%s%s#%s" % (spec["prefix"], n, spec["facet"]) for n in names]
|
||||
|
||||
@staticmethod
|
||||
def _diff(channel, ref, cpp):
|
||||
lines = list(difflib.unified_diff(ref.splitlines(), cpp.splitlines(), "tai.py:" + channel, "detz:" + channel, lineterm="", n=1))
|
||||
out = ["%s differs (%d diff lines)" % (channel, len(lines))]
|
||||
out.extend(lines[:MAX_DIFF_LINES])
|
||||
if len(lines) > MAX_DIFF_LINES:
|
||||
out.append("... %d more diff lines" % (len(lines) - MAX_DIFF_LINES))
|
||||
return out
|
||||
|
||||
def _llm(self, case, verdict):
|
||||
checker = Checker(case)
|
||||
ratios = {}
|
||||
for name, side in (("ref", self.ref), ("cpp", self.cpp)):
|
||||
ok, last = 0, []
|
||||
for attempt in range(case.attempts):
|
||||
cap = self.runner.capture(case, side, "t%d" % attempt)
|
||||
fails = checker.evaluate(cap, name)
|
||||
for i, s in enumerate(cap.steps):
|
||||
if s["timed_out"]:
|
||||
fails.append("step %d timed out" % i)
|
||||
if fails:
|
||||
last = fails
|
||||
else:
|
||||
ok += 1
|
||||
ratios[name] = "%d/%d" % (ok, case.attempts)
|
||||
if ok < case.min_pass:
|
||||
verdict.problems.append("%s: %d/%d attempts passed, need %d" % (name, ok, case.attempts, case.min_pass))
|
||||
verdict.problems.extend(" " + f for f in last[:MAX_DIFF_LINES])
|
||||
verdict.ratio = ratios
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self, verdicts, meta):
|
||||
self.verdicts, self.meta = verdicts, meta
|
||||
|
||||
def write(self):
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
suffix = "-partial" if self.meta["filtered"] else "" # a filtered run never feeds the ledger
|
||||
with open(os.path.join(OUT, "diff-results%s.json" % suffix), "w") as fh:
|
||||
json.dump({"meta": self.meta, "results": [v.as_dict() for v in self.verdicts]}, fh, indent=1)
|
||||
counts = {}
|
||||
for v in self.verdicts:
|
||||
counts[v.status] = counts.get(v.status, 0) + 1
|
||||
lines = ["# Differential report (generated)", "",
|
||||
"tai.py sha256 `%s`, detz sha256 `%s`" % (self.meta["tai_sha"][:16], self.meta["detz_sha"][:16]), "",
|
||||
"Counts: " + ", ".join("%s=%d" % kv for kv in sorted(counts.items())), "",
|
||||
"| case | status | llm | secs | note |", "|---|---|---|---|---|"]
|
||||
for v in sorted(self.verdicts, key=lambda x: x.case.id):
|
||||
note = (v.case.expected_fail or "") if v.status in ("xfail", "xpass") else ""
|
||||
if v.ratio:
|
||||
note += " ratio %s" % json.dumps(v.ratio)
|
||||
lines.append("| %s | %s | %s | %.1f | %s |" % (v.case.id, v.status, "yes" if v.case.llm else "", v.secs, note.strip()))
|
||||
lines += ["", "## Details of non-passing cases", ""]
|
||||
for v in sorted(self.verdicts, key=lambda x: x.case.id):
|
||||
if v.status in ("pass", "skipped"):
|
||||
continue
|
||||
lines += ["### %s (%s)" % (v.case.id, v.status), ""]
|
||||
if v.case.expected_fail:
|
||||
lines += ["expected_fail: " + v.case.expected_fail, ""]
|
||||
lines += ["```"] + v.problems[:MAX_DIFF_LINES * 3] + ["```", ""]
|
||||
unused = [(v.case.id, w) for v in self.verdicts for w in v.unused_rules]
|
||||
if unused:
|
||||
lines += ["## Unused normalization rules (suspect)", ""] + ["- %s: %s" % u for u in unused] + [""]
|
||||
with open(os.path.join(OUT, "diff-report%s.md" % suffix), "w") as fh:
|
||||
fh.write("\n".join(lines))
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--detz", default=os.environ.get("DETZ_BIN", os.path.join(ROOT, "build", "detz")))
|
||||
ap.add_argument("--py", default=os.environ.get("TAI_PY", os.path.join(REPO, "tai.py")))
|
||||
ap.add_argument("--cases", default=CASES_DIR)
|
||||
ap.add_argument("--filter", default=None)
|
||||
ap.add_argument("--no-llm", action="store_true")
|
||||
ap.add_argument("--jobs", type=int, default=4)
|
||||
ap.add_argument("--list", action="store_true")
|
||||
args = ap.parse_args()
|
||||
cases = load_cases(args.cases)
|
||||
if args.filter:
|
||||
cases = [c for c in cases if args.filter in c.id]
|
||||
if args.list:
|
||||
for c in cases:
|
||||
print(c.id, "llm" if c.llm else "", c.expected_fail or "")
|
||||
return 0
|
||||
if not os.path.isfile(args.py):
|
||||
print("tai.py not found: %s" % args.py, file=sys.stderr)
|
||||
return 2
|
||||
detz_ok = os.path.isfile(args.detz) and os.access(args.detz, os.X_OK)
|
||||
if not detz_ok:
|
||||
print("detz binary missing: %s (every case will fail; build it with `make`)" % args.detz, file=sys.stderr)
|
||||
work = os.path.join(OUT, "work")
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
runner = Runner(work)
|
||||
ref = Side("ref", [sys.executable, args.py])
|
||||
cpp = Side("cpp", [args.detz] if detz_ok else ["/nonexistent/detz"])
|
||||
executor = CaseExecutor(runner, ref, cpp)
|
||||
todo, verdicts = [], []
|
||||
for c in cases:
|
||||
if c.llm and args.no_llm:
|
||||
v = Verdict(c)
|
||||
v.status = "skipped"
|
||||
verdicts.append(v)
|
||||
else:
|
||||
todo.append(c)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(args.jobs, MAX_JOBS))) as pool:
|
||||
for v in pool.map(executor.execute, todo):
|
||||
print("%-7s %s" % (v.status.upper(), v.case.id), flush=True)
|
||||
verdicts.append(v)
|
||||
meta = {"tai_sha": sha256_file(args.py), "detz_sha": sha256_file(args.detz) if detz_ok else "missing",
|
||||
"detz_path": args.detz, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "filtered": bool(args.filter),
|
||||
"no_llm": args.no_llm}
|
||||
counts = Report(verdicts, meta).write()
|
||||
print("counts:", counts, "(report dir: %s)" % OUT)
|
||||
bad = counts.get("fail", 0) + counts.get("xpass", 0)
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate tests/diff/cases/cli_generated.json from tai.py's build_parser().
|
||||
|
||||
Every argparse error and choice message is exercised from the parser's own
|
||||
declaration (read with ast, tai.py untouched), so the case list cannot drift
|
||||
from the flags. Usage: tools/parity_gen_cases.py [--check]
|
||||
--check exits 1 if the committed file differs from what the parser implies.
|
||||
|
||||
Facet note: these cases verify argument PARSING only (messages, exit code),
|
||||
so they name symbols as `flag:--x#parse`, never the flag's behavior.
|
||||
"""
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
PY = os.path.join(os.path.dirname(ROOT), "tai.py")
|
||||
TARGET = os.path.join(ROOT, "tests", "diff", "cases", "cli_generated.json")
|
||||
|
||||
|
||||
class Opt:
|
||||
def __init__(self, call):
|
||||
self.flag = next(a.value for a in call.args if isinstance(a, ast.Constant) and isinstance(a.value, str))
|
||||
kw = {k.arg: k.value for k in call.keywords}
|
||||
self.action = ast.literal_eval(kw["action"]) if "action" in kw and isinstance(kw["action"], ast.Constant) else None
|
||||
self.type = kw["type"].id if "type" in kw and isinstance(kw["type"], ast.Name) else None
|
||||
self.choices = list(ast.literal_eval(kw["choices"])) if "choices" in kw else None
|
||||
nargs = kw.get("nargs")
|
||||
self.nargs = None
|
||||
if isinstance(nargs, ast.Constant):
|
||||
self.nargs = nargs.value
|
||||
elif isinstance(nargs, ast.Attribute):
|
||||
self.nargs = nargs.attr # REMAINDER
|
||||
|
||||
@property
|
||||
def takes_value(self):
|
||||
return self.action != "store_true" and self.nargs != "REMAINDER"
|
||||
|
||||
@property
|
||||
def optional_value(self):
|
||||
return self.nargs == "?"
|
||||
|
||||
|
||||
class CaseBuilder:
|
||||
def __init__(self):
|
||||
self.cases = []
|
||||
|
||||
def add(self, cid, doc, argv, flag, facet="parse"):
|
||||
self.cases.append({
|
||||
"id": cid, "doc": doc, "symbols": ["flag:%s#%s" % (flag, facet)] if flag else [],
|
||||
"steps": [{"argv": argv, "timeout": 30}],
|
||||
"compare": {"stdout": True, "stderr": True, "exit": True},
|
||||
})
|
||||
|
||||
|
||||
def options():
|
||||
with open(PY, encoding="utf-8") as fh:
|
||||
tree = ast.parse(fh.read())
|
||||
fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "build_parser")
|
||||
out = []
|
||||
for node in ast.walk(fn):
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument":
|
||||
if any(isinstance(a, ast.Constant) and isinstance(a.value, str) and a.value.startswith("--") for a in node.args):
|
||||
out.append(Opt(node))
|
||||
return out
|
||||
|
||||
|
||||
def build():
|
||||
b = CaseBuilder()
|
||||
opts = options()
|
||||
for o in opts:
|
||||
slug = o.flag[2:]
|
||||
if o.takes_value and not o.optional_value:
|
||||
b.add("cli.err.%s.missing" % slug, "%s with no value" % o.flag, [o.flag], o.flag)
|
||||
if o.choices:
|
||||
b.add("cli.err.%s.choice" % slug, "%s with a value outside its choices" % o.flag, [o.flag, "zzz-not-a-choice"], o.flag)
|
||||
if o.type == "int":
|
||||
b.add("cli.err.%s.int" % slug, "%s with a non-integer" % o.flag, [o.flag, "abc"], o.flag)
|
||||
if o.action == "store_true":
|
||||
b.add("cli.err.%s.explicit" % slug, "%s given an explicit value" % o.flag, ["%s=1" % o.flag], o.flag)
|
||||
# Prefix abbreviation: python argparse allows unique prefixes and rejects ambiguous ones.
|
||||
b.add("cli.err.unknown.long", "unknown long flag", ["--bogus"], None)
|
||||
b.add("cli.err.unknown.short", "unknown short flag", ["-z"], None)
|
||||
b.add("cli.err.ambiguous.debug", "--deb matches --debug and --debug-format", ["--deb"], "--debug")
|
||||
b.add("cli.err.ambiguous.cap", "--cap-m matches --cap-model and --cap-max-turns", ["--cap-m"], "--cap-model")
|
||||
b.add("cli.err.ambiguous.install", "--ins matches --install, --install-telegram", ["--ins"], "--install")
|
||||
b.add("cli.err.ambiguous.tui", "--tu matches --tui and --tui-local", ["--tu"], "--tui")
|
||||
b.add("cli.prefix.version", "unique prefix --vers resolves to --version", ["--vers"], "--version")
|
||||
b.add("cli.version.long", "--version", ["--version"], "--version", "output")
|
||||
b.add("cli.version.with_choice", "--version alongside a valid choice flag", ["--version", "--cap-permission-mode", "none"], "--cap-permission-mode")
|
||||
b.add("cli.help.long", "--help text, exact", ["--help"], None)
|
||||
b.add("cli.help.short", "-h text, exact", ["-h"], None)
|
||||
b.add("cli.help.wins_over_error", "-h before a bad flag prints help, exit 0", ["-h", "--bogus"], None)
|
||||
b.add("cli.err.error_before_help", "bad choice before -h is an error, exit 2", ["--cap-permission-mode", "zzz", "-h"], "--cap-permission-mode")
|
||||
b.add("cli.cap_manifest", "--cap-manifest text, exact", ["--cap-manifest"], "--cap-manifest", "output")
|
||||
b.cases[-1]["tests"] = ["CapCliTests.test_cap_manifest_flag_prints_valid_toml"]
|
||||
# Choice success path for each choices option, parse-only via --version short-circuit is not
|
||||
# reliable for all flags, so only the flags whose success path is side-effect free are listed.
|
||||
for o in opts:
|
||||
if o.choices:
|
||||
for ch in o.choices:
|
||||
if o.flag == "--debug":
|
||||
continue # --debug starts logging; success behavior is a behavior case
|
||||
b.add("cli.ok.%s.%s" % (o.flag[2:], ch), "%s %s accepted, then --version" % (o.flag, ch), [o.flag, ch, "--version"], o.flag)
|
||||
return {"cases": b.cases}
|
||||
|
||||
|
||||
def main():
|
||||
text = json.dumps(build(), indent=1) + "\n"
|
||||
if "--check" in sys.argv:
|
||||
have = open(TARGET).read() if os.path.exists(TARGET) else ""
|
||||
if have != text:
|
||||
print("cli_generated.json is stale: run tools/parity_gen_cases.py")
|
||||
return 1
|
||||
print("cli_generated.json current")
|
||||
return 0
|
||||
os.makedirs(os.path.dirname(TARGET), exist_ok=True)
|
||||
with open(TARGET, "w") as fh:
|
||||
fh.write(text)
|
||||
print("wrote %s (%d cases)" % (TARGET, len(json.loads(text)["cases"])))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Regular → Executable
+290
-77
@@ -1,15 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parity ledger: every tai.py symbol against the C++ tree.
|
||||
"""Parity ledger: every tai.py symbol against the C++ tree and the differential results.
|
||||
|
||||
Usage: tools/parity_ledger.py [--check] (writes build/parity/ledger.json and ledger.md)
|
||||
Usage: tools/parity_ledger.py [--check] [--publish]
|
||||
|
||||
Status per symbol:
|
||||
missing no identifier in src/ or main.cpp matches the normalized name
|
||||
name-only an identifier matches; behavior is NOT verified by this tool
|
||||
Verification (diff of real outputs against tai.py) is a separate, later status
|
||||
and is never inferred here. --check exits 1 while any symbol is missing.
|
||||
Writes build/parity/ledger.json, ledger.md and PARITY.md (all under build/).
|
||||
--publish also copies the generated PARITY.md over taicpp/PARITY.md (the committed copy).
|
||||
--check exits 1 while ANY symbol is missing or unverified, a case names a symbol the
|
||||
ledger does not know, or the differential results are absent or stale.
|
||||
|
||||
Status per symbol (strongest first):
|
||||
verified a passing differential case (tools/parity_diff.py, unfiltered run, tai.py and
|
||||
detz hashes equal to the current files) names the symbol. Facets list WHAT the
|
||||
case verified (`parse`, `output`, ...); verified never means "all behavior".
|
||||
name-only an identifier in src/ or main.cpp matches the normalized name. Behavior unproven.
|
||||
missing no identifier matches.
|
||||
Only a differential case sets `verified`; nothing here infers it.
|
||||
|
||||
Symbol ids are `kind:name`, optionally `#facet` in case files:
|
||||
class:Store method:Store.load function:main tool:shell flag:--version
|
||||
setting:model command:/help
|
||||
"""
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -17,93 +29,294 @@ import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
PY = os.path.join(os.path.dirname(ROOT), "tai.py")
|
||||
REPO = os.path.dirname(ROOT)
|
||||
PY = os.path.join(REPO, "tai.py")
|
||||
OUT = os.path.join(ROOT, "build", "parity")
|
||||
DETZ = os.environ.get("DETZ_BIN", os.path.join(ROOT, "build", "detz"))
|
||||
CASES_DIR = os.path.join(ROOT, "tests", "diff", "cases")
|
||||
PARITY_MD = os.path.join(ROOT, "PARITY.md")
|
||||
KINDS = ("class", "method", "function", "tool", "flag", "setting", "command")
|
||||
|
||||
|
||||
def norm(name):
|
||||
return re.sub(r"[^a-z0-9]", "", name.lower())
|
||||
|
||||
|
||||
def cpp_idents():
|
||||
idents = set()
|
||||
files = [os.path.join(ROOT, "main.cpp")]
|
||||
for base, _dirs, names in os.walk(os.path.join(ROOT, "src")):
|
||||
files += [os.path.join(base, n) for n in names if n.endswith((".cpp", ".h", ".hpp"))]
|
||||
for path in files:
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
idents.update(norm(t) for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", fh.read()))
|
||||
return idents
|
||||
def sha256_file(path):
|
||||
if not os.path.isfile(path):
|
||||
return "missing"
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for block in iter(lambda: fh.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def python_symbols(tree):
|
||||
out = []
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.ClassDef):
|
||||
out.append(("class", node.name, node.lineno))
|
||||
for sub in node.body:
|
||||
if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
out.append(("method", "%s.%s" % (node.name, sub.name), sub.lineno))
|
||||
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
out.append(("function", node.name, node.lineno))
|
||||
return out
|
||||
class PySymbols:
|
||||
"""Symbols extracted from tai.py with ast."""
|
||||
|
||||
def __init__(self, path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
self.tree = ast.parse(fh.read())
|
||||
|
||||
def code(self):
|
||||
out = []
|
||||
for node in self.tree.body:
|
||||
if isinstance(node, ast.ClassDef):
|
||||
out.append(("class", node.name, node.lineno))
|
||||
for sub in node.body:
|
||||
if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
out.append(("method", "%s.%s" % (node.name, sub.name), sub.lineno))
|
||||
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
out.append(("function", node.name, node.lineno))
|
||||
return out
|
||||
|
||||
def tools(self):
|
||||
out = []
|
||||
for node in ast.walk(self.tree):
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "tool_schema" and node.args:
|
||||
arg = node.args[0]
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
||||
out.append(("tool", arg.value, node.lineno))
|
||||
return out
|
||||
|
||||
def flags(self):
|
||||
out = []
|
||||
for node in ast.walk(self.tree):
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument":
|
||||
for arg in node.args:
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and arg.value.startswith("--"):
|
||||
out.append(("flag", arg.value, node.lineno))
|
||||
return out
|
||||
|
||||
def settings(self):
|
||||
out = []
|
||||
for node in self.tree.body:
|
||||
if isinstance(node, ast.Assign) and any(getattr(t, "id", "") == "SETTINGS_CATALOG" for t in node.targets):
|
||||
for sub in ast.walk(node):
|
||||
if isinstance(sub, ast.Call) and getattr(sub.func, "id", "") == "setting_spec" and sub.args:
|
||||
arg = sub.args[0]
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
||||
out.append(("setting", arg.value, sub.lineno))
|
||||
return out
|
||||
|
||||
def commands(self):
|
||||
out = []
|
||||
for node in self.tree.body:
|
||||
if isinstance(node, ast.Assign) and any(getattr(t, "id", "") == "HELP_TEXT" for t in node.targets):
|
||||
text = "".join(c.value for c in ast.walk(node.value) if isinstance(c, ast.Constant) and isinstance(c.value, str))
|
||||
seen = set()
|
||||
for line in text.splitlines():
|
||||
m = re.match(r"\s+(/[a-z]+)", line)
|
||||
if m and m.group(1) not in seen:
|
||||
seen.add(m.group(1))
|
||||
out.append(("command", m.group(1), node.lineno))
|
||||
return out
|
||||
|
||||
def all(self):
|
||||
return self.code() + self.tools() + self.flags() + self.settings() + self.commands()
|
||||
|
||||
|
||||
def tool_names(tree):
|
||||
names = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "tool_schema" and node.args:
|
||||
arg = node.args[0]
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
||||
names.append((arg.value, node.lineno))
|
||||
return names
|
||||
class CppIndex:
|
||||
"""Normalized identifiers found in the C++ tree."""
|
||||
|
||||
def __init__(self):
|
||||
self.idents = set()
|
||||
files = [os.path.join(ROOT, "main.cpp")]
|
||||
for base, _dirs, names in os.walk(os.path.join(ROOT, "src")):
|
||||
files += [os.path.join(base, n) for n in names if n.endswith((".cpp", ".h", ".hpp"))]
|
||||
for path in files:
|
||||
if os.path.isfile(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
self.idents.update(norm(t) for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", fh.read()))
|
||||
|
||||
def flag_names(tree):
|
||||
flags = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add_argument":
|
||||
for arg in node.args:
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and arg.value.startswith("--"):
|
||||
flags.append((arg.value, node.lineno))
|
||||
return flags
|
||||
|
||||
|
||||
def build():
|
||||
with open(PY, encoding="utf-8") as fh:
|
||||
tree = ast.parse(fh.read())
|
||||
idents = cpp_idents()
|
||||
rows = []
|
||||
for kind, name, line in python_symbols(tree):
|
||||
def has(self, kind, name):
|
||||
if kind in ("tool", "setting"):
|
||||
return norm(name) in self.idents
|
||||
if kind == "flag":
|
||||
return norm(name) in self.idents
|
||||
if kind == "command":
|
||||
return norm(name.lstrip("/")) in self.idents and len(norm(name)) > 3
|
||||
leaf = name.split(".")[-1]
|
||||
hit = norm(name.replace(".", "")) in idents or (kind != "class" and len(norm(leaf)) > 5 and norm(leaf) in idents)
|
||||
rows.append({"kind": kind, "name": name, "line": line, "status": "name-only" if hit else "missing"})
|
||||
for name, line in tool_names(tree):
|
||||
rows.append({"kind": "tool", "name": name, "line": line, "status": "name-only" if norm(name) in idents else "missing"})
|
||||
for name, line in flag_names(tree):
|
||||
rows.append({"kind": "flag", "name": name, "line": line, "status": "name-only" if norm(name) in idents else "missing"})
|
||||
return rows
|
||||
if norm(name.replace(".", "")) in self.idents:
|
||||
return True
|
||||
return kind != "class" and len(norm(leaf)) > 5 and norm(leaf) in self.idents
|
||||
|
||||
|
||||
class Evidence:
|
||||
"""Passing differential cases, and whether they may be trusted for the current files."""
|
||||
|
||||
def __init__(self):
|
||||
self.path = os.path.join(OUT, "diff-results.json")
|
||||
self.problems = []
|
||||
self.by_symbol = {} # "kind:name" -> {"cases": [ids], "facets": set}
|
||||
self.cases = {} # id -> result dict
|
||||
self.meta = None
|
||||
if not os.path.isfile(self.path):
|
||||
self.problems.append("no differential results (run tools/parity_diff.py)")
|
||||
return
|
||||
with open(self.path) as fh:
|
||||
doc = json.load(fh)
|
||||
self.meta = doc["meta"]
|
||||
now_py, now_detz = sha256_file(PY), sha256_file(self.meta.get("detz_path", DETZ))
|
||||
if self.meta["tai_sha"] != now_py:
|
||||
self.problems.append("differential results are stale: tai.py changed since the run")
|
||||
if self.meta["detz_sha"] != now_detz:
|
||||
self.problems.append("differential results are stale: detz binary changed or missing since the run")
|
||||
if self.meta.get("no_llm"):
|
||||
self.problems.append("last differential run skipped LLM cases (--no-llm)")
|
||||
trusted = not any("stale" in p for p in self.problems)
|
||||
for res in doc["results"]:
|
||||
self.cases[res["id"]] = res
|
||||
if res["status"] != "pass" or not trusted:
|
||||
continue
|
||||
for sym in res["symbols"]:
|
||||
base, _, facet = sym.partition("#")
|
||||
slot = self.by_symbol.setdefault(base, {"cases": [], "facets": set()})
|
||||
slot["cases"].append(res["id"])
|
||||
slot["facets"].add(facet or "all")
|
||||
|
||||
|
||||
class Ledger:
|
||||
def __init__(self):
|
||||
self.py = PySymbols(PY)
|
||||
self.cpp = CppIndex()
|
||||
self.evidence = Evidence()
|
||||
self.rows = []
|
||||
self.case_symbol_errors = []
|
||||
self._build()
|
||||
|
||||
def _build(self):
|
||||
known = set()
|
||||
for kind, name, line in self.py.all():
|
||||
sid = "%s:%s" % (kind, name)
|
||||
if sid in known:
|
||||
continue
|
||||
known.add(sid)
|
||||
ev = self.evidence.by_symbol.get(sid)
|
||||
if ev:
|
||||
status, facets, cases = "verified", sorted(ev["facets"]), ev["cases"]
|
||||
else:
|
||||
status, facets, cases = ("name-only" if self.cpp.has(kind, name) else "missing"), [], []
|
||||
self.rows.append({"id": sid, "kind": kind, "name": name, "line": line, "status": status, "facets": facets, "cases": cases})
|
||||
for cid, res in self.evidence.cases.items():
|
||||
for sym in res["symbols"]:
|
||||
if sym.partition("#")[0] not in known:
|
||||
self.case_symbol_errors.append("case %s names unknown symbol %s" % (cid, sym))
|
||||
|
||||
def count(self, kind=None):
|
||||
rows = [r for r in self.rows if kind is None or r["kind"] == kind]
|
||||
tally = {"total": len(rows), "verified": 0, "name-only": 0, "missing": 0}
|
||||
for r in rows:
|
||||
tally[r["status"]] += 1
|
||||
return tally
|
||||
|
||||
def problems(self):
|
||||
out = list(self.evidence.problems) + self.case_symbol_errors
|
||||
t = self.count()
|
||||
if t["missing"]:
|
||||
out.append("%d symbols missing" % t["missing"])
|
||||
if t["name-only"]:
|
||||
out.append("%d symbols name-only (unverified)" % t["name-only"])
|
||||
return out
|
||||
|
||||
|
||||
class Writer:
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
|
||||
def tally_table(self):
|
||||
lines = ["| kind | total | verified | name-only | missing |", "|---|---|---|---|---|"]
|
||||
for kind in KINDS + (None,):
|
||||
t = self.ledger.count(kind)
|
||||
lines.append("| %s | %d | %d | %d | %d |" % (kind or "**all**", t["total"], t["verified"], t["name-only"], t["missing"]))
|
||||
return lines
|
||||
|
||||
def parity_md(self):
|
||||
led, ev = self.ledger, self.ledger.evidence
|
||||
meta = ev.meta or {}
|
||||
lines = ["# Parity detz vs tai.py (generated, do not edit)", "",
|
||||
"Generated by `tools/parity_ledger.py` from the symbol ledger and the last differential run.",
|
||||
"Only a passing differential case (`tools/parity_diff.py`) makes a symbol `verified`; a facet",
|
||||
"after the symbol says what that case proved (for example `parse` means argument parsing only).",
|
||||
"`name-only` means an identifier matches in the C++ tree and NOTHING about behavior is known.", "",
|
||||
"- tai.py sha256: `%s`" % sha256_file(PY)[:16],
|
||||
"- detz sha256: `%s`" % sha256_file(meta.get("detz_path", DETZ))[:16],
|
||||
"- differential run: %s (tai.py `%s`, detz `%s`)" % (meta.get("ts", "none"), meta.get("tai_sha", "-")[:16], meta.get("detz_sha", "-")[:16]), ""]
|
||||
if ev.problems:
|
||||
lines += ["## Evidence problems", ""] + ["- " + p for p in ev.problems] + [""]
|
||||
lines += ["## Summary", ""] + self.tally_table() + [""]
|
||||
results = list(ev.cases.values())
|
||||
counts = {}
|
||||
for r in results:
|
||||
counts[r["status"]] = counts.get(r["status"], 0) + 1
|
||||
lines += ["## Differential cases", "", "Counts: " + (", ".join("%s=%d" % kv for kv in sorted(counts.items())) or "none"), ""]
|
||||
failing = [r for r in results if r["status"] in ("fail", "xfail", "xpass")]
|
||||
if failing:
|
||||
lines += ["| case | status | reason |", "|---|---|---|"]
|
||||
for r in sorted(failing, key=lambda x: x["id"]):
|
||||
reason = r["expected_fail"] or (r["problems"][0] if r["problems"] else "")
|
||||
lines.append("| %s | %s | %s |" % (r["id"], r["status"], reason.replace("|", "/")[:120]))
|
||||
lines.append("")
|
||||
tm = os.path.join(OUT, "tests-map.json")
|
||||
if os.path.isfile(tm):
|
||||
with open(tm) as fh:
|
||||
cov = json.load(fh)["summary"]
|
||||
lines += ["## Python test-suite coverage", "",
|
||||
"%d test classes, %d tests; tests reproduced by a passing diff case: %d (in %d classes); classes with no covered test: %d."
|
||||
% (cov["classes"], cov["tests"], cov["tests_covered"], cov["classes_covered"], cov["classes_uncovered"]),
|
||||
"Full table: `build/parity/tests-map.md`.", ""]
|
||||
for kind in ("flag", "tool", "setting", "command"):
|
||||
rows = [r for r in led.rows if r["kind"] == kind]
|
||||
lines += ["## %ss (%d)" % (kind, len(rows)), "", "| name | status | facets | cases |", "|---|---|---|---|"]
|
||||
for r in sorted(rows, key=lambda x: x["name"]):
|
||||
lines.append("| `%s` | %s | %s | %s |" % (r["name"], r["status"], ",".join(r["facets"]), ",".join(r["cases"][:3])))
|
||||
lines.append("")
|
||||
miss = {}
|
||||
for r in led.rows:
|
||||
if r["kind"] == "method" and r["status"] == "missing":
|
||||
miss.setdefault(r["name"].split(".")[0], 0)
|
||||
miss[r["name"].split(".")[0]] += 1
|
||||
top = sorted(miss.items(), key=lambda kv: -kv[1])[:25]
|
||||
lines += ["## Classes with the most missing methods (top 25 of %d)" % len(miss), "", "| class | missing methods |", "|---|---|"]
|
||||
lines += ["| %s | %d |" % kv for kv in top]
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
def ledger_md(self):
|
||||
lines = ["# Parity ledger (generated, do not edit)", ""] + self.tally_table() + [""]
|
||||
lines += ["| id | status | facets | cases |", "|---|---|---|---|"]
|
||||
for r in self.ledger.rows:
|
||||
lines.append("| `%s` | %s | %s | %s |" % (r["id"], r["status"], ",".join(r["facets"]), ",".join(r["cases"][:3])))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
def write(self, publish):
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
with open(os.path.join(OUT, "ledger.json"), "w") as fh:
|
||||
json.dump(self.ledger.rows, fh, indent=1)
|
||||
with open(os.path.join(OUT, "ledger.md"), "w") as fh:
|
||||
fh.write(self.ledger_md())
|
||||
text = self.parity_md()
|
||||
with open(os.path.join(OUT, "PARITY.md"), "w") as fh:
|
||||
fh.write(text)
|
||||
if publish:
|
||||
with open(PARITY_MD, "w") as fh:
|
||||
fh.write(text)
|
||||
|
||||
|
||||
def main():
|
||||
rows = build()
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
with open(os.path.join(OUT, "ledger.json"), "w") as fh:
|
||||
json.dump(rows, fh, indent=1)
|
||||
kinds = sorted({r["kind"] for r in rows})
|
||||
lines = ["# Parity ledger (generated, do not edit)", "",
|
||||
"Source: tai.py. `name-only` means an identifier matches, NOT that behavior is verified.", "",
|
||||
"| kind | total | name-only | missing |", "|---|---|---|---|"]
|
||||
for kind in kinds:
|
||||
sub = [r for r in rows if r["kind"] == kind]
|
||||
have = sum(1 for r in sub if r["status"] == "name-only")
|
||||
lines.append("| %s | %d | %d | %d |" % (kind, len(sub), have, len(sub) - have))
|
||||
total_missing = sum(1 for r in rows if r["status"] == "missing")
|
||||
lines += ["", "Total symbols: %d, missing: %d, verified: 0" % (len(rows), total_missing)]
|
||||
with open(os.path.join(OUT, "ledger.md"), "w") as fh:
|
||||
fh.write("\n".join(lines) + "\n")
|
||||
print("\n".join(lines))
|
||||
return 1 if "--check" in sys.argv and total_missing else 0
|
||||
ledger = Ledger()
|
||||
Writer(ledger).write("--publish" in sys.argv)
|
||||
print("\n".join(Writer(ledger).tally_table()))
|
||||
problems = ledger.problems()
|
||||
for p in problems:
|
||||
print("PROBLEM: %s" % p)
|
||||
if "--check" in sys.argv:
|
||||
if problems:
|
||||
print("parity-check: FAIL (%d problems)" % len(problems))
|
||||
return 1
|
||||
print("parity-check: OK")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mine ../test_tai.py into a coverage table: test class -> tai.py symbols -> diff case or `uncovered`.
|
||||
|
||||
Usage: tools/parity_tests.py (writes build/parity/tests-map.json and tests-map.md)
|
||||
|
||||
For every unittest class in test_tai.py (ast, never imported): the number of tests, and the
|
||||
tai.py symbols it touches (`tai.X`, `tai.X.y`, and `"tai.X.y"` mock.patch strings), resolved
|
||||
against the ledger ids (class:X, function:X, method:X.y; module data shows as data:X).
|
||||
|
||||
A test (or whole class) maps to a differential case ONLY when the case lists `Class` or `Class.test_name` in its `tests` field
|
||||
AND that case passed in the last unfiltered run. Everything else is `uncovered`. Listing a
|
||||
class in a case means the case reproduces what those tests assert through the real binaries;
|
||||
it is a claim the reviewer can read in the case file, not something inferred here.
|
||||
"""
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
import parity_ledger as pl # noqa: E402
|
||||
|
||||
TEST_FILE = os.path.join(pl.REPO, "test_tai.py")
|
||||
|
||||
|
||||
class TestMiner:
|
||||
def __init__(self, path, symbol_ids):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
self.tree = ast.parse(fh.read())
|
||||
self.ids = symbol_ids
|
||||
|
||||
def classes(self):
|
||||
out = []
|
||||
for node in self.tree.body:
|
||||
if isinstance(node, ast.ClassDef):
|
||||
tests = [n for n in node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name.startswith("test")]
|
||||
if tests:
|
||||
out.append({"class": node.name, "line": node.lineno, "tests": len(tests), "names": [t.name for t in tests], "symbols": self.symbols(node)})
|
||||
return out
|
||||
|
||||
def symbols(self, cls):
|
||||
found = set()
|
||||
for node in ast.walk(cls):
|
||||
if isinstance(node, ast.Attribute):
|
||||
inner = node.value
|
||||
if isinstance(inner, ast.Name) and inner.id == "tai":
|
||||
found.add(node.attr)
|
||||
elif isinstance(inner, ast.Attribute) and isinstance(inner.value, ast.Name) and inner.value.id == "tai":
|
||||
found.add("%s.%s" % (inner.attr, node.attr))
|
||||
elif isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
for m in re.finditer(r"\btai\.([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?)\b", node.value):
|
||||
found.add(m.group(1))
|
||||
return sorted({self.resolve(n) for n in found})
|
||||
|
||||
def resolve(self, name):
|
||||
for kind in ("method", "class", "function"):
|
||||
sid = "%s:%s" % (kind, name)
|
||||
if sid in self.ids:
|
||||
return sid
|
||||
head = name.split(".")[0]
|
||||
if "." in name and "class:%s" % head in self.ids:
|
||||
return "class:%s" % head # attribute or nested name on a known class
|
||||
return "data:%s" % name
|
||||
|
||||
|
||||
def main():
|
||||
ledger = pl.Ledger()
|
||||
ids = {r["id"] for r in ledger.rows}
|
||||
classes = TestMiner(TEST_FILE, ids).classes()
|
||||
ev = ledger.evidence
|
||||
by_test = {} # "Class" or "Class.test" -> passing case ids
|
||||
for cid, res in ev.cases.items():
|
||||
if res["status"] == "pass":
|
||||
for tc in res["tests"]:
|
||||
by_test.setdefault(tc, []).append(cid)
|
||||
errors = []
|
||||
names = {c["class"]: set(c["names"]) for c in classes}
|
||||
for cid, res in ev.cases.items():
|
||||
for tc in res["tests"]:
|
||||
cls, _, test = tc.partition(".")
|
||||
if cls not in names or (test and test not in names[cls]):
|
||||
errors.append("case %s names unknown test %s" % (cid, tc))
|
||||
for c in classes:
|
||||
whole = by_test.get(c["class"], [])
|
||||
per = {n: sorted(set(whole + by_test.get("%s.%s" % (c["class"], n), []))) for n in c["names"]}
|
||||
c["covered_tests"] = sum(1 for v in per.values() if v)
|
||||
c["cases"] = sorted({cid for v in per.values() for cid in v})
|
||||
c["status"] = "uncovered" if not c["covered_tests"] else ("covered" if c["covered_tests"] == c["tests"] else "partial")
|
||||
total_tests = sum(c["tests"] for c in classes)
|
||||
cov = [c for c in classes if c["covered_tests"]]
|
||||
summary = {"classes": len(classes), "tests": total_tests, "classes_covered": len(cov),
|
||||
"tests_covered": sum(c["covered_tests"] for c in classes), "classes_uncovered": len(classes) - len(cov)}
|
||||
os.makedirs(pl.OUT, exist_ok=True)
|
||||
with open(os.path.join(pl.OUT, "tests-map.json"), "w") as fh:
|
||||
json.dump({"summary": summary, "classes": classes, "errors": sorted(set(errors))}, fh, indent=1)
|
||||
lines = ["# test_tai.py coverage map (generated)", "",
|
||||
"%(classes)d classes, %(tests)d tests. Tests reproduced by a passing diff case: %(tests_covered)d, in %(classes_covered)d classes. Fully uncovered classes: %(classes_uncovered)d." % summary, "",
|
||||
"| test class | tests | covered | status | diff cases | tai.py symbols |", "|---|---|---|---|---|---|"]
|
||||
for c in sorted(classes, key=lambda x: (x["status"], -x["tests"], x["class"])):
|
||||
syms = ", ".join(c["symbols"][:8]) + (" ... +%d" % (len(c["symbols"]) - 8) if len(c["symbols"]) > 8 else "")
|
||||
lines.append("| %s | %d | %d | %s | %s | %s |" % (c["class"], c["tests"], c["covered_tests"], c["status"], ",".join(c["cases"]) or "uncovered", syms))
|
||||
if errors:
|
||||
lines += ["", "## Errors", ""] + ["- " + e for e in sorted(set(errors))]
|
||||
with open(os.path.join(pl.OUT, "tests-map.md"), "w") as fh:
|
||||
fh.write("\n".join(lines) + "\n")
|
||||
print("test classes %(classes)d, tests %(tests)d, covered classes %(classes_covered)d (tests %(tests_covered)d), uncovered classes %(classes_uncovered)d" % summary)
|
||||
for e in sorted(set(errors)):
|
||||
print("ERROR:", e)
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user