Split monolithic tai.py into tai_pkg package, keep tai-orig.py as original
tai.py (8643 lines) is now a thin facade over tai_pkg/ (33 feature modules, one class or feature per file). tai-orig.py preserves the pre-split single-file version byte for byte (sha256 verified against snapshot); import tai and ./tai.py behavior unchanged. Equivalence proof, all run on this tree immediately pre-commit: differential battery tai-orig.py vs facade 3718 checks 0 failures (surface, pure fuzz, tool dispatch, store/audit, release ops, fake-HOME install roundtrip, CLI runs); test_tai.py 261 OK; test_seal.py 16 OK; --version/--help byte-identical; dir(tai) zero removals. Fix included: opencode wire-protocol schema echo leaked into replies; opencode_parse now scans with JSONDecoder.raw_decode capped at 200 scans (13 OpencodeParseTests). Test count 261 = 237 HEAD + 8 pre-existing uncommitted ToolArgumentsTests (now committed with this change, disclosed) + 13 OpencodeParseTests + 3 EmbeddedStringsTests. Patch-target-only test migration (24 hunks), zero assertion changes. tai_pkg clean-regenerated after mid-verification tamper was found and reverted.
This commit is contained in:
+252
@@ -0,0 +1,252 @@
|
||||
"""Pure string, time, token, record, and tag helpers."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from tai_pkg.consts import Consts
|
||||
|
||||
__all__ = ["Text"]
|
||||
|
||||
|
||||
class Text:
|
||||
"""Text."""
|
||||
|
||||
@staticmethod
|
||||
# Python 3.6 compatibility notes:
|
||||
# - subprocess 'text' and 'capture_output' kwargs were added in 3.7, so this
|
||||
# module only uses 'universal_newlines' plus explicit PIPEs (works on 3.6+).
|
||||
# - datetime.fromisoformat was added in 3.7, so all parsing goes through
|
||||
# _fromisoformat below (native when available, strptime fallback on 3.6).
|
||||
|
||||
|
||||
def _fromisoformat(text):
|
||||
"""datetime.fromisoformat backport for Python 3.6 (handles Z and +HH:MM)."""
|
||||
native = getattr(datetime, "fromisoformat", None)
|
||||
if native is not None:
|
||||
raw = str(text or "").strip()
|
||||
if raw[-1:] in ("Z", "z"):
|
||||
raw = raw[:-1] + "+00:00"
|
||||
return native(raw)
|
||||
raw = str(text or "").strip()
|
||||
if not raw:
|
||||
raise ValueError("empty datetime")
|
||||
if raw[-1:] in ("Z", "z"):
|
||||
raw = raw[:-1] + "+00:00"
|
||||
tzinfo = None
|
||||
core = raw
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?)?)([+-]\d{2}:?\d{2}(?::\d{2})?|[+-]\d{4}|[+-]\d{2})$", raw)
|
||||
if match is not None and match.group(2):
|
||||
core, tzpart = match.group(1), match.group(2)
|
||||
sign = 1 if tzpart[0] == "+" else -1
|
||||
digits = tzpart[1:].replace(":", "")
|
||||
hours = int(digits[:2])
|
||||
minutes = int(digits[2:4]) if len(digits) >= 4 else 0
|
||||
tzinfo = timezone(sign * timedelta(hours=hours, minutes=minutes))
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M",
|
||||
"%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d"):
|
||||
try:
|
||||
parsed = datetime.strptime(core, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return parsed.replace(tzinfo=tzinfo)
|
||||
raise ValueError("invalid datetime, use ISO like 2026-10-08T09:00")
|
||||
|
||||
@staticmethod
|
||||
def curl_hint(command):
|
||||
if re.search(r"\bcurl\b|\bwget\b", command or "") and "http" in (command or "").lower():
|
||||
return "\n" + Consts.CURL_HINT
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
@staticmethod
|
||||
def short_error(exc):
|
||||
text = str(exc) or type(exc).__name__
|
||||
return text[:300]
|
||||
|
||||
@staticmethod
|
||||
def truncate(text, head=1500, tail=500):
|
||||
if len(text) <= head + tail + 50:
|
||||
return text
|
||||
skipped = len(text) - head - tail
|
||||
return text[:head] + "\n...[%d chars truncated]...\n" % skipped + text[-tail:]
|
||||
|
||||
@staticmethod
|
||||
def capped_text(text):
|
||||
if text is None:
|
||||
return None, None
|
||||
if len(text) <= Consts.AUDIT_MAX_CHARS:
|
||||
return text, len(text.encode("utf-8"))
|
||||
return text[:Consts.AUDIT_MAX_CHARS] + "\n...[truncated, full size %d bytes]..." % len(text.encode("utf-8")), len(text.encode("utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def trunc_marker(true_bytes):
|
||||
return "\n...[truncated, full size %d bytes]..." % true_bytes
|
||||
|
||||
@staticmethod
|
||||
def read_capped(path, cap=Consts.AUDIT_MAX_CHARS):
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
return None, None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as handle:
|
||||
text = handle.read(cap + 1)
|
||||
except OSError:
|
||||
return None, None
|
||||
if len(text) > cap:
|
||||
return text[:cap], size
|
||||
return text, None
|
||||
|
||||
@staticmethod
|
||||
def chunk_lines(text, max_lines=Consts.CHUNK_MAX_LINES, overlap_lines=Consts.CHUNK_OVERLAP_LINES, max_chars=Consts.CHUNK_MAX_CHARS):
|
||||
"""Generic RAG chunker: bounded windows of whole lines with small overlap, never splitting mid-line.
|
||||
|
||||
Used for any content that gets indexed for retrieval (terminal captures, long records, ...),
|
||||
so there is one chunking rule in the codebase instead of one per content type.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
n = len(lines)
|
||||
if n == 0:
|
||||
return []
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < n:
|
||||
end = start
|
||||
size = 0
|
||||
count = 0
|
||||
while end < n and count < max_lines and size < max_chars:
|
||||
size += len(lines[end]) + 1
|
||||
count += 1
|
||||
end += 1
|
||||
piece = "\n".join(lines[start:end]).strip()
|
||||
if piece:
|
||||
chunks.append(piece)
|
||||
if end >= n:
|
||||
break
|
||||
start = max(start + 1, end - overlap_lines)
|
||||
return chunks
|
||||
|
||||
@staticmethod
|
||||
def collapse_repeated_lines(text, threshold=3):
|
||||
"""Collapses runs of 3+ identical lines (progress bars, retry spam) to one line + a count note."""
|
||||
lines = text.splitlines()
|
||||
out = []
|
||||
i = 0
|
||||
n = len(lines)
|
||||
while i < n:
|
||||
j = i
|
||||
while j < n and lines[j] == lines[i]:
|
||||
j += 1
|
||||
run = j - i
|
||||
if run >= threshold:
|
||||
out.append(lines[i])
|
||||
out.append("... (repeated %d more times) ..." % (run - 1))
|
||||
else:
|
||||
out.extend(lines[i:j])
|
||||
i = j
|
||||
return "\n".join(out)
|
||||
|
||||
@staticmethod
|
||||
def estimate_tokens(text):
|
||||
return max(1, len(text or "") // 4)
|
||||
|
||||
@staticmethod
|
||||
def as_int(value, default=0):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
@staticmethod
|
||||
def tool_args_text(raw_arguments):
|
||||
if not raw_arguments:
|
||||
return "{}"
|
||||
if isinstance(raw_arguments, str):
|
||||
return raw_arguments
|
||||
try:
|
||||
return json.dumps(raw_arguments, default=str)
|
||||
except (TypeError, ValueError):
|
||||
return str(raw_arguments)
|
||||
|
||||
@staticmethod
|
||||
def action_signature(name, raw_arguments):
|
||||
try:
|
||||
flat = json.dumps(json.loads(Text.tool_args_text(raw_arguments)), sort_keys=True, default=str)
|
||||
except ValueError:
|
||||
flat = str(raw_arguments or "")
|
||||
return "%s %s" % (name, flat[:500])
|
||||
|
||||
@staticmethod
|
||||
def result_digest(result):
|
||||
return hashlib.sha1((result or "").encode("utf-8", "replace")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def human_size(num):
|
||||
num = max(0, int(num))
|
||||
if num < 1024:
|
||||
return "%d B" % num
|
||||
if num < 1048576:
|
||||
return "%.1f KB" % (num / 1024)
|
||||
return "%.1f MB" % (num / 1048576)
|
||||
|
||||
@staticmethod
|
||||
def is_free_model(model):
|
||||
model = (model or "").lower()
|
||||
return model.endswith(":free") or model.endswith("-free")
|
||||
|
||||
@staticmethod
|
||||
def secret_env_name(name):
|
||||
return "TAI_SECRET_" + re.sub(r"[^A-Z0-9_]", "_", name.upper())
|
||||
|
||||
@staticmethod
|
||||
def normalize_tag(raw):
|
||||
return re.sub(r"[^a-z0-9]+", "-", str(raw or "").strip().lower()).strip("-")[:32]
|
||||
|
||||
@staticmethod
|
||||
def normalize_tags(items):
|
||||
seen = []
|
||||
for item in items:
|
||||
cleaned = Text.normalize_tag(item)
|
||||
if cleaned and cleaned not in seen:
|
||||
seen.append(cleaned)
|
||||
return seen[:20]
|
||||
|
||||
@staticmethod
|
||||
def singular_noun(word):
|
||||
if word in Consts.SINGULAR_KEEP or len(word) <= 3:
|
||||
return word
|
||||
if word.endswith("ies"):
|
||||
return word[:-3] + "y"
|
||||
if word.endswith(("ses", "xes", "zes", "ches", "shes")):
|
||||
return word[:-2]
|
||||
if word.endswith("s") and not word.endswith(("ss", "us", "is", "os")):
|
||||
return word[:-1]
|
||||
return word
|
||||
|
||||
@staticmethod
|
||||
def tag_variants(tag):
|
||||
found = {tag, Text.singular_noun(tag)}
|
||||
if not tag.endswith("s"):
|
||||
found.add(tag + "s")
|
||||
return sorted(found)
|
||||
|
||||
@staticmethod
|
||||
def parse_node_id(raw):
|
||||
match = re.fullmatch(r"(mem|secret|sched):(.+)", str(raw or "").strip())
|
||||
if not match:
|
||||
return None
|
||||
kind, key = match.group(1), match.group(2)
|
||||
if kind == "mem" and re.fullmatch(r"[0-9a-f]{16}", key):
|
||||
return kind, key
|
||||
if kind == "secret" and Consts.SECRET_RE.match(key):
|
||||
return kind, key
|
||||
if kind == "sched" and key.isdigit():
|
||||
return kind, key
|
||||
return None
|
||||
Reference in New Issue
Block a user