retoor
8df5cf7a45
tai.py (8643 lines) is now a thin facade over tai_pkg/ (33 feature modules, one class or feature per file). tai-orig.py preserves the pre-split single-file version byte for byte (sha256 verified against snapshot); import tai and ./tai.py behavior unchanged. Equivalence proof, all run on this tree immediately pre-commit: differential battery tai-orig.py vs facade 3718 checks 0 failures (surface, pure fuzz, tool dispatch, store/audit, release ops, fake-HOME install roundtrip, CLI runs); test_tai.py 261 OK; test_seal.py 16 OK; --version/--help byte-identical; dir(tai) zero removals. Fix included: opencode wire-protocol schema echo leaked into replies; opencode_parse now scans with JSONDecoder.raw_decode capped at 200 scans (13 OpencodeParseTests). Test count 261 = 237 HEAD + 8 pre-existing uncommitted ToolArgumentsTests (now committed with this change, disclosed) + 13 OpencodeParseTests + 3 EmbeddedStringsTests. Patch-target-only test migration (24 hunks), zero assertion changes. tai_pkg clean-regenerated after mid-verification tamper was found and reverted.
969 lines
47 KiB
Python
969 lines
47 KiB
Python
"""SQLite vault storage, sessions, FTS, and migrations."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
from tai_pkg.consts import Consts
|
|
from tai_pkg.release import Release
|
|
from tai_pkg.seal import Seal
|
|
from tai_pkg.seal import SealError
|
|
from tai_pkg.text import Text
|
|
|
|
__all__ = ["Store"]
|
|
|
|
|
|
class Store:
|
|
def __init__(self, config, seal):
|
|
self.config = config
|
|
self.seal = seal
|
|
self.db = sqlite3.connect(config.db_path, timeout=30)
|
|
self.db.execute("PRAGMA journal_mode=WAL")
|
|
self.db.execute("PRAGMA synchronous=NORMAL")
|
|
self.db.execute("PRAGMA busy_timeout=30000")
|
|
self.schema_notes = Store.ensure_schema(self.db)
|
|
Store.migrate_profiles(self.db)
|
|
self.db.commit()
|
|
self.profile = config.profile
|
|
self.secret_cache = {}
|
|
self.fts_ok = Store.setup_fts(self.db)
|
|
if seal.enabled:
|
|
self.db.create_function("tai_enc", 1, seal.lock)
|
|
self.db.create_function("tai_dec", 1, seal.safe_unlock)
|
|
self.db.execute("UPDATE events SET text = tai_enc(text) WHERE text NOT LIKE 'tai1$%'")
|
|
self.db.execute("UPDATE secrets SET value = tai_enc(value) WHERE value NOT LIKE 'tai1$%'")
|
|
self.db.execute("UPDATE secrets SET meta = tai_enc(COALESCE(meta, '{}')) WHERE meta IS NULL OR meta NOT LIKE 'tai1$%'")
|
|
self.db.execute("UPDATE schedules SET prompt = tai_enc(prompt) WHERE prompt NOT LIKE 'tai1$%'")
|
|
self.db.commit()
|
|
self.seal_files()
|
|
else:
|
|
sealed = self.db.execute("SELECT COUNT(*) FROM events WHERE text LIKE 'tai1$%'").fetchone()[0]
|
|
if sealed:
|
|
raise SealError("memory is sealed, set TAI_PASSPHRASE")
|
|
locked = self.db.execute("SELECT COUNT(*) FROM secrets WHERE value LIKE 'tai1$%' OR meta LIKE 'tai1$%'").fetchone()[0]
|
|
if locked:
|
|
raise SealError("secrets are sealed, set TAI_PASSPHRASE")
|
|
planned = self.db.execute("SELECT COUNT(*) FROM schedules WHERE prompt LIKE 'tai1$%'").fetchone()[0]
|
|
if planned:
|
|
raise SealError("schedules are sealed, set TAI_PASSPHRASE")
|
|
self.memdb = None
|
|
self.mem_events_max = 0
|
|
if seal.enabled and self.fts_ok:
|
|
try:
|
|
self.memdb = Store.open_mem_events()
|
|
floor = self.db.execute("SELECT COALESCE(MAX(id), 0) FROM events").fetchone()[0] - Consts.MEM_EVENTS_CAP
|
|
self.mem_events_max = max(0, floor)
|
|
self._sync_mem_events()
|
|
except sqlite3.Error:
|
|
self.memdb = None
|
|
|
|
def _sync_mem_events(self):
|
|
if self.memdb is None:
|
|
return
|
|
try:
|
|
rows = self.db.execute("SELECT id, profile, ts, role, kind, tai_dec(text) FROM events WHERE id > ? ORDER BY id ASC", (self.mem_events_max,)).fetchall()
|
|
except sqlite3.Error:
|
|
return
|
|
if not rows:
|
|
return
|
|
try:
|
|
self.memdb.executemany("INSERT INTO mem_events(item, kind, sub, title, body, profile) VALUES (?, 'event', ? || '/' || ?, ? || ' ' || ? || '/' || ?, ?, ?)", [(row[0], row[3], row[4], row[2], row[3], row[4], row[5], row[1]) for row in rows])
|
|
self.memdb.commit()
|
|
except sqlite3.Error:
|
|
return
|
|
self.mem_events_max = rows[-1][0]
|
|
try:
|
|
self.memdb.execute("DELETE FROM mem_events WHERE CAST(item AS INTEGER) <= ?", (self.mem_events_max - Consts.MEM_EVENTS_CAP,))
|
|
self.memdb.commit()
|
|
except sqlite3.Error:
|
|
pass
|
|
|
|
def seal_files(self):
|
|
for entry in os.listdir(self.config.profiles_dir):
|
|
if not entry.endswith((".sys.md", ".session.json", ".bots.json")):
|
|
continue
|
|
path = os.path.join(self.config.profiles_dir, entry)
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
raw = handle.read()
|
|
except OSError:
|
|
continue
|
|
if raw.startswith(Seal.PREFIX):
|
|
continue
|
|
try:
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
handle.write(self.seal.lock(raw))
|
|
except OSError:
|
|
pass
|
|
|
|
def log_event(self, profile, role, kind, text, tags=()):
|
|
value = (text or "")[:2000]
|
|
stamped = " " + " ".join([role, kind] + Text.normalize_tags(tags)) + " "
|
|
try:
|
|
if self.seal.enabled:
|
|
self.db.execute("INSERT INTO events (profile, ts, role, kind, text, tags) VALUES (?, ?, ?, ?, tai_enc(?), ?)", (profile, Text.now_iso(), role, kind, value, stamped))
|
|
else:
|
|
self.db.execute("INSERT INTO events (profile, ts, role, kind, text, tags) VALUES (?, ?, ?, ?, ?, ?)", (profile, Text.now_iso(), role, kind, value, stamped))
|
|
self.db.commit()
|
|
self._sync_mem_events()
|
|
except sqlite3.Error:
|
|
pass
|
|
|
|
def fts_search(self, query, kinds=(), profile=None, limit=10):
|
|
profile = self._scope(profile)
|
|
if not self.fts_ok:
|
|
return []
|
|
terms = Store.fts_escape(query)
|
|
if terms is None:
|
|
return []
|
|
sql = "SELECT item, kind, sub, title, snippet(fts_docs, 4, '[', ']', '...', 12), bm25(fts_docs) FROM fts_docs WHERE fts_docs MATCH ? AND profile = ?"
|
|
params = [Store.fts_match(terms), profile]
|
|
if kinds:
|
|
marks = ", ".join("?" * len(kinds))
|
|
sql += " AND kind IN (%s)" % marks
|
|
params += list(kinds)
|
|
if self.seal.enabled:
|
|
sql += " AND kind != 'event'"
|
|
sql += " ORDER BY bm25(fts_docs) LIMIT ?"
|
|
capped = max(1, min(limit, 100))
|
|
params.append(capped)
|
|
try:
|
|
rows = self.db.execute(sql, tuple(params)).fetchall()
|
|
except sqlite3.Error:
|
|
return []
|
|
hits = [{"item": row[0], "kind": row[1], "sub": row[2], "title": row[3], "snippet": row[4], "rank": row[5]} for row in rows]
|
|
if self.seal.enabled and self.memdb is not None and (not kinds or "event" in kinds):
|
|
self._sync_mem_events()
|
|
try:
|
|
mem_rows = self.memdb.execute("SELECT item, kind, sub, title, snippet(mem_events, 4, '[', ']', '...', 12), bm25(mem_events) FROM mem_events WHERE mem_events MATCH ? AND profile = ? ORDER BY bm25(mem_events) LIMIT ?", (Store.fts_match(terms), profile, capped)).fetchall()
|
|
except sqlite3.Error:
|
|
mem_rows = []
|
|
hits += [{"item": row[0], "kind": row[1], "sub": row[2], "title": row[3], "snippet": row[4], "rank": row[5]} for row in mem_rows]
|
|
hits.sort(key=lambda hit: hit["rank"])
|
|
hits = hits[:capped]
|
|
return hits
|
|
|
|
def audit_search(self, query, profile=None, limit=20):
|
|
profile = self._scope(profile)
|
|
hits = self.fts_search(query, ("audit",), profile, limit)
|
|
found = []
|
|
for hit in hits:
|
|
try:
|
|
row_id = int(str(hit["item"]).split(":", 1)[1])
|
|
except (IndexError, ValueError):
|
|
continue
|
|
row = self.audit_get(row_id, profile)
|
|
if row is not None:
|
|
row["snippet"] = hit["snippet"]
|
|
found.append(row)
|
|
return found
|
|
|
|
def search_events(self, profile, query, limit=8, tags=()):
|
|
clean = Text.normalize_tags(tags)
|
|
if self.fts_ok:
|
|
hits = self.fts_search(query, ("event",), profile, max(limit * 5, 20))
|
|
if hits:
|
|
ids = [int(hit["item"]) for hit in hits]
|
|
marks = ", ".join("?" * len(ids))
|
|
text_col = "tai_dec(text)" if self.seal.enabled else "text"
|
|
rows = {row[0]: row[1:] for row in self.db.execute("SELECT id, ts, role, kind, %s, tags FROM events WHERE id IN (%s)" % (text_col, marks), tuple(ids)).fetchall()}
|
|
found = []
|
|
for hit in hits:
|
|
row = rows.get(int(hit["item"]))
|
|
if row is None:
|
|
continue
|
|
if clean and not all((" %s " % tag) in (" %s " % (row[4] or "")) for tag in clean):
|
|
continue
|
|
found.append(row[:4])
|
|
if len(found) >= limit:
|
|
break
|
|
if found:
|
|
return found
|
|
return self.events_like(profile, query, limit, clean)
|
|
|
|
def events_like(self, profile, query, limit=8, tags=()):
|
|
clean = Text.normalize_tags(tags)
|
|
extra = "".join(" AND tags LIKE ?" for _tag in clean)
|
|
wild = tuple("% %s %%" % tag for tag in clean)
|
|
if self.seal.enabled:
|
|
return self.db.execute("SELECT ts, role, kind, text FROM (SELECT id, ts, role, kind, tags, tai_dec(text) AS text FROM events WHERE profile = ? ORDER BY id DESC LIMIT 5000) WHERE text LIKE ?" + extra + " ORDER BY id DESC LIMIT ?", (profile, "%" + query + "%") + wild + (limit,)).fetchall()
|
|
return self.db.execute("SELECT ts, role, kind, text FROM events WHERE profile = ? AND text LIKE ?" + extra + " ORDER BY id DESC LIMIT ?", (profile, "%" + query + "%") + wild + (limit,)).fetchall()
|
|
|
|
def _scope(self, profile):
|
|
return profile or self.profile
|
|
|
|
def save_secret(self, name, value, meta=None, tags=None, profile=None):
|
|
profile = self._scope(profile)
|
|
self.secret_cache.pop(profile, None)
|
|
stored = self.seal.lock(value) if self.seal.enabled else value
|
|
blob = json.dumps(meta or {})
|
|
locked_meta = self.seal.lock(blob) if self.seal.enabled else blob
|
|
self.db.execute("INSERT OR REPLACE INTO secrets (profile, name, value, meta, updated) VALUES (?, ?, ?, ?, ?)", (profile, name, stored, locked_meta, Text.now_iso()))
|
|
if tags is None:
|
|
self.tag_item("secret:" + name, ["secret"], profile)
|
|
else:
|
|
self.set_item_tags("secret:" + name, ["secret"] + list(tags), profile)
|
|
self.db.commit()
|
|
|
|
def secret_meta(self, name, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT meta FROM secrets WHERE profile = ? AND name = ?", (profile, name)).fetchone()
|
|
if row is None or not row[0]:
|
|
return {}
|
|
raw = self.seal.safe_unlock(row[0]) if row[0].startswith(Seal.PREFIX) else row[0]
|
|
try:
|
|
data = json.loads(raw)
|
|
except ValueError:
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
def list_secret_infos(self, profile=None):
|
|
profile = self._scope(profile)
|
|
rows = self.db.execute("SELECT name, meta, updated FROM secrets WHERE profile = ? ORDER BY name", (profile,)).fetchall()
|
|
items = []
|
|
for name, meta, updated in rows:
|
|
raw = None
|
|
if meta:
|
|
raw = self.seal.safe_unlock(meta) if meta.startswith(Seal.PREFIX) else meta
|
|
try:
|
|
data = json.loads(raw or "{}")
|
|
except ValueError:
|
|
data = {}
|
|
items.append({"name": name, "meta": data if isinstance(data, dict) else {}, "tags": self.item_tags("secret:" + name, profile), "updated": updated})
|
|
return items
|
|
|
|
def known_tags(self, profile=None):
|
|
profile = self._scope(profile)
|
|
return {row[0] for row in self.db.execute("SELECT DISTINCT tag FROM tags WHERE profile = ?", (profile,)).fetchall()}
|
|
|
|
def canonical_tag(self, tag, vocab=None, profile=None):
|
|
if vocab is None:
|
|
vocab = self.known_tags(profile)
|
|
if tag in vocab:
|
|
return tag
|
|
single = Text.singular_noun(tag)
|
|
if single != tag and single in vocab:
|
|
return single
|
|
return tag
|
|
|
|
def canonical_tags(self, tags, vocab=None, profile=None):
|
|
profile = self._scope(profile)
|
|
if vocab is None:
|
|
vocab = self.known_tags(profile)
|
|
seen = []
|
|
for tag in Text.normalize_tags(tags):
|
|
clean = self.canonical_tag(tag, vocab)
|
|
if clean not in seen:
|
|
seen.append(clean)
|
|
return seen
|
|
|
|
def tag_counts(self, prefix="", limit=50, profile=None):
|
|
profile = self._scope(profile)
|
|
clean = Text.normalize_tag(prefix)
|
|
sql = "SELECT tag, COUNT(*) FROM tags WHERE profile = ?"
|
|
params = [profile]
|
|
if clean:
|
|
sql += " AND tag LIKE ? ESCAPE '\\'"
|
|
params.append(clean.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%")
|
|
sql += " GROUP BY tag ORDER BY COUNT(*) DESC, tag LIMIT ?"
|
|
params.append(max(1, min(limit, 200)))
|
|
return [(row[0], row[1]) for row in self.db.execute(sql, tuple(params)).fetchall()]
|
|
|
|
def auto_tags_for(self, text, explicit=(), vocab=None):
|
|
if vocab is None:
|
|
vocab = self.known_tags()
|
|
skip = set(explicit) | set(Consts.TAG_AUTO_SKIP)
|
|
words = re.findall(r"[a-z0-9]+", (text or "").lower())
|
|
found = []
|
|
for pos, word in enumerate(words):
|
|
if len(found) >= Consts.TAG_AUTO_MAX:
|
|
break
|
|
if pos + 1 < len(words):
|
|
pair = word + "-" + words[pos + 1]
|
|
if pair in vocab and pair not in skip and pair not in found:
|
|
found.append(pair)
|
|
continue
|
|
if word in vocab and word not in skip and word not in found:
|
|
found.append(word)
|
|
continue
|
|
single = Text.singular_noun(word)
|
|
if single != word and single in vocab and single not in skip and single not in found:
|
|
found.append(single)
|
|
return found
|
|
|
|
def tag_neighbors(self, record_id, limit=None, profile=None):
|
|
profile = self._scope(profile)
|
|
rows = self.db.execute("SELECT r.id, MIN(t.tag) FROM records r JOIN tags t ON t.item = r.id WHERE t.tag IN (SELECT tag FROM tags WHERE item = ? AND profile = ? AND tag NOT IN ('record', 'file')) AND r.id != ? AND r.profile = ? AND t.profile = ? GROUP BY r.id ORDER BY r.updated DESC LIMIT ?", (record_id, profile, record_id, profile, profile, max(1, limit or Consts.TAG_LINK_MAX))).fetchall()
|
|
return [(row[0], row[1]) for row in rows]
|
|
|
|
def tag_item(self, item, tags, profile=None):
|
|
profile = self._scope(profile)
|
|
for tag in self.canonical_tags(tags, None, profile):
|
|
self.db.execute("INSERT OR IGNORE INTO tags (item, tag, profile) VALUES (?, ?, ?)", (item, tag, profile))
|
|
self.db.commit()
|
|
|
|
def set_item_tags(self, item, tags, profile=None):
|
|
profile = self._scope(profile)
|
|
self.db.execute("DELETE FROM tags WHERE item = ? AND profile = ?", (item, profile))
|
|
for tag in self.canonical_tags(tags, None, profile):
|
|
self.db.execute("INSERT OR IGNORE INTO tags (item, tag, profile) VALUES (?, ?, ?)", (item, tag, profile))
|
|
self.db.commit()
|
|
|
|
def item_tags(self, item, profile=None):
|
|
profile = self._scope(profile)
|
|
return [row[0] for row in self.db.execute("SELECT tag FROM tags WHERE item = ? AND profile = ? ORDER BY tag", (item, profile)).fetchall()]
|
|
|
|
def tagged_items(self, tags, profile=None):
|
|
profile = self._scope(profile)
|
|
clean = Text.normalize_tags(tags)
|
|
if not clean:
|
|
return None
|
|
marks = ", ".join("?" * len(clean))
|
|
rows = self.db.execute("SELECT item FROM tags WHERE profile = ? AND tag IN (%s) GROUP BY item HAVING COUNT(DISTINCT tag) = ?" % marks, (profile,) + tuple(clean) + (len(clean),)).fetchall()
|
|
return [row[0] for row in rows]
|
|
|
|
def has_node(self, kind, key, profile=None):
|
|
profile = self._scope(profile)
|
|
if kind == "mem":
|
|
row = self.db.execute("SELECT 1 FROM records WHERE profile = ? AND id = ?", (profile, "mem:" + key,)).fetchone()
|
|
elif kind == "secret":
|
|
row = self.db.execute("SELECT 1 FROM secrets WHERE profile = ? AND name = ?", (profile, key,)).fetchone()
|
|
else:
|
|
row = self.db.execute("SELECT 1 FROM schedules WHERE profile = ? AND id = ?", (profile, int(key),)).fetchone()
|
|
return row is not None
|
|
|
|
def node_title(self, node, profile=None):
|
|
profile = self._scope(profile)
|
|
parsed = Text.parse_node_id(node)
|
|
if parsed is None:
|
|
return node
|
|
kind, key = parsed
|
|
if kind == "mem":
|
|
row = self.db.execute("SELECT title FROM records WHERE profile = ? AND id = ?", (profile, node,)).fetchone()
|
|
return row[0] or node if row else node
|
|
if kind == "secret":
|
|
return "secret:" + key
|
|
row = self.db.execute("SELECT name, prompt FROM schedules WHERE profile = ? AND id = ?", (profile, int(key),)).fetchone()
|
|
if not row:
|
|
return node
|
|
return row[0] or "schedule #%s" % key
|
|
|
|
def add_record(self, kind, title, content, tags=(), profile=None):
|
|
profile = self._scope(profile)
|
|
record_id = "mem:" + uuid.uuid4().hex[:16]
|
|
while self.db.execute("SELECT 1 FROM records WHERE id = ?", (record_id,)).fetchone():
|
|
record_id = "mem:" + uuid.uuid4().hex[:16]
|
|
stamp = datetime.now(timezone.utc).isoformat()
|
|
size = len(content.encode("utf-8"))
|
|
self.db.execute("INSERT INTO records(id, kind, profile, title, content, size, reads, created, updated) VALUES(?, ?, ?, ?, ?, ?, 0, ?, ?)", (record_id, kind, profile, title, content, size, stamp, stamp))
|
|
vocab = self.known_tags(profile)
|
|
explicit = self.canonical_tags(tags, vocab, profile)
|
|
scan = "%s\n%s" % (title, content.split("\n...[truncated", 1)[0])
|
|
auto = self.auto_tags_for(scan, explicit, vocab)
|
|
self.set_item_tags(record_id, ["record"] + explicit + auto, profile)
|
|
for other, shared in self.tag_neighbors(record_id, None, profile):
|
|
self.db.execute("INSERT OR IGNORE INTO edges(src, dst, relation, profile, created) VALUES(?, ?, ?, ?, ?)", (record_id, other, "shares-" + shared, profile, stamp))
|
|
self.db.commit()
|
|
return record_id
|
|
|
|
def add_chunks(self, parent_kind, parent_id, texts, profile=None):
|
|
"""Replaces any prior chunks for this parent, so re-indexing is idempotent."""
|
|
profile = self._scope(profile)
|
|
self.db.execute("DELETE FROM chunks WHERE parent_kind = ? AND parent_id = ? AND profile = ?", (parent_kind, parent_id, profile))
|
|
stamp = datetime.now(timezone.utc).isoformat()
|
|
for idx, text in enumerate(texts):
|
|
self.db.execute("INSERT INTO chunks(parent_kind, parent_id, idx, text, profile, created) VALUES (?, ?, ?, ?, ?, ?)", (parent_kind, parent_id, idx, text, profile, stamp))
|
|
self.db.commit()
|
|
|
|
def get_chunks_by_ids(self, ids, profile=None):
|
|
if not ids:
|
|
return {}
|
|
profile = self._scope(profile)
|
|
marks = ", ".join("?" * len(ids))
|
|
rows = self.db.execute("SELECT id, parent_kind, parent_id, idx, text FROM chunks WHERE profile = ? AND id IN (%s)" % marks, tuple([profile] + list(ids))).fetchall()
|
|
return {row[0]: {"parent_kind": row[1], "parent_id": row[2], "idx": row[3], "text": row[4]} for row in rows}
|
|
|
|
def get_record(self, record_id, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT id, kind, title, content, size, reads, created, updated FROM records WHERE profile = ? AND id = ?", (profile, record_id,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
self.db.execute("UPDATE records SET reads = reads + 1 WHERE id = ?", (record_id,))
|
|
self.db.commit()
|
|
return {"id": row[0], "kind": row[1], "title": row[2], "content": row[3], "size": Text.as_int(row[4]), "reads": Text.as_int(row[5]) + 1, "created": row[6], "updated": row[7], "tags": self.item_tags(row[0], profile)}
|
|
|
|
def read_record(self, record_id, offset=0, limit=4000, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT content FROM records WHERE profile = ? AND id = ?", (profile, record_id,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
content = row[0]
|
|
total = len(content)
|
|
start = max(0, min(offset, total))
|
|
end = min(total, start + max(1, limit))
|
|
self.db.execute("UPDATE records SET reads = reads + 1 WHERE id = ?", (record_id,))
|
|
self.db.commit()
|
|
return {"id": record_id, "slice": content[start:end], "start": start, "end": end, "total": total, "tags": self.item_tags(record_id, profile)}
|
|
|
|
def search_records(self, query="", kind=None, tags=(), limit=8, profile=None):
|
|
profile = self._scope(profile)
|
|
clean = Text.normalize_tags(tags)
|
|
terms = Store.fts_escape(query) if query else None
|
|
if terms is not None and self.fts_ok:
|
|
ranked = self.search_records_fts(terms, kind, clean, limit, profile)
|
|
if ranked:
|
|
return ranked
|
|
sql = "SELECT id, kind, title, size, reads, updated FROM records"
|
|
clauses = ["profile = ?"]
|
|
params = [profile]
|
|
if kind:
|
|
clauses.append("kind = ?")
|
|
params.append(kind)
|
|
if query:
|
|
clauses.append("(title LIKE ? OR content LIKE ?)")
|
|
params += ["%" + query + "%", "%" + query + "%"]
|
|
if clean:
|
|
for tag in clean:
|
|
options = Text.tag_variants(tag)
|
|
marks = ", ".join("?" * len(options))
|
|
clauses.append("id IN (SELECT item FROM tags WHERE profile = ? AND tag IN (%s))" % marks)
|
|
params += [profile] + options
|
|
sql += " WHERE " + " AND ".join(clauses)
|
|
sql += " ORDER BY updated DESC LIMIT ?"
|
|
params.append(max(1, min(limit, 50)))
|
|
rows = self.db.execute(sql, tuple(params)).fetchall()
|
|
return [{"id": row[0], "kind": row[1], "title": row[2], "size": row[3], "reads": row[4], "updated": row[5], "tags": self.item_tags(row[0], profile)} for row in rows]
|
|
|
|
def search_records_fts(self, terms, kind, clean, limit, profile):
|
|
sql = "SELECT r.id, r.kind, r.title, r.size, r.reads, r.updated FROM records r JOIN fts_docs f ON f.item = r.id AND f.kind = 'record' WHERE r.profile = ? AND f MATCH ?"
|
|
params = [profile, Store.fts_match(terms)]
|
|
if kind:
|
|
sql += " AND r.kind = ?"
|
|
params.append(kind)
|
|
for tag in clean:
|
|
options = Text.tag_variants(tag)
|
|
marks = ", ".join("?" * len(options))
|
|
sql += " AND r.id IN (SELECT item FROM tags WHERE profile = ? AND tag IN (%s))" % marks
|
|
params += [profile] + options
|
|
sql += " ORDER BY bm25(f) LIMIT ?"
|
|
params.append(max(1, min(limit, 50)))
|
|
try:
|
|
rows = self.db.execute(sql, tuple(params)).fetchall()
|
|
except sqlite3.Error:
|
|
return []
|
|
return [{"id": row[0], "kind": row[1], "title": row[2], "size": row[3], "reads": row[4], "updated": row[5], "tags": self.item_tags(row[0], profile)} for row in rows]
|
|
|
|
def delete_record(self, record_id, profile=None):
|
|
profile = self._scope(profile)
|
|
done = self.db.execute("DELETE FROM records WHERE profile = ? AND id = ?", (profile, record_id,))
|
|
if done.rowcount > 0:
|
|
self.db.execute("DELETE FROM tags WHERE item = ? AND profile = ?", (record_id, profile))
|
|
self.db.execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", (profile, record_id, record_id))
|
|
self.db.execute("DELETE FROM chunks WHERE parent_kind = 'record' AND parent_id = ? AND profile = ?", (record_id, profile))
|
|
self.db.commit()
|
|
return True
|
|
self.db.commit()
|
|
return False
|
|
|
|
def add_edge(self, src, dst, relation="linked", profile=None):
|
|
profile = self._scope(profile)
|
|
for node in (src, dst):
|
|
parsed = Text.parse_node_id(node)
|
|
if parsed is None or not self.has_node(*parsed, profile):
|
|
raise ValueError("unknown node " + node)
|
|
clean = re.sub(r"[^a-z0-9]+", "-", str(relation or "").lower()).strip("-")[:32] or "linked"
|
|
self.db.execute("INSERT OR IGNORE INTO edges(src, dst, relation, profile, created) VALUES(?, ?, ?, ?, ?)", (src, dst, clean, profile, datetime.now(timezone.utc).isoformat()))
|
|
self.db.commit()
|
|
return clean
|
|
|
|
def edges_for(self, node, profile=None):
|
|
profile = self._scope(profile)
|
|
rows = self.db.execute("SELECT src, dst, relation FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", (profile, node, node)).fetchall()
|
|
return [{"relation": row[2], "other": row[1] if row[0] == node else row[0], "direction": "out" if row[0] == node else "in"} for row in rows]
|
|
|
|
def remove_edges(self, node, profile=None):
|
|
profile = self._scope(profile)
|
|
done = self.db.execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", (profile, node, node))
|
|
self.db.commit()
|
|
return done.rowcount
|
|
|
|
def traverse(self, start, depth=2, limit=50, profile=None):
|
|
profile = self._scope(profile)
|
|
seen = {start}
|
|
found = [{"node": start, "title": self.node_title(start, profile), "depth": 0, "via": ""}]
|
|
frontier = [(start, 0, "")]
|
|
capped = max(1, min(depth, 4))
|
|
while frontier and len(found) < max(1, min(limit, 200)):
|
|
node, level, _via = frontier.pop(0)
|
|
if level >= capped:
|
|
continue
|
|
for edge in self.edges_for(node, profile):
|
|
other = edge["other"]
|
|
if other in seen:
|
|
continue
|
|
seen.add(other)
|
|
hop = "%s -[%s]-> %s" % (node, edge["relation"], other) if edge["direction"] == "out" else "%s <-[%s]- %s" % (node, edge["relation"], other)
|
|
found.append({"node": other, "title": self.node_title(other, profile), "depth": level + 1, "via": hop})
|
|
frontier.append((other, level + 1, hop))
|
|
if len(found) >= max(1, min(limit, 200)):
|
|
break
|
|
return found
|
|
|
|
def audit_event(self, actor, action, path, message="", old=None, new=None, tags=(), old_size=None, new_size=None, profile=None):
|
|
profile = self._scope(profile)
|
|
old_text, computed_old = Text.capped_text(old)
|
|
new_text, computed_new = Text.capped_text(new)
|
|
if old is not None and old_size is not None and old_size > len(old.encode("utf-8")):
|
|
old_text = old[:Consts.AUDIT_MAX_CHARS] + Text.trunc_marker(old_size)
|
|
computed_old = old_size
|
|
if new is not None and new_size is not None and new_size > len(new.encode("utf-8")):
|
|
new_text = new[:Consts.AUDIT_MAX_CHARS] + Text.trunc_marker(new_size)
|
|
computed_new = new_size
|
|
cursor = self.db.execute("INSERT INTO audit(profile, ts, actor, action, path, message, old_size, new_size, old, new, tags) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (profile, datetime.now(timezone.utc).isoformat(), actor, action, path, str(message or "")[:2000], computed_old, computed_new, old_text, new_text, " ".join(Text.normalize_tags(tags))))
|
|
self.db.commit()
|
|
return cursor.lastrowid
|
|
|
|
def audit_history(self, path=None, tag=None, limit=20, profile=None):
|
|
profile = self._scope(profile)
|
|
sql = "SELECT id, ts, actor, action, path, message, old_size, new_size, tags FROM audit"
|
|
clauses = ["profile = ?"]
|
|
params = [profile]
|
|
if path:
|
|
clauses.append("path = ?")
|
|
params.append(path)
|
|
if tag:
|
|
clauses.append("tags LIKE ?")
|
|
params.append("%" + tag + "%")
|
|
sql += " WHERE " + " AND ".join(clauses)
|
|
sql += " ORDER BY id DESC LIMIT ?"
|
|
params.append(max(1, min(limit, 100)))
|
|
rows = self.db.execute(sql, tuple(params)).fetchall()
|
|
return [{"id": row[0], "ts": row[1], "actor": row[2], "action": row[3], "path": row[4], "message": row[5], "old_size": row[6], "new_size": row[7], "tags": row[8]} for row in rows]
|
|
|
|
def audit_get(self, row_id, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT id, ts, actor, action, path, message, old_size, new_size, old, new, tags FROM audit WHERE profile = ? AND id = ?", (profile, row_id,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
return {"id": row[0], "ts": row[1], "actor": row[2], "action": row[3], "path": row[4], "message": row[5], "old_size": row[6], "new_size": row[7], "old": row[8], "new": row[9], "tags": row[10]}
|
|
|
|
def file_record_id(self, path, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT id FROM records WHERE profile = ? AND kind = 'file' AND title = ?", (profile, path,)).fetchone()
|
|
return row[0] if row else None
|
|
|
|
def upsert_file_record(self, path, content, true_size=None, profile=None):
|
|
profile = self._scope(profile)
|
|
full_bytes = len(content.encode("utf-8"))
|
|
if true_size is not None and true_size > full_bytes:
|
|
text = content + Text.trunc_marker(true_size)
|
|
elif len(content) <= Consts.FILE_RECORD_MAX:
|
|
text = content
|
|
else:
|
|
text = content[:Consts.FILE_RECORD_MAX] + Text.trunc_marker(full_bytes)
|
|
ext = os.path.splitext(path)[1].lstrip(".").lower()
|
|
explicit = ["file"] + ([ext] if ext and ext.isalnum() and len(ext) <= 5 else [])
|
|
vocab = self.known_tags(profile)
|
|
tags = explicit + self.auto_tags_for("%s\n%s" % (path, content), explicit, vocab)
|
|
record_id = self.file_record_id(path, profile)
|
|
stamp = datetime.now(timezone.utc).isoformat()
|
|
if record_id is None:
|
|
return self.add_record("file", path, text, tags, profile)
|
|
self.db.execute("UPDATE records SET content = ?, size = ?, updated = ? WHERE id = ?", (text, len(text.encode("utf-8")), stamp, record_id))
|
|
self.set_item_tags(record_id, tags, profile)
|
|
self.db.commit()
|
|
return record_id
|
|
|
|
def delete_file_record(self, path, profile=None):
|
|
profile = self._scope(profile)
|
|
record_id = self.file_record_id(path, profile)
|
|
if record_id is None:
|
|
return False
|
|
return self.delete_record(record_id, profile)
|
|
|
|
def load_secret(self, name, profile=None):
|
|
profile = self._scope(profile)
|
|
row = self.db.execute("SELECT value FROM secrets WHERE profile = ? AND name = ?", (profile, name,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
if row[0].startswith(Seal.PREFIX):
|
|
return self.seal.unlock(row[0]) if self.seal.enabled else None
|
|
return row[0]
|
|
|
|
def delete_secret(self, name, profile=None):
|
|
profile = self._scope(profile)
|
|
self.secret_cache.pop(profile, None)
|
|
done = self.db.execute("DELETE FROM secrets WHERE profile = ? AND name = ?", (profile, name,))
|
|
if done.rowcount > 0:
|
|
item = "secret:" + name
|
|
self.db.execute("DELETE FROM tags WHERE item = ? AND profile = ?", (item, profile))
|
|
self.db.execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", (profile, item, item))
|
|
self.db.commit()
|
|
return True
|
|
self.db.commit()
|
|
return False
|
|
|
|
def secret_values(self, profile=None):
|
|
profile = self._scope(profile)
|
|
if profile not in self.secret_cache:
|
|
found = {}
|
|
for name, value in self.db.execute("SELECT name, value FROM secrets WHERE profile = ?", (profile,)).fetchall():
|
|
if value.startswith(Seal.PREFIX):
|
|
if not self.seal.enabled:
|
|
continue
|
|
value = self.seal.unlock(value)
|
|
if len(value) >= 4:
|
|
found[name] = value
|
|
self.secret_cache[profile] = found
|
|
return self.secret_cache[profile]
|
|
|
|
def redact(self, text, profile=None):
|
|
if not isinstance(text, str):
|
|
return text
|
|
for name, value in sorted(self.secret_values(profile).items(), key=lambda item: -len(item[1])):
|
|
if value in text:
|
|
text = text.replace(value, "[redacted:%s]" % name)
|
|
return text
|
|
|
|
def add_schedule(self, name, prompt, profile, every_sec, next_run, timeout, tags=None):
|
|
stored = self.seal.lock(prompt) if self.seal.enabled else prompt
|
|
done = self.db.execute("INSERT INTO schedules (name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created, updated) VALUES (?, ?, ?, ?, ?, ?, 'pending', '', '', ?, ?)", (name, stored, profile, every_sec, next_run, timeout, Text.now_iso(), Text.now_iso()))
|
|
item = "sched:%s" % done.lastrowid
|
|
if tags is None:
|
|
self.tag_item(item, ["schedule"], profile)
|
|
else:
|
|
self.set_item_tags(item, ["schedule"] + list(tags), profile)
|
|
self.db.commit()
|
|
return done.lastrowid
|
|
|
|
def remove_schedule(self, row_id, profile=None):
|
|
profile = self._scope(profile)
|
|
done = self.db.execute("DELETE FROM schedules WHERE id = ? AND profile = ?", (row_id, profile))
|
|
if done.rowcount > 0:
|
|
item = "sched:%s" % row_id
|
|
self.db.execute("DELETE FROM tags WHERE item = ? AND profile = ?", (item, profile))
|
|
self.db.execute("DELETE FROM edges WHERE profile = ? AND (src = ? OR dst = ?)", (profile, item, item))
|
|
self.db.commit()
|
|
return True
|
|
self.db.commit()
|
|
return False
|
|
|
|
def list_schedules(self, profile=None):
|
|
profile = self._scope(profile)
|
|
rows = self.db.execute("SELECT id, name, prompt, profile, every_sec, next_run, timeout, status, last_status, last_result, created FROM schedules WHERE profile = ? ORDER BY next_run", (profile,)).fetchall()
|
|
items = []
|
|
for row in rows:
|
|
prompt = self.seal.safe_unlock(row[2]) if row[2].startswith(Seal.PREFIX) else row[2]
|
|
items.append({"id": row[0], "name": row[1], "prompt": prompt, "profile": row[3], "every": row[4], "next_run": row[5], "timeout": row[6], "status": row[7], "last": row[8], "result": row[9], "created": row[10], "tags": self.item_tags("sched:%s" % row[0], row[3])})
|
|
return items
|
|
|
|
def profile_path(self, name):
|
|
return os.path.join(self.config.profiles_dir, name + ".sys.md")
|
|
|
|
def session_path(self, name):
|
|
return os.path.join(self.config.profiles_dir, name + ".session.json")
|
|
|
|
def list_profiles(self):
|
|
names = []
|
|
for entry in sorted(os.listdir(self.config.profiles_dir)):
|
|
if entry.endswith(".sys.md") and ".bot." not in entry:
|
|
names.append(entry[:-7])
|
|
return names
|
|
|
|
def unseal_file(self, raw):
|
|
if raw.startswith(Seal.PREFIX):
|
|
if not self.seal.enabled:
|
|
raise SealError("profile is sealed, set TAI_PASSPHRASE")
|
|
return self.seal.unlock(raw)
|
|
return raw
|
|
|
|
def load_system(self, name):
|
|
path = self.profile_path(name)
|
|
if not os.path.exists(path):
|
|
return None
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return self.unseal_file(handle.read())
|
|
except (OSError, ValueError):
|
|
Release.quarantine(path)
|
|
return None
|
|
except SealError:
|
|
if not self.seal.enabled:
|
|
raise
|
|
Release.quarantine(path)
|
|
return None
|
|
|
|
def save_system(self, name, text):
|
|
try:
|
|
Release.sealed_write(self.profile_path(name), self.seal.lock(text))
|
|
except OSError:
|
|
pass
|
|
|
|
def load_session(self, name, bot="main"):
|
|
path = self.session_path(name) if bot == "main" else self.bot_session_path(name, bot)
|
|
if not os.path.exists(path):
|
|
return []
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
raw = handle.read()
|
|
data = json.loads(self.unseal_file(raw))
|
|
return [item for item in data if isinstance(item, dict) and item.get("role") in ("user", "assistant", "tool")]
|
|
except (OSError, ValueError):
|
|
Release.quarantine(path)
|
|
return []
|
|
except SealError:
|
|
if not self.seal.enabled:
|
|
raise
|
|
Release.quarantine(path)
|
|
return []
|
|
|
|
def save_session(self, name, messages, bot="main"):
|
|
path = self.session_path(name) if bot == "main" else self.bot_session_path(name, bot)
|
|
try:
|
|
Release.sealed_write(path, self.seal.lock(json.dumps([item for item in messages if item.get("role") != "system"])))
|
|
except OSError:
|
|
pass
|
|
|
|
def turn_state_path(self, profile, bot="main"):
|
|
if bot == "main":
|
|
return os.path.join(self.config.profiles_dir, "%s.turn.json" % profile)
|
|
return os.path.join(self.config.profiles_dir, "%s.bot.%s.turn.json" % (profile, bot))
|
|
|
|
def load_turn_state(self, profile, bot="main"):
|
|
path = self.turn_state_path(profile, bot)
|
|
if not os.path.exists(path):
|
|
return {}
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
data = json.loads(self.unseal_file(handle.read()))
|
|
return data if isinstance(data, dict) else {}
|
|
except (OSError, ValueError):
|
|
Release.quarantine(path)
|
|
return {}
|
|
except SealError:
|
|
if not self.seal.enabled:
|
|
raise
|
|
Release.quarantine(path)
|
|
return {}
|
|
|
|
def save_turn_state(self, profile, bot, state):
|
|
try:
|
|
Release.sealed_write(self.turn_state_path(profile, bot), self.seal.lock(json.dumps(state)))
|
|
except OSError:
|
|
pass
|
|
|
|
def bot_system_path(self, profile, bot):
|
|
return os.path.join(self.config.profiles_dir, "%s.bot.%s.sys.md" % (profile, bot))
|
|
|
|
def bot_session_path(self, profile, bot):
|
|
return os.path.join(self.config.profiles_dir, "%s.bot.%s.session.json" % (profile, bot))
|
|
|
|
def bots_registry_path(self, profile):
|
|
return os.path.join(self.config.profiles_dir, "%s.bots.json" % profile)
|
|
|
|
def load_bots(self, profile):
|
|
path = self.bots_registry_path(profile)
|
|
if not os.path.exists(path):
|
|
return {}
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
data = json.loads(self.unseal_file(handle.read()))
|
|
bots = data.get("bots") if isinstance(data, dict) else None
|
|
return bots if isinstance(bots, dict) else {}
|
|
except (OSError, ValueError):
|
|
Release.quarantine(path)
|
|
return {}
|
|
except SealError:
|
|
if not self.seal.enabled:
|
|
raise
|
|
Release.quarantine(path)
|
|
return {}
|
|
|
|
def save_bots(self, profile, bots):
|
|
try:
|
|
Release.sealed_write(self.bots_registry_path(profile), self.seal.lock(json.dumps({"bots": bots})))
|
|
except OSError:
|
|
pass
|
|
|
|
def list_bots(self, profile):
|
|
bots = self.load_bots(profile)
|
|
found = [{"name": "main", "nicknames": []}]
|
|
for name in sorted(bots):
|
|
found.append({"name": name, "nicknames": (bots.get(name) or {}).get("nicknames", [])})
|
|
return found
|
|
|
|
def resolve_bot(self, profile, mention):
|
|
want = str(mention or "").strip().lower()
|
|
if want in ("", "main"):
|
|
return "main"
|
|
bots = self.load_bots(profile)
|
|
if want in bots:
|
|
return want
|
|
for name in sorted(bots):
|
|
if want in (bots.get(name) or {}).get("nicknames", []):
|
|
return name
|
|
return None
|
|
|
|
def load_bot_system(self, profile, bot):
|
|
if bot == "main":
|
|
return self.load_system(profile)
|
|
path = self.bot_system_path(profile, bot)
|
|
if not os.path.exists(path):
|
|
return None
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return self.unseal_file(handle.read())
|
|
except (OSError, ValueError):
|
|
Release.quarantine(path)
|
|
return None
|
|
except SealError:
|
|
if not self.seal.enabled:
|
|
raise
|
|
Release.quarantine(path)
|
|
return None
|
|
|
|
def save_bot_system(self, profile, bot, text):
|
|
try:
|
|
Release.sealed_write(self.bot_system_path(profile, bot), self.seal.lock(text))
|
|
except OSError:
|
|
pass
|
|
|
|
def close(self):
|
|
try:
|
|
self.db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
except sqlite3.Error:
|
|
pass
|
|
try:
|
|
self.db.close()
|
|
except sqlite3.Error:
|
|
pass
|
|
if getattr(self, "memdb", None) is not None:
|
|
try:
|
|
self.memdb.close()
|
|
except sqlite3.Error:
|
|
pass
|
|
self.memdb = None
|
|
|
|
@staticmethod
|
|
def ensure_column(db, table, column, ddl):
|
|
names = [row[1] for row in db.execute("PRAGMA table_info(%s)" % table).fetchall()]
|
|
if column not in names:
|
|
db.execute("ALTER TABLE %s ADD COLUMN %s %s" % (table, column, ddl))
|
|
|
|
@staticmethod
|
|
def fts_escape(query):
|
|
terms = re.findall(r"[a-z0-9]+", (query or "").lower())
|
|
if not terms:
|
|
return None
|
|
return ['"%s"' % term for term in terms]
|
|
|
|
@staticmethod
|
|
def fts_match(terms):
|
|
return " AND ".join("{title body} : %s" % term for term in terms)
|
|
|
|
@staticmethod
|
|
def open_mem_events():
|
|
mem = sqlite3.connect(":memory:")
|
|
try:
|
|
mem.execute("CREATE VIRTUAL TABLE mem_events USING fts5(item, kind, sub, title, body, profile, tokenize='porter unicode61 remove_diacritics 2')")
|
|
except sqlite3.Error:
|
|
mem.execute("CREATE VIRTUAL TABLE mem_events USING fts5(item, kind, sub, title, body, profile)")
|
|
return mem
|
|
|
|
@staticmethod
|
|
def setup_fts(db):
|
|
try:
|
|
exists = db.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'fts_docs'").fetchone()
|
|
if not exists:
|
|
try:
|
|
db.execute("CREATE VIRTUAL TABLE fts_docs USING fts5(item, kind, sub, title, body, profile, tokenize='porter unicode61 remove_diacritics 2')")
|
|
except sqlite3.Error:
|
|
db.execute("CREATE VIRTUAL TABLE fts_docs USING fts5(item, kind, sub, title, body, profile)")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_records_ai AFTER INSERT ON records BEGIN
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'record', new.kind, new.title, new.content, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_records_au AFTER UPDATE ON records WHEN old.title != new.title OR old.content != new.content OR old.profile != new.profile BEGIN
|
|
DELETE FROM fts_docs WHERE item = old.id AND kind = 'record';
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'record', new.kind, new.title, new.content, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_records_ad AFTER DELETE ON records BEGIN
|
|
DELETE FROM fts_docs WHERE item = old.id AND kind = 'record';
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_events_ai AFTER INSERT ON events BEGIN
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'event', new.role || '/' || new.kind, new.ts || ' ' || new.role || '/' || new.kind, new.text, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_events_au AFTER UPDATE ON events WHEN old.text != new.text OR old.profile != new.profile BEGIN
|
|
DELETE FROM fts_docs WHERE item = old.id AND kind = 'event';
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES(new.id, 'event', new.role || '/' || new.kind, new.ts || ' ' || new.role || '/' || new.kind, new.text, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_audit_ai AFTER INSERT ON audit BEGIN
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES('audit:' || new.id, 'audit', new.action, new.path, new.message, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_chunks_ai AFTER INSERT ON chunks BEGIN
|
|
INSERT INTO fts_docs(item, kind, sub, title, body, profile) VALUES('chunk:' || new.id, 'chunk', new.parent_kind, new.parent_id, new.text, new.profile);
|
|
END""")
|
|
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_chunks_ad AFTER DELETE ON chunks BEGIN
|
|
DELETE FROM fts_docs WHERE item = 'chunk:' || old.id AND kind = 'chunk';
|
|
END""")
|
|
if db.execute("SELECT COUNT(*) FROM fts_docs").fetchone()[0] == 0:
|
|
db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT id, 'record', kind, title, content, profile FROM records")
|
|
db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT id, 'event', role || '/' || kind, ts || ' ' || role || '/' || kind, text, profile FROM events")
|
|
db.execute("INSERT INTO fts_docs(item, kind, sub, title, body, profile) SELECT 'audit:' || id, 'audit', action, path, message, profile FROM audit")
|
|
db.commit()
|
|
return True
|
|
except sqlite3.Error:
|
|
return False
|
|
|
|
@staticmethod
|
|
def table_pk_columns(db, table):
|
|
for row in db.execute("PRAGMA index_list(%s)" % table).fetchall():
|
|
if row[3] == "pk":
|
|
return [info[2] for info in db.execute("PRAGMA index_info(%s)" % row[1]).fetchall()]
|
|
return []
|
|
|
|
@staticmethod
|
|
def rebuild_with_profile(db, table, columns, pk):
|
|
if Store.table_pk_columns(db, table) == list(pk):
|
|
return False
|
|
names = [column.split()[0] for column in columns]
|
|
have = {row[1] for row in db.execute("PRAGMA table_info(%s)" % table).fetchall()}
|
|
keep = [name for name in names if name in have]
|
|
db.execute("CREATE TABLE %s_new (%s, PRIMARY KEY (%s))" % (table, ", ".join(columns), ", ".join(pk)))
|
|
if keep:
|
|
db.execute("INSERT OR IGNORE INTO %s_new (%s) SELECT %s FROM %s" % (table, ", ".join(keep), ", ".join(keep), table))
|
|
db.execute("DROP TABLE %s" % table)
|
|
db.execute("ALTER TABLE %s_new RENAME TO %s" % (table, table))
|
|
return True
|
|
|
|
@staticmethod
|
|
def migrate_profiles(db):
|
|
for table in ("secrets", "records", "tags", "edges", "audit", "events", "schedules"):
|
|
Store.ensure_column(db, table, "profile", "TEXT")
|
|
db.execute("UPDATE %s SET profile = 'default' WHERE profile IS NULL" % table)
|
|
Store.rebuild_with_profile(db, "secrets", ("profile TEXT", "name TEXT", "value TEXT", "meta TEXT", "updated TEXT"), ("profile", "name"))
|
|
Store.rebuild_with_profile(db, "tags", ("item TEXT", "tag TEXT", "profile TEXT"), ("item", "tag", "profile"))
|
|
Store.rebuild_with_profile(db, "edges", ("src TEXT", "dst TEXT", "relation TEXT", "profile TEXT", "created TEXT"), ("src", "dst", "relation", "profile"))
|
|
|
|
@staticmethod
|
|
def ensure_schema(db):
|
|
notes = []
|
|
for table, columns, pk in Consts.SCHEMA:
|
|
exists = db.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)).fetchone()
|
|
if not exists:
|
|
extra = ", PRIMARY KEY (%s)" % ", ".join(pk) if pk else ""
|
|
db.execute("CREATE TABLE %s (%s%s)" % (table, ", ".join(columns), extra))
|
|
continue
|
|
have = {row[1] for row in db.execute("PRAGMA table_info(%s)" % table).fetchall()}
|
|
for column in columns:
|
|
parts = column.split()
|
|
if parts[0] not in have:
|
|
db.execute("ALTER TABLE %s ADD COLUMN %s" % (table, " ".join(parts[:2])))
|
|
notes.append("added %s.%s" % (table, parts[0]))
|
|
for index, target in Consts.SCHEMA_INDEXES:
|
|
db.execute("CREATE INDEX IF NOT EXISTS %s ON %s" % (index, target))
|
|
db.commit()
|
|
return notes
|