feat: add audit log tables, indexes, and CLI/content recording hooks
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.audit import store, record, query, categories
|
||||
from devplacepy.services.audit.service import AuditService
|
||||
|
||||
__all__ = ["store", "record", "query", "categories", "AuditService"]
|
||||
@@ -0,0 +1,49 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
CATEGORY_BY_PREFIX: dict[str, str] = {
|
||||
"auth": "auth",
|
||||
"profile": "account",
|
||||
"follow": "social",
|
||||
"push": "push",
|
||||
"notification": "notification",
|
||||
"message": "message",
|
||||
"post": "content",
|
||||
"comment": "content",
|
||||
"gist": "content",
|
||||
"bug": "content",
|
||||
"vote": "engagement",
|
||||
"reaction": "engagement",
|
||||
"bookmark": "engagement",
|
||||
"poll": "engagement",
|
||||
"project": "project",
|
||||
"file": "project_files",
|
||||
"dir": "project_files",
|
||||
"files": "project_files",
|
||||
"job.zip": "project_files",
|
||||
"attachment": "attachment",
|
||||
"news": "news",
|
||||
"admin": "admin",
|
||||
"service": "service",
|
||||
"container": "container",
|
||||
"proxy": "ingress",
|
||||
"ai": "ai",
|
||||
"devii": "devii",
|
||||
"cli": "cli",
|
||||
"reward": "reward",
|
||||
"security": "security",
|
||||
}
|
||||
|
||||
EVENT_RESULTS = ("success", "failure", "denied")
|
||||
ACTOR_KINDS = ("user", "guest", "system", "cli", "service")
|
||||
ORIGINS = ("web", "api", "devii", "cli", "service", "scheduler")
|
||||
|
||||
|
||||
def category_for(event_key: str) -> str:
|
||||
if not event_key:
|
||||
return "other"
|
||||
parts = event_key.split(".")
|
||||
for size in range(len(parts), 0, -1):
|
||||
prefix = ".".join(parts[:size])
|
||||
if prefix in CATEGORY_BY_PREFIX:
|
||||
return CATEGORY_BY_PREFIX[prefix]
|
||||
return parts[0]
|
||||
@@ -0,0 +1,93 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.database import db, build_pagination
|
||||
from devplacepy.services.audit import store
|
||||
from devplacepy.services.audit.store import AUDIT_TABLE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FILTERABLE = ("event_key", "category", "actor_role", "actor_uid", "origin", "result")
|
||||
|
||||
_options_cache = TTLCache(ttl=60, max_size=8)
|
||||
|
||||
|
||||
def _build_where(filters: dict) -> tuple[str, dict]:
|
||||
clauses: list[str] = []
|
||||
params: dict = {}
|
||||
for key in FILTERABLE:
|
||||
value = filters.get(key)
|
||||
if value:
|
||||
clauses.append(f"{key} = :{key}")
|
||||
params[key] = value
|
||||
query_text = (filters.get("q") or "").strip()
|
||||
if query_text:
|
||||
clauses.append("(summary LIKE :q OR event_key LIKE :q OR target_label LIKE :q)")
|
||||
params["q"] = f"%{query_text}%"
|
||||
date_from = (filters.get("date_from") or "").strip()
|
||||
if date_from:
|
||||
clauses.append("created_at >= :date_from")
|
||||
params["date_from"] = date_from
|
||||
date_to = (filters.get("date_to") or "").strip()
|
||||
if date_to:
|
||||
clauses.append("created_at <= :date_to")
|
||||
params["date_to"] = date_to + "~"
|
||||
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
return where, params
|
||||
|
||||
|
||||
def list_events(filters: dict, page: int, per_page: int = 25) -> tuple[list, dict]:
|
||||
if AUDIT_TABLE not in db.tables:
|
||||
return [], build_pagination(page, 0, per_page)
|
||||
where, params = _build_where(filters)
|
||||
total_rows = list(
|
||||
db.query(f"SELECT COUNT(*) AS c FROM {AUDIT_TABLE}{where}", **params)
|
||||
)
|
||||
total = int(total_rows[0]["c"]) if total_rows else 0
|
||||
pagination = build_pagination(page, total, per_page)
|
||||
offset = (pagination["page"] - 1) * pagination["per_page"]
|
||||
rows = list(
|
||||
db.query(
|
||||
f"SELECT * FROM {AUDIT_TABLE}{where} "
|
||||
f"ORDER BY created_at DESC LIMIT :limit OFFSET :offset",
|
||||
limit=pagination["per_page"],
|
||||
offset=offset,
|
||||
**params,
|
||||
)
|
||||
)
|
||||
return rows, pagination
|
||||
|
||||
|
||||
def _distinct(column: str) -> list[str]:
|
||||
if AUDIT_TABLE not in db.tables:
|
||||
return []
|
||||
rows = db.query(
|
||||
f"SELECT DISTINCT {column} AS value FROM {AUDIT_TABLE} "
|
||||
f"WHERE {column} IS NOT NULL AND {column} != '' ORDER BY {column}"
|
||||
)
|
||||
return [row["value"] for row in rows]
|
||||
|
||||
|
||||
def filter_options() -> dict:
|
||||
cached = _options_cache.get("options")
|
||||
if cached is not None:
|
||||
return cached
|
||||
options = {
|
||||
"event_key": _distinct("event_key"),
|
||||
"category": _distinct("category"),
|
||||
"actor_role": _distinct("actor_role"),
|
||||
"origin": _distinct("origin"),
|
||||
"result": _distinct("result"),
|
||||
}
|
||||
_options_cache.set("options", options)
|
||||
return options
|
||||
|
||||
|
||||
def get_event_with_links(uid: str) -> Optional[dict]:
|
||||
event = store.get_event(uid)
|
||||
if not event:
|
||||
return None
|
||||
return {"event": event, "links": store.get_links(uid)}
|
||||
@@ -0,0 +1,327 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from devplacepy.utils import strip_html
|
||||
from devplacepy.services.audit import store
|
||||
from devplacepy.services.audit.categories import category_for
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNSET = object()
|
||||
SUMMARY_LIMIT = 140
|
||||
|
||||
|
||||
def link(
|
||||
relation: str,
|
||||
object_type: str,
|
||||
object_uid: Optional[str],
|
||||
object_label: Optional[str] = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"relation": relation,
|
||||
"object_type": object_type,
|
||||
"object_uid": object_uid,
|
||||
"object_label": object_label,
|
||||
}
|
||||
|
||||
|
||||
def actor(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("actor", "user", uid, label)
|
||||
|
||||
|
||||
def target(object_type: str, uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("target", object_type, uid, label)
|
||||
|
||||
|
||||
def parent(object_type: str, uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("parent", object_type, uid, label)
|
||||
|
||||
|
||||
def author(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("author", "user", uid, label)
|
||||
|
||||
|
||||
def recipient(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("recipient", "user", uid, label)
|
||||
|
||||
|
||||
def mention(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("mention", "user", uid, label)
|
||||
|
||||
|
||||
def project(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("project", "project", uid, label)
|
||||
|
||||
|
||||
def attachment_link(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("attachment", "attachment", uid, label)
|
||||
|
||||
|
||||
def source(object_type: str, uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("source", object_type, uid, label)
|
||||
|
||||
|
||||
def destination(object_type: str, uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("destination", object_type, uid, label)
|
||||
|
||||
|
||||
def schedule(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("schedule", "schedule", uid, label)
|
||||
|
||||
|
||||
def instance(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("instance", "instance", uid, label)
|
||||
|
||||
|
||||
def setting(key: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("setting", "setting", key, label or key)
|
||||
|
||||
|
||||
def service_link(name: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("target", "service", name, label or name)
|
||||
|
||||
|
||||
def job(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("job", "job", uid, label)
|
||||
|
||||
|
||||
def poll(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("poll", "poll", uid, label)
|
||||
|
||||
|
||||
def option(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("option", "poll_option", uid, label)
|
||||
|
||||
|
||||
def task(uid: Optional[str], label: Optional[str] = None) -> dict:
|
||||
return link("target", "task", uid, label)
|
||||
|
||||
|
||||
def _actor_from(user: Optional[dict]) -> dict:
|
||||
if not user:
|
||||
return {
|
||||
"actor_kind": "guest",
|
||||
"actor_uid": None,
|
||||
"actor_username": None,
|
||||
"actor_role": "guest",
|
||||
}
|
||||
return {
|
||||
"actor_kind": "user",
|
||||
"actor_uid": user.get("uid"),
|
||||
"actor_username": user.get("username"),
|
||||
"actor_role": "admin" if user.get("role") == "Admin" else "member",
|
||||
}
|
||||
|
||||
|
||||
def _request_fields(request) -> dict:
|
||||
fields: dict = {
|
||||
"request_method": None,
|
||||
"request_path": None,
|
||||
"actor_ip": None,
|
||||
"user_agent": None,
|
||||
"devii": False,
|
||||
}
|
||||
if request is None:
|
||||
return fields
|
||||
try:
|
||||
fields["request_method"] = getattr(request, "method", None)
|
||||
url = getattr(request, "url", None)
|
||||
fields["request_path"] = url.path if url is not None else None
|
||||
client = getattr(request, "client", None)
|
||||
fields["actor_ip"] = client.host if client is not None else None
|
||||
headers = getattr(request, "headers", None)
|
||||
if headers is not None:
|
||||
fields["user_agent"] = headers.get("user-agent")
|
||||
fields["devii"] = headers.get("x-devii-agent") == "1"
|
||||
except Exception as exc:
|
||||
logger.debug("audit request field read failed: %s", exc)
|
||||
return fields
|
||||
|
||||
|
||||
def _sanitize_summary(text: Optional[str], limit: int = SUMMARY_LIMIT) -> Optional[str]:
|
||||
if not text:
|
||||
return None
|
||||
cleaned = strip_html(str(text))
|
||||
if not cleaned:
|
||||
return None
|
||||
if len(cleaned) > limit:
|
||||
return cleaned[: limit - 3].rstrip() + "..."
|
||||
return cleaned
|
||||
|
||||
|
||||
def _coerce_scalar(value: Any) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return "1" if value else "0"
|
||||
return str(value)
|
||||
|
||||
|
||||
def _write(
|
||||
*,
|
||||
event_key: str,
|
||||
base_actor: dict,
|
||||
request_fields: dict,
|
||||
target_type: Optional[str],
|
||||
target_uid: Optional[str],
|
||||
target_label: Optional[str],
|
||||
old_value: Any,
|
||||
new_value: Any,
|
||||
summary: Optional[str],
|
||||
metadata: Optional[dict],
|
||||
result: str,
|
||||
origin: Optional[str],
|
||||
via_agent: Optional[int],
|
||||
links: Optional[list[dict]],
|
||||
category: Optional[str],
|
||||
) -> Optional[str]:
|
||||
resolved_origin = origin
|
||||
if resolved_origin is None:
|
||||
resolved_origin = "devii" if request_fields.get("devii") else "web"
|
||||
resolved_via = via_agent
|
||||
if resolved_via is None:
|
||||
resolved_via = 1 if request_fields.get("devii") else 0
|
||||
row = {
|
||||
"event_key": event_key,
|
||||
"category": category or category_for(event_key),
|
||||
"actor_kind": base_actor.get("actor_kind"),
|
||||
"actor_uid": base_actor.get("actor_uid"),
|
||||
"actor_username": base_actor.get("actor_username"),
|
||||
"actor_role": base_actor.get("actor_role"),
|
||||
"origin": resolved_origin,
|
||||
"via_agent": int(resolved_via),
|
||||
"request_method": request_fields.get("request_method"),
|
||||
"request_path": request_fields.get("request_path"),
|
||||
"actor_ip": request_fields.get("actor_ip"),
|
||||
"user_agent": request_fields.get("user_agent"),
|
||||
"target_type": target_type,
|
||||
"target_uid": target_uid,
|
||||
"target_label": _sanitize_summary(target_label, 200),
|
||||
"old_value": _coerce_scalar(old_value),
|
||||
"new_value": _coerce_scalar(new_value),
|
||||
"summary": _sanitize_summary(summary),
|
||||
"metadata": json.dumps(metadata) if metadata else None,
|
||||
"result": result,
|
||||
}
|
||||
all_links = list(links or [])
|
||||
if base_actor.get("actor_uid") and not any(
|
||||
item.get("relation") == "actor" for item in all_links
|
||||
):
|
||||
all_links.append(
|
||||
actor(base_actor["actor_uid"], base_actor.get("actor_username"))
|
||||
)
|
||||
audit_uid = store.insert_event(row)
|
||||
store.insert_links(audit_uid, all_links)
|
||||
logger.debug(
|
||||
"audit %s by %s/%s result=%s origin=%s",
|
||||
event_key,
|
||||
row["actor_kind"],
|
||||
row["actor_username"],
|
||||
result,
|
||||
resolved_origin,
|
||||
)
|
||||
return audit_uid
|
||||
|
||||
|
||||
def record(
|
||||
request,
|
||||
event_key: str,
|
||||
*,
|
||||
user=_UNSET,
|
||||
actor_kind: Optional[str] = None,
|
||||
target_type: Optional[str] = None,
|
||||
target_uid: Optional[str] = None,
|
||||
target_label: Optional[str] = None,
|
||||
old_value: Any = None,
|
||||
new_value: Any = None,
|
||||
summary: Optional[str] = None,
|
||||
metadata: Optional[dict] = None,
|
||||
result: str = "success",
|
||||
origin: Optional[str] = None,
|
||||
via_agent: Optional[int] = None,
|
||||
links: Optional[list[dict]] = None,
|
||||
category: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
try:
|
||||
if user is _UNSET:
|
||||
from devplacepy.utils import get_current_user
|
||||
|
||||
try:
|
||||
user = get_current_user(request)
|
||||
except Exception:
|
||||
user = None
|
||||
base_actor = _actor_from(user)
|
||||
if actor_kind:
|
||||
base_actor["actor_kind"] = actor_kind
|
||||
return _write(
|
||||
event_key=event_key,
|
||||
base_actor=base_actor,
|
||||
request_fields=_request_fields(request),
|
||||
target_type=target_type,
|
||||
target_uid=target_uid,
|
||||
target_label=target_label,
|
||||
old_value=old_value,
|
||||
new_value=new_value,
|
||||
summary=summary,
|
||||
metadata=metadata,
|
||||
result=result,
|
||||
origin=origin,
|
||||
via_agent=via_agent,
|
||||
links=links,
|
||||
category=category,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("audit.record failed for %s: %s", event_key, exc)
|
||||
return None
|
||||
|
||||
|
||||
def record_system(
|
||||
event_key: str,
|
||||
*,
|
||||
actor_kind: str = "system",
|
||||
actor_uid: Optional[str] = None,
|
||||
actor_username: Optional[str] = None,
|
||||
actor_role: str = "system",
|
||||
target_type: Optional[str] = None,
|
||||
target_uid: Optional[str] = None,
|
||||
target_label: Optional[str] = None,
|
||||
old_value: Any = None,
|
||||
new_value: Any = None,
|
||||
summary: Optional[str] = None,
|
||||
metadata: Optional[dict] = None,
|
||||
result: str = "success",
|
||||
origin: str = "service",
|
||||
via_agent: int = 0,
|
||||
links: Optional[list[dict]] = None,
|
||||
category: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
try:
|
||||
base_actor = {
|
||||
"actor_kind": actor_kind,
|
||||
"actor_uid": actor_uid,
|
||||
"actor_username": actor_username,
|
||||
"actor_role": actor_role,
|
||||
}
|
||||
return _write(
|
||||
event_key=event_key,
|
||||
base_actor=base_actor,
|
||||
request_fields={},
|
||||
target_type=target_type,
|
||||
target_uid=target_uid,
|
||||
target_label=target_label,
|
||||
old_value=old_value,
|
||||
new_value=new_value,
|
||||
summary=summary,
|
||||
metadata=metadata,
|
||||
result=result,
|
||||
origin=origin,
|
||||
via_agent=via_agent,
|
||||
links=links,
|
||||
category=category,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("audit.record_system failed for %s: %s", event_key, exc)
|
||||
return None
|
||||
@@ -0,0 +1,53 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from devplacepy.database import get_int_setting
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
from devplacepy.services.audit import store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RETENTION_KEY = "audit_log_retention_days"
|
||||
DEFAULT_RETENTION_DAYS = 90
|
||||
|
||||
|
||||
class AuditService(BaseService):
|
||||
title = "Audit retention"
|
||||
description = "Prunes audit_log rows and their links older than the retention window."
|
||||
default_enabled = True
|
||||
min_interval = 3600
|
||||
config_fields = [
|
||||
ConfigField(
|
||||
RETENTION_KEY,
|
||||
"Retention (days)",
|
||||
type="int",
|
||||
default=DEFAULT_RETENTION_DAYS,
|
||||
minimum=0,
|
||||
help="Audit rows older than this are pruned. 0 disables pruning.",
|
||||
group="General",
|
||||
)
|
||||
]
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__("audit", interval_seconds=86400)
|
||||
|
||||
async def run_once(self) -> None:
|
||||
days = get_int_setting(RETENTION_KEY, DEFAULT_RETENTION_DAYS)
|
||||
if days <= 0:
|
||||
self.log("Retention disabled (0 days); nothing pruned")
|
||||
return
|
||||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
||||
removed_links, removed_events = store.sweep(cutoff)
|
||||
self.log(
|
||||
f"Pruned {removed_events} audit rows and {removed_links} links older than {days}d"
|
||||
)
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
from devplacepy.database import db
|
||||
|
||||
if store.AUDIT_TABLE not in db.tables:
|
||||
return {"total_events": 0}
|
||||
rows = list(db.query(f"SELECT COUNT(*) AS c FROM {store.AUDIT_TABLE}"))
|
||||
return {"total_events": int(rows[0]["c"]) if rows else 0}
|
||||
@@ -0,0 +1,186 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.database import db, get_table
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AUDIT_TABLE = "audit_log"
|
||||
LINKS_TABLE = "audit_log_links"
|
||||
|
||||
EVENT_COLUMNS = (
|
||||
"uid",
|
||||
"created_at",
|
||||
"event_key",
|
||||
"category",
|
||||
"actor_kind",
|
||||
"actor_uid",
|
||||
"actor_username",
|
||||
"actor_role",
|
||||
"origin",
|
||||
"via_agent",
|
||||
"request_method",
|
||||
"request_path",
|
||||
"actor_ip",
|
||||
"user_agent",
|
||||
"target_type",
|
||||
"target_uid",
|
||||
"target_label",
|
||||
"old_value",
|
||||
"new_value",
|
||||
"summary",
|
||||
"metadata",
|
||||
"result",
|
||||
)
|
||||
|
||||
LINK_COLUMNS = (
|
||||
"uid",
|
||||
"audit_uid",
|
||||
"relation",
|
||||
"object_type",
|
||||
"object_uid",
|
||||
"object_label",
|
||||
"created_at",
|
||||
)
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _empty_event() -> dict:
|
||||
row: dict = {column: None for column in EVENT_COLUMNS}
|
||||
row["via_agent"] = 0
|
||||
return row
|
||||
|
||||
|
||||
def _empty_link() -> dict:
|
||||
return {column: None for column in LINK_COLUMNS}
|
||||
|
||||
|
||||
def ensure_tables() -> None:
|
||||
try:
|
||||
if AUDIT_TABLE not in db.tables:
|
||||
sentinel = _empty_event()
|
||||
sentinel["uid"] = "__audit_sentinel__"
|
||||
sentinel["created_at"] = now()
|
||||
sentinel["event_key"] = "__sentinel__"
|
||||
sentinel["category"] = "system"
|
||||
sentinel["actor_kind"] = "system"
|
||||
sentinel["actor_role"] = "system"
|
||||
sentinel["origin"] = "system"
|
||||
sentinel["result"] = "success"
|
||||
table = get_table(AUDIT_TABLE)
|
||||
table.insert(sentinel)
|
||||
table.delete(uid="__audit_sentinel__")
|
||||
if LINKS_TABLE not in db.tables:
|
||||
sentinel = _empty_link()
|
||||
sentinel["uid"] = "__audit_link_sentinel__"
|
||||
sentinel["audit_uid"] = "__audit_sentinel__"
|
||||
sentinel["relation"] = "actor"
|
||||
sentinel["object_type"] = "user"
|
||||
sentinel["object_uid"] = "__sentinel__"
|
||||
sentinel["created_at"] = now()
|
||||
table = get_table(LINKS_TABLE)
|
||||
table.insert(sentinel)
|
||||
table.delete(uid="__audit_link_sentinel__")
|
||||
except Exception as exc:
|
||||
logger.warning("audit ensure_tables failed: %s", exc)
|
||||
|
||||
|
||||
def insert_event(row: dict) -> str:
|
||||
record = _empty_event()
|
||||
record.update({key: value for key, value in row.items() if key in EVENT_COLUMNS})
|
||||
if not record.get("uid"):
|
||||
record["uid"] = generate_uid()
|
||||
if not record.get("created_at"):
|
||||
record["created_at"] = now()
|
||||
if record.get("via_agent") is None:
|
||||
record["via_agent"] = 0
|
||||
get_table(AUDIT_TABLE).insert(record)
|
||||
return record["uid"]
|
||||
|
||||
|
||||
def insert_links(audit_uid: str, links: list[dict]) -> int:
|
||||
if not links:
|
||||
return 0
|
||||
table = get_table(LINKS_TABLE)
|
||||
stamp = now()
|
||||
count = 0
|
||||
for link in links:
|
||||
record = _empty_link()
|
||||
record.update(
|
||||
{key: value for key, value in link.items() if key in LINK_COLUMNS}
|
||||
)
|
||||
record["uid"] = generate_uid()
|
||||
record["audit_uid"] = audit_uid
|
||||
record["created_at"] = stamp
|
||||
if not record.get("object_uid"):
|
||||
continue
|
||||
table.insert(record)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def get_event(uid: str) -> Optional[dict]:
|
||||
if AUDIT_TABLE not in db.tables:
|
||||
return None
|
||||
return get_table(AUDIT_TABLE).find_one(uid=uid)
|
||||
|
||||
|
||||
def get_links(audit_uid: str) -> list[dict]:
|
||||
if LINKS_TABLE not in db.tables:
|
||||
return []
|
||||
return list(
|
||||
get_table(LINKS_TABLE).find(audit_uid=audit_uid, order_by=["created_at"])
|
||||
)
|
||||
|
||||
|
||||
def sweep(cutoff: str, batch_size: int = 5000, max_batches: int = 100) -> tuple[int, int]:
|
||||
if AUDIT_TABLE not in db.tables:
|
||||
return (0, 0)
|
||||
removed_links = 0
|
||||
removed_events = 0
|
||||
for _ in range(max_batches):
|
||||
rows = list(
|
||||
db.query(
|
||||
f"SELECT uid FROM {AUDIT_TABLE} WHERE created_at < :cutoff "
|
||||
f"ORDER BY created_at LIMIT :limit",
|
||||
cutoff=cutoff,
|
||||
limit=batch_size,
|
||||
)
|
||||
)
|
||||
uids = [row["uid"] for row in rows]
|
||||
if not uids:
|
||||
break
|
||||
placeholders, params = _in_clause(uids)
|
||||
if LINKS_TABLE in db.tables:
|
||||
link_rows = list(
|
||||
db.query(
|
||||
f"SELECT COUNT(*) AS c FROM {LINKS_TABLE} "
|
||||
f"WHERE audit_uid IN ({placeholders})",
|
||||
**params,
|
||||
)
|
||||
)
|
||||
removed_links += int(link_rows[0]["c"]) if link_rows else 0
|
||||
db.query(
|
||||
f"DELETE FROM {LINKS_TABLE} WHERE audit_uid IN ({placeholders})",
|
||||
**params,
|
||||
)
|
||||
db.query(
|
||||
f"DELETE FROM {AUDIT_TABLE} WHERE uid IN ({placeholders})", **params
|
||||
)
|
||||
removed_events += len(uids)
|
||||
if len(uids) < batch_size:
|
||||
break
|
||||
return (removed_links, removed_events)
|
||||
|
||||
|
||||
def _in_clause(uids: list[str], prefix: str = "u") -> tuple[str, dict]:
|
||||
placeholders = ", ".join(f":{prefix}{index}" for index in range(len(uids)))
|
||||
params = {f"{prefix}{index}": uid for index, uid in enumerate(uids)}
|
||||
return placeholders, params
|
||||
Reference in New Issue
Block a user