forked from retoor/devplacepy
feat: add container manager API, islop router, and container runtime files with vim/bot/d stealth clients
This commit is contained in:
@@ -0,0 +1,637 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy import config, project_files, stealth
|
||||
from devplacepy.services.containers import store
|
||||
from devplacepy.services.containers.backend.base import (
|
||||
WORKSPACE_MOUNT,
|
||||
Mount,
|
||||
PortMapping,
|
||||
RunSpec,
|
||||
)
|
||||
from devplacepy.services.containers.runtime import get_backend
|
||||
from devplacepy.services.devii.tasks.schedule import (
|
||||
Schedule,
|
||||
now_utc,
|
||||
to_iso,
|
||||
)
|
||||
|
||||
IMAGE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}$")
|
||||
INGRESS_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
MEM_RE = re.compile(r"^\d+(\.\d+)?[bkmgBKMG]?$")
|
||||
CPU_RE = re.compile(r"^\d+(\.\d+)?$")
|
||||
ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
INSTANCE_LABEL = "devplace.instance"
|
||||
PROJECT_LABEL = "devplace.project"
|
||||
HOST_PORT_MIN = 20001
|
||||
HOST_PORT_MAX = 65535
|
||||
BOOT_LANGUAGES = ("none", "python", "bash")
|
||||
BOOT_SCRIPT_FILES = {"python": ".devplace_boot.py", "bash": ".devplace_boot.sh"}
|
||||
BOOT_SCRIPT_RUNNERS = {"python": "python", "bash": "bash"}
|
||||
MAX_BOOT_SCRIPT_CHARS = 100_000
|
||||
|
||||
|
||||
class ContainerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _validate_limits(cpu_limit: str, mem_limit: str) -> None:
|
||||
if cpu_limit and not CPU_RE.match(str(cpu_limit)):
|
||||
raise ContainerError("cpu limit must be a number, e.g. 1 or 1.5")
|
||||
if mem_limit and not MEM_RE.match(str(mem_limit)):
|
||||
raise ContainerError("memory limit must look like 512m, 1g, or a byte count")
|
||||
|
||||
|
||||
def validate_run_as(run_as_uid) -> str:
|
||||
uid = str(run_as_uid or "").strip()
|
||||
if not uid:
|
||||
return ""
|
||||
from devplacepy import database
|
||||
|
||||
user = database.get_users_by_uids([uid]).get(uid)
|
||||
if not user:
|
||||
raise ContainerError(f"run-as user not found: {uid}")
|
||||
return uid
|
||||
|
||||
|
||||
def validate_boot(boot_language, boot_script) -> tuple:
|
||||
language = str(boot_language or "none").strip().lower() or "none"
|
||||
if language not in BOOT_LANGUAGES:
|
||||
raise ContainerError(
|
||||
f"boot language must be one of {', '.join(BOOT_LANGUAGES)}"
|
||||
)
|
||||
script = str(boot_script or "")
|
||||
if language == "none":
|
||||
script = ""
|
||||
if len(script) > MAX_BOOT_SCRIPT_CHARS:
|
||||
raise ContainerError(
|
||||
f"boot script exceeds the {MAX_BOOT_SCRIPT_CHARS}-character limit"
|
||||
)
|
||||
if language != "none" and not script.strip():
|
||||
raise ContainerError("boot script is required when a boot language is set")
|
||||
return language, script
|
||||
|
||||
|
||||
def parse_ports(value) -> list:
|
||||
ports = []
|
||||
if not value:
|
||||
return ports
|
||||
items = (
|
||||
value if isinstance(value, list) else str(value).replace(",", "\n").splitlines()
|
||||
)
|
||||
for item in items:
|
||||
item = str(item).strip()
|
||||
if not item:
|
||||
continue
|
||||
proto = "tcp"
|
||||
if "/" in item:
|
||||
item, proto = item.split("/", 1)
|
||||
if ":" in item:
|
||||
host, container = item.split(":", 1)
|
||||
else:
|
||||
host, container = "0", item
|
||||
if not host.isdigit() or not container.isdigit():
|
||||
raise ContainerError(
|
||||
f"port '{item}' must be numeric host:container or a bare container port"
|
||||
)
|
||||
ports.append(PortMapping(int(host), int(container), proto.strip() or "tcp"))
|
||||
return ports
|
||||
|
||||
|
||||
def used_host_ports() -> set:
|
||||
ports = set()
|
||||
for instance in store.all_instances():
|
||||
for mapping in json.loads(instance.get("ports_json") or "[]"):
|
||||
host = int(mapping.get("host") or 0)
|
||||
if host:
|
||||
ports.add(host)
|
||||
return ports
|
||||
|
||||
|
||||
def _host_port_free(port: int) -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind(("0.0.0.0", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def allocate_host_port(reserved: set) -> int:
|
||||
for port in range(HOST_PORT_MIN, HOST_PORT_MAX + 1):
|
||||
if port in reserved:
|
||||
continue
|
||||
if _host_port_free(port):
|
||||
return port
|
||||
raise ContainerError(
|
||||
f"no free host port available in range {HOST_PORT_MIN}-{HOST_PORT_MAX}"
|
||||
)
|
||||
|
||||
|
||||
def assign_host_ports(port_list: list) -> list:
|
||||
reserved = used_host_ports()
|
||||
assigned = []
|
||||
for mapping in port_list:
|
||||
host = mapping.host
|
||||
if not host:
|
||||
host = allocate_host_port(reserved)
|
||||
elif host in reserved:
|
||||
raise ContainerError(
|
||||
f"host port {host} is already published by another instance"
|
||||
)
|
||||
reserved.add(host)
|
||||
assigned.append(PortMapping(host, mapping.container, mapping.proto))
|
||||
return assigned
|
||||
|
||||
|
||||
def parse_env(value) -> dict:
|
||||
env = {}
|
||||
if not value:
|
||||
return env
|
||||
if isinstance(value, dict):
|
||||
items = value.items()
|
||||
else:
|
||||
items = (line.split("=", 1) for line in str(value).splitlines() if "=" in line)
|
||||
for key, val in items:
|
||||
key = str(key).strip()
|
||||
if not ENV_KEY_RE.match(key):
|
||||
raise ContainerError(f"invalid environment variable name: {key}")
|
||||
env[key] = str(val)
|
||||
return env
|
||||
|
||||
|
||||
# ---------------- instances ----------------
|
||||
|
||||
|
||||
def validate_ingress(slug: str, port, port_list) -> tuple:
|
||||
slug = (slug or "").strip().lower()
|
||||
if not slug:
|
||||
return "", 0
|
||||
if not INGRESS_SLUG_RE.match(slug):
|
||||
raise ContainerError(
|
||||
"ingress slug must be lowercase letters, digits, or '-' (max 63 chars)"
|
||||
)
|
||||
for other in store.all_instances():
|
||||
if other.get("ingress_slug") == slug:
|
||||
raise ContainerError(f"ingress slug '{slug}' is already in use")
|
||||
ingress_port = int(port) if port else 0
|
||||
container_ports = {p.container for p in port_list}
|
||||
if ingress_port and ingress_port not in container_ports:
|
||||
raise ContainerError(
|
||||
f"ingress_port {ingress_port} must be one of the container ports you mapped"
|
||||
)
|
||||
if not ingress_port and len(container_ports) != 1:
|
||||
raise ContainerError(
|
||||
"set ingress_port to choose which mapped container port to publish"
|
||||
)
|
||||
return slug, ingress_port
|
||||
|
||||
|
||||
async def create_instance(
|
||||
project: dict,
|
||||
*,
|
||||
name: str,
|
||||
boot_command: str = "",
|
||||
boot_language: str = "none",
|
||||
boot_script: str = "",
|
||||
run_as_uid: str = "",
|
||||
start_on_boot: bool = False,
|
||||
env="",
|
||||
cpu_limit: str = "",
|
||||
mem_limit: str = "",
|
||||
ports="",
|
||||
volumes="",
|
||||
restart_policy: str = "never",
|
||||
autostart: bool = True,
|
||||
ingress_slug: str = "",
|
||||
ingress_port=None,
|
||||
actor=("system", "system"),
|
||||
) -> dict:
|
||||
if not await get_backend().image_exists(config.CONTAINER_IMAGE):
|
||||
raise ContainerError(
|
||||
f"the '{config.CONTAINER_IMAGE}' image is not built - run 'make ppy'"
|
||||
)
|
||||
if restart_policy not in store.RESTART_POLICIES:
|
||||
raise ContainerError(
|
||||
f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
|
||||
)
|
||||
_validate_limits(cpu_limit, mem_limit)
|
||||
run_as_uid = validate_run_as(run_as_uid)
|
||||
boot_language, boot_script = validate_boot(boot_language, boot_script)
|
||||
port_list = assign_host_ports(parse_ports(ports))
|
||||
env_map = parse_env(env)
|
||||
name = (name or "").strip()
|
||||
if not name:
|
||||
raise ContainerError("instance name is required")
|
||||
ingress_slug, ingress_port = validate_ingress(ingress_slug, ingress_port, port_list)
|
||||
|
||||
workspace = Path(config.CONTAINER_WORKSPACES_DIR) / project["uid"]
|
||||
await asyncio.to_thread(
|
||||
project_files.export_to_dir, project["uid"], "", str(workspace)
|
||||
)
|
||||
|
||||
row = {
|
||||
"project_uid": project["uid"],
|
||||
"created_by": actor[1] if actor and actor[0] == "user" else "",
|
||||
"owner_uid": project.get("user_uid", ""),
|
||||
"run_as_uid": run_as_uid,
|
||||
"name": name,
|
||||
"boot_command": boot_command or "",
|
||||
"boot_language": boot_language,
|
||||
"boot_script": boot_script,
|
||||
"start_on_boot": 1 if start_on_boot else 0,
|
||||
"env_json": json.dumps(env_map),
|
||||
"cpu_limit": str(cpu_limit or ""),
|
||||
"mem_limit": str(mem_limit or ""),
|
||||
"ports_json": json.dumps(
|
||||
[
|
||||
{"host": p.host, "container": p.container, "proto": p.proto}
|
||||
for p in port_list
|
||||
]
|
||||
),
|
||||
"volumes_json": volumes
|
||||
if isinstance(volumes, str)
|
||||
else json.dumps(volumes or []),
|
||||
"restart_policy": restart_policy,
|
||||
"ingress_slug": ingress_slug,
|
||||
"ingress_port": ingress_port,
|
||||
"desired_state": store.DESIRED_RUNNING if autostart else store.DESIRED_STOPPED,
|
||||
"status": store.ST_CREATED,
|
||||
"workspace_dir": str(workspace),
|
||||
}
|
||||
instance = store.create_instance(row)
|
||||
store.record_event(
|
||||
instance, "created", actor[0], actor[1], {"image": config.CONTAINER_IMAGE}
|
||||
)
|
||||
if actor and actor[0] == "user":
|
||||
from devplacepy.utils import track_action
|
||||
|
||||
track_action(actor[1], "container")
|
||||
return instance
|
||||
|
||||
|
||||
def set_desired_state(
|
||||
instance: dict, desired: str, *, actor=("system", "system")
|
||||
) -> dict:
|
||||
if desired not in (
|
||||
store.DESIRED_RUNNING,
|
||||
store.DESIRED_STOPPED,
|
||||
store.DESIRED_PAUSED,
|
||||
):
|
||||
raise ContainerError("desired state must be running, stopped, or paused")
|
||||
store.update_instance(instance["uid"], {"desired_state": desired})
|
||||
store.record_event(instance, f"desire_{desired}", actor[0], actor[1])
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def request_restart(instance: dict, *, actor=("system", "system")) -> dict:
|
||||
store.update_instance(
|
||||
instance["uid"],
|
||||
{"desired_state": store.DESIRED_RUNNING, "status": store.ST_RESTARTING},
|
||||
)
|
||||
store.record_event(instance, "restart", actor[0], actor[1])
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None:
|
||||
store.update_instance(
|
||||
instance["uid"],
|
||||
{"desired_state": store.DESIRED_STOPPED, "status": store.ST_REMOVING},
|
||||
)
|
||||
store.record_event(instance, "remove", actor[0], actor[1])
|
||||
|
||||
|
||||
def update_instance_config(
|
||||
instance: dict,
|
||||
*,
|
||||
run_as_uid=None,
|
||||
boot_language=None,
|
||||
boot_script=None,
|
||||
boot_command=None,
|
||||
restart_policy=None,
|
||||
start_on_boot=None,
|
||||
cpu_limit=None,
|
||||
mem_limit=None,
|
||||
actor=("system", "system"),
|
||||
) -> dict:
|
||||
changes: dict = {}
|
||||
if run_as_uid is not None:
|
||||
changes["run_as_uid"] = validate_run_as(run_as_uid)
|
||||
if boot_language is not None or boot_script is not None:
|
||||
language = (
|
||||
boot_language
|
||||
if boot_language is not None
|
||||
else instance.get("boot_language", "none")
|
||||
)
|
||||
script = (
|
||||
boot_script if boot_script is not None else instance.get("boot_script", "")
|
||||
)
|
||||
language, script = validate_boot(language, script)
|
||||
changes["boot_language"] = language
|
||||
changes["boot_script"] = script
|
||||
if boot_command is not None:
|
||||
changes["boot_command"] = str(boot_command or "")[:500]
|
||||
if restart_policy is not None:
|
||||
if restart_policy not in store.RESTART_POLICIES:
|
||||
raise ContainerError(
|
||||
f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
|
||||
)
|
||||
changes["restart_policy"] = restart_policy
|
||||
if start_on_boot is not None:
|
||||
changes["start_on_boot"] = 1 if start_on_boot else 0
|
||||
if cpu_limit is not None or mem_limit is not None:
|
||||
cpu = cpu_limit if cpu_limit is not None else instance.get("cpu_limit", "")
|
||||
mem = mem_limit if mem_limit is not None else instance.get("mem_limit", "")
|
||||
_validate_limits(cpu, mem)
|
||||
changes["cpu_limit"] = str(cpu or "")
|
||||
changes["mem_limit"] = str(mem or "")
|
||||
if not changes:
|
||||
return store.get_instance(instance["uid"])
|
||||
store.update_instance(instance["uid"], changes)
|
||||
store.record_event(
|
||||
instance, "configure", actor[0], actor[1], {"fields": sorted(changes)}
|
||||
)
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def set_start_on_boot(
|
||||
instance: dict, enabled: bool, *, actor=("system", "system")
|
||||
) -> dict:
|
||||
store.update_instance(instance["uid"], {"start_on_boot": 1 if enabled else 0})
|
||||
store.record_event(
|
||||
instance, "start_on_boot", actor[0], actor[1], {"enabled": bool(enabled)}
|
||||
)
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def materialize_boot_script(instance: dict) -> None:
|
||||
language = (instance.get("boot_language") or "none").strip().lower()
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
return
|
||||
for filename in BOOT_SCRIPT_FILES.values():
|
||||
stale = Path(workspace) / filename
|
||||
if stale.is_file():
|
||||
try:
|
||||
stale.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
if language not in BOOT_SCRIPT_FILES:
|
||||
return
|
||||
script = instance.get("boot_script") or ""
|
||||
if not script.strip():
|
||||
return
|
||||
target = Path(workspace) / BOOT_SCRIPT_FILES[language]
|
||||
try:
|
||||
Path(workspace).mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(script, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def pravda_env(instance: dict) -> dict:
|
||||
from devplacepy import database, seo
|
||||
|
||||
base_url = seo.public_base_url()
|
||||
api_key = ""
|
||||
user_uid = instance.get("owner_uid") or ""
|
||||
for uid in (
|
||||
instance.get("run_as_uid"),
|
||||
instance.get("created_by"),
|
||||
instance.get("owner_uid"),
|
||||
):
|
||||
if not uid:
|
||||
continue
|
||||
user = database.get_users_by_uids([uid]).get(uid)
|
||||
if user and user.get("api_key"):
|
||||
api_key = user["api_key"]
|
||||
break
|
||||
slug = instance.get("ingress_slug") or ""
|
||||
ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else ""
|
||||
return {
|
||||
"PRAVDA_BASE_URL": base_url,
|
||||
"PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
|
||||
"PRAVDA_API_KEY": api_key,
|
||||
"PRAVDA_USER_UID": instance.get("run_as_uid") or user_uid,
|
||||
"PRAVDA_CONTAINER_NAME": instance.get("name") or "",
|
||||
"PRAVDA_CONTAINER_UID": instance.get("uid") or "",
|
||||
"PRAVDA_INGRESS_URL": ingress_url,
|
||||
}
|
||||
|
||||
|
||||
def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
|
||||
env = {**json.loads(instance.get("env_json") or "{}"), **pravda_env(instance)}
|
||||
ports = [
|
||||
PortMapping(p["host"], p["container"], p.get("proto", "tcp"))
|
||||
for p in json.loads(instance.get("ports_json") or "[]")
|
||||
]
|
||||
mounts = [Mount(instance["workspace_dir"], WORKSPACE_MOUNT, "rw")]
|
||||
for extra in json.loads(instance.get("volumes_json") or "[]"):
|
||||
if isinstance(extra, dict) and extra.get("host") and extra.get("container"):
|
||||
mounts.append(
|
||||
Mount(extra["host"], extra["container"], extra.get("mode", "rw"))
|
||||
)
|
||||
language = (instance.get("boot_language") or "none").strip().lower()
|
||||
boot = (instance.get("boot_command") or "").strip()
|
||||
if language in BOOT_SCRIPT_FILES and (instance.get("boot_script") or "").strip():
|
||||
script_path = f"{WORKSPACE_MOUNT}/{BOOT_SCRIPT_FILES[language]}"
|
||||
command = [BOOT_SCRIPT_RUNNERS[language], script_path]
|
||||
elif boot:
|
||||
command = ["/bin/sh", "-c", boot]
|
||||
else:
|
||||
command = ["sleep", "infinity"]
|
||||
return RunSpec(
|
||||
image=image_tag,
|
||||
name=instance["slug"],
|
||||
labels={
|
||||
INSTANCE_LABEL: instance["uid"],
|
||||
PROJECT_LABEL: instance["project_uid"],
|
||||
},
|
||||
env=env,
|
||||
cpu_limit=instance.get("cpu_limit", ""),
|
||||
mem_limit=instance.get("mem_limit", ""),
|
||||
ports=ports,
|
||||
mounts=mounts,
|
||||
restart_policy=instance.get("restart_policy", "never"),
|
||||
command=command,
|
||||
)
|
||||
|
||||
|
||||
async def sync_workspace(instance: dict, user: dict) -> dict:
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
raise ContainerError("instance has no workspace")
|
||||
counts = await asyncio.to_thread(
|
||||
project_files.sync_dir_bidirectional, instance["project_uid"], workspace, user
|
||||
)
|
||||
store.record_event(
|
||||
instance,
|
||||
"sync",
|
||||
"user",
|
||||
user["uid"],
|
||||
{"exported": counts["exported"], "imported": counts["imported"]},
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def sync_bidirectional_sync(instance: dict, user: dict) -> dict:
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
return {"exported": 0, "imported": 0}
|
||||
counts = project_files.sync_dir_bidirectional(
|
||||
instance["project_uid"], workspace, user
|
||||
)
|
||||
if counts["exported"] or counts["imported"]:
|
||||
store.record_event(
|
||||
instance,
|
||||
"sync",
|
||||
"service",
|
||||
"system",
|
||||
{"exported": counts["exported"], "imported": counts["imported"]},
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def add_schedule(instance: dict, action: str, schedule: Schedule) -> dict:
|
||||
if action not in ("start", "stop"):
|
||||
raise ContainerError("schedule action must be start or stop")
|
||||
first = schedule.first_run(now_utc())
|
||||
return store.create_schedule(instance, action, schedule.columns(), to_iso(first))
|
||||
|
||||
|
||||
# ---------------- aggregation ----------------
|
||||
|
||||
|
||||
def _percentile(values: list, pct: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1))))
|
||||
return float(ordered[index])
|
||||
|
||||
|
||||
def instance_stats(instance_uid: str) -> dict:
|
||||
metrics = store.recent_metrics(instance_uid, limit=720)
|
||||
cpu = [m.get("cpu_pct", 0) for m in metrics]
|
||||
mem = [m.get("mem_bytes", 0) for m in metrics]
|
||||
return {
|
||||
"samples": len(metrics),
|
||||
"cpu_avg": round(sum(cpu) / len(cpu), 2) if cpu else 0.0,
|
||||
"cpu_p95": round(_percentile(cpu, 95), 2),
|
||||
"mem_max": max(mem) if mem else 0,
|
||||
"mem_avg": int(sum(mem) / len(mem)) if mem else 0,
|
||||
}
|
||||
|
||||
|
||||
def _port_reachable(host: str, port: int, timeout: float = 0.3) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _http_probe(host: str, port: int, timeout: float = 1.0) -> str:
|
||||
try:
|
||||
with stealth.stealth_sync_client(timeout=timeout) as client:
|
||||
response = client.get(f"http://{host}:{port}/")
|
||||
return f"HTTP {response.status_code}"
|
||||
except Exception as exc: # noqa: BLE001 - diagnostic, any failure is informative
|
||||
return f"unreachable: {type(exc).__name__}"
|
||||
|
||||
|
||||
def _net_entry(data: dict) -> dict:
|
||||
net = (data or {}).get("NetworkSettings") or {}
|
||||
if net.get("IPAddress") or net.get("Gateway"):
|
||||
return net
|
||||
for entry in (net.get("Networks") or {}).values():
|
||||
if entry and (entry.get("IPAddress") or entry.get("Gateway")):
|
||||
return entry
|
||||
return {}
|
||||
|
||||
|
||||
def container_ip_from_inspect(data: dict) -> str:
|
||||
return (_net_entry(data).get("IPAddress") or "").strip()
|
||||
|
||||
|
||||
def container_gateway_from_inspect(data: dict) -> str:
|
||||
return (_net_entry(data).get("Gateway") or "").strip()
|
||||
|
||||
|
||||
def _ingress_container_port(instance: dict, port_maps: list) -> int:
|
||||
ingress_port = int(instance.get("ingress_port") or 0)
|
||||
if ingress_port:
|
||||
for mapping in port_maps:
|
||||
if int(mapping.get("container") or 0) == ingress_port:
|
||||
return ingress_port
|
||||
return 0
|
||||
return int(port_maps[0].get("container") or 0) if port_maps else 0
|
||||
|
||||
|
||||
def _host_port_for(port_maps: list, container_port: int) -> int:
|
||||
for mapping in port_maps:
|
||||
if int(mapping.get("container") or 0) == container_port:
|
||||
return int(mapping.get("host") or 0)
|
||||
return 0
|
||||
|
||||
|
||||
def proxy_target(instance: dict) -> tuple:
|
||||
port_maps = json.loads(instance.get("ports_json") or "[]")
|
||||
container_port = _ingress_container_port(instance, port_maps)
|
||||
if not container_port:
|
||||
return None, None
|
||||
host_port = _host_port_for(port_maps, container_port)
|
||||
if config.CONTAINER_PROXY_HOST:
|
||||
return (config.CONTAINER_PROXY_HOST, host_port) if host_port else (None, None)
|
||||
if not host_port:
|
||||
return None, None
|
||||
gateway = (instance.get("container_gateway") or "").strip()
|
||||
return (gateway or "127.0.0.1", host_port)
|
||||
|
||||
|
||||
def instance_runtime(instance: dict) -> dict:
|
||||
boot = (instance.get("boot_command") or "").strip()
|
||||
port_maps = json.loads(instance.get("ports_json") or "[]")
|
||||
container_ip = (instance.get("container_ip") or "").strip()
|
||||
container_gateway = (instance.get("container_gateway") or "").strip()
|
||||
target_host, target_port = proxy_target(instance)
|
||||
probe_host = config.CONTAINER_PROXY_HOST or container_gateway or "127.0.0.1"
|
||||
ports = []
|
||||
for mapping in port_maps:
|
||||
host_port = int(mapping.get("host") or 0)
|
||||
ports.append(
|
||||
{
|
||||
"container": int(mapping.get("container") or 0),
|
||||
"host": host_port,
|
||||
"proto": mapping.get("proto", "tcp"),
|
||||
"reachable": _port_reachable(probe_host, host_port)
|
||||
if host_port
|
||||
else False,
|
||||
}
|
||||
)
|
||||
ingress_serving = (
|
||||
_http_probe(target_host, target_port)
|
||||
if target_host and target_port
|
||||
else "no ingress port mapped"
|
||||
)
|
||||
return {
|
||||
"command": boot or "image CMD (no boot_command set)",
|
||||
"ports": ports,
|
||||
"container_ip": container_ip,
|
||||
"container_gateway": container_gateway,
|
||||
"ingress_target": (
|
||||
f"{target_host}:{target_port}" if target_host and target_port else ""
|
||||
),
|
||||
"ingress_port": int(instance.get("ingress_port") or 0),
|
||||
"ingress_serving": ingress_serving,
|
||||
"status_ok_for_ingress": instance.get("status") == store.ST_RUNNING,
|
||||
"restart_count": int(instance.get("restart_count") or 0),
|
||||
"exit_code": instance.get("exit_code"),
|
||||
"container_id": (instance.get("container_id") or "")[:12],
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
" retoor <retoor@molodetz.nl>
|
||||
" Self-contained config for the ppy container. No external plugins or managers:
|
||||
" it works out of the box with the stock vim, and the AI edit feature uses only
|
||||
" curl and the PRAVDA_* gateway env that every instance is launched with.
|
||||
|
||||
set nocompatible
|
||||
filetype plugin indent on
|
||||
syntax on
|
||||
|
||||
set encoding=utf-8
|
||||
set fileencoding=utf-8
|
||||
set termencoding=utf-8
|
||||
set mouse=a
|
||||
set backspace=indent,eol,start
|
||||
set autoindent
|
||||
set smartindent
|
||||
set tabstop=4
|
||||
set shiftwidth=4
|
||||
set expandtab
|
||||
set number
|
||||
set showmatch
|
||||
set showtabline=2
|
||||
set laststatus=2
|
||||
set hidden
|
||||
set incsearch
|
||||
set hlsearch
|
||||
set wildmenu
|
||||
set ttimeoutlen=50
|
||||
|
||||
if has('clipboard')
|
||||
set clipboard=unnamedplus
|
||||
endif
|
||||
|
||||
if !isdirectory(expand('~/.vim/undo'))
|
||||
call mkdir(expand('~/.vim/undo'), 'p')
|
||||
endif
|
||||
set undofile
|
||||
set undodir=~/.vim/undo
|
||||
|
||||
set statusline=%f\ %h%m%r\ %=\ [%{&filetype}]\ [%l,%c]\ %p%%
|
||||
highlight StatusLine cterm=bold ctermfg=15 ctermbg=24
|
||||
highlight StatusLineNC cterm=none ctermfg=250 ctermbg=236
|
||||
highlight ErrorMsg cterm=bold ctermfg=red ctermbg=none
|
||||
|
||||
let mapleader = ","
|
||||
|
||||
inoremap <C-n> <ESC>:tabnext<CR>
|
||||
inoremap <C-p> <ESC>:tabprevious<CR>
|
||||
nnoremap <C-n> :tabnext<CR>
|
||||
nnoremap <C-p> :tabprevious<CR>
|
||||
nnoremap <Tab> :tabnext<CR>
|
||||
nnoremap <C-Tab> :tabprevious<CR>
|
||||
nnoremap <C-e> :tabnew<Space>
|
||||
inoremap <C-e> <ESC>:tabnew<Space>
|
||||
|
||||
if has('autocmd')
|
||||
autocmd BufReadPost * if line("'\"") > 0 && line("'\"") <= line("$") | exe "normal! g'\"" | endif
|
||||
endif
|
||||
|
||||
function! s:GetVisualSelection() abort
|
||||
let [l:line_start, l:col_start] = [line("'<"), col("'<")]
|
||||
let [l:line_end, l:col_end] = [line("'>"), col("'>")]
|
||||
let l:lines = getline(l:line_start, l:line_end)
|
||||
if empty(l:lines)
|
||||
return ''
|
||||
endif
|
||||
let l:lines[-1] = l:lines[-1][: l:col_end - (l:line_start == l:line_end ? 1 : 2)]
|
||||
let l:lines[0] = l:lines[0][l:col_start - 1 :]
|
||||
return join(l:lines, "\n")
|
||||
endfunction
|
||||
|
||||
function! s:GatewayUrl() abort
|
||||
let l:base = substitute($PRAVDA_OPENAI_URL, '/\+$', '', '')
|
||||
if empty(l:base)
|
||||
return 'https://openai.app.molodetz.nl/v1/chat/completions'
|
||||
elseif l:base =~# '/chat/completions$'
|
||||
return l:base
|
||||
endif
|
||||
return l:base . '/chat/completions'
|
||||
endfunction
|
||||
|
||||
function! AiEditSelection() abort
|
||||
let l:instruction = input('AI instruction: ')
|
||||
if empty(l:instruction)
|
||||
echo 'Cancelled.'
|
||||
return
|
||||
endif
|
||||
let l:orig = s:GetVisualSelection()
|
||||
if empty(l:orig)
|
||||
echo 'No selection.'
|
||||
return
|
||||
endif
|
||||
let l:prompt = l:instruction . "\n\nHere is the text:\n" . l:orig
|
||||
\ . "\n\nOutput only the transformed text. No explanations, markdown, or code blocks."
|
||||
let l:api_key = !empty($PRAVDA_API_KEY) ? $PRAVDA_API_KEY : $DEEPSEEK_API_KEY
|
||||
let l:json = '{"model":"deepseek-chat","messages":[{"role":"user","content":' . json_encode(l:prompt) . '}]}'
|
||||
let l:cmd = 'curl -sS -X POST ' . shellescape(s:GatewayUrl())
|
||||
\ . ' -H ' . shellescape('Authorization: Bearer ' . l:api_key)
|
||||
\ . ' -H ' . shellescape('Content-Type: application/json')
|
||||
\ . ' -d ' . shellescape(l:json)
|
||||
let l:reply = system(l:cmd)
|
||||
if v:shell_error
|
||||
echohl ErrorMsg | echom 'AI request failed' | echohl None
|
||||
return
|
||||
endif
|
||||
let l:text = matchstr(l:reply, '"content":\s*"\zs\(.\{-}\)\ze"\s*[,}]')
|
||||
if empty(l:text)
|
||||
echohl ErrorMsg | echom 'No content in AI response' | echohl None
|
||||
return
|
||||
endif
|
||||
let l:text = substitute(l:text, '\\n', "\n", 'g')
|
||||
let l:text = substitute(l:text, '\\"', '"', 'g')
|
||||
let l:text = substitute(l:text, '\\t', "\t", 'g')
|
||||
normal! gv
|
||||
normal! c
|
||||
call feedkeys(l:text, 'n')
|
||||
endfunction
|
||||
|
||||
xnoremap <silent> <Leader>a :<C-u>call AiEditSelection()<CR>
|
||||
File diff suppressed because it is too large
Load Diff
+2337
File diff suppressed because it is too large
Load Diff
+2669
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,200 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
|
||||
def arg(
|
||||
name: str, description: str, required: bool = False, kind: str = "string"
|
||||
) -> Param:
|
||||
return Param(
|
||||
name=name,
|
||||
location="body",
|
||||
description=description,
|
||||
required=required,
|
||||
type=kind,
|
||||
)
|
||||
|
||||
|
||||
SLUG = arg(
|
||||
"project_slug",
|
||||
"Project slug or uid that owns the container resources.",
|
||||
required=True,
|
||||
)
|
||||
|
||||
CONTAINER_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="container_list_instances",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="List a project's container instances and their status",
|
||||
params=(SLUG,),
|
||||
),
|
||||
Action(
|
||||
name="container_create_instance",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Create and start a container instance (runs the shared ppy image with the project files mounted at /app)",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("name", "Instance name.", required=True),
|
||||
arg(
|
||||
"boot_command", "Optional command to run on boot, e.g. 'python app.py'."
|
||||
),
|
||||
arg(
|
||||
"boot_language",
|
||||
"Optional boot source language: 'none', 'python', or 'bash'. When set with boot_script, the script is materialized into /app and run on launch (takes precedence over boot_command).",
|
||||
),
|
||||
arg(
|
||||
"boot_script",
|
||||
"Optional boot source code (the body of the python or bash script) run on launch when boot_language is python or bash.",
|
||||
),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"Optional DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user, which is always pravda (uid 1000).",
|
||||
),
|
||||
arg(
|
||||
"start_on_boot",
|
||||
"Force this instance to running whenever the container service starts ('true' or 'false', default false).",
|
||||
),
|
||||
arg("restart_policy", "never, always, on-failure, or unless-stopped."),
|
||||
arg("env", "Optional env vars as KEY=VALUE lines."),
|
||||
arg(
|
||||
"ports",
|
||||
"Port maps per line or comma separated. Use a bare container port (e.g. '8899') to auto-assign a unique host port above 20000, or 'host:container' to pin one.",
|
||||
),
|
||||
arg("cpu_limit", "Optional CPU limit, e.g. 1 or 1.5."),
|
||||
arg("mem_limit", "Optional memory limit, e.g. 512m or 1g."),
|
||||
arg("autostart", "Start immediately ('true' or 'false', default true)."),
|
||||
arg(
|
||||
"ingress_slug",
|
||||
"Optional public ingress slug; the service is then reachable at /p/<slug>.",
|
||||
),
|
||||
arg(
|
||||
"ingress_port",
|
||||
"Container port to publish at /p/<slug> (must be one of the mapped ports).",
|
||||
kind="integer",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_instance_action",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Control an instance: start, stop, restart, pause, resume, delete, or sync",
|
||||
description="sync imports the container /app workspace back into the project files.",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"action",
|
||||
"start, stop, restart, pause, resume, delete, or sync.",
|
||||
required=True,
|
||||
),
|
||||
arg(
|
||||
"confirm",
|
||||
"Required only for action=delete: set true ONLY after the user has explicitly "
|
||||
"confirmed destroying the instance. Leave unset otherwise; the delete is refused "
|
||||
"until you pass confirm=true.",
|
||||
kind="boolean",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_configure_instance",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Update an instance's run-as user, boot language/script/command, restart policy, start-on-boot flag, and resource limits",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID); pass empty to clear. Does NOT change the container OS user (always pravda, uid 1000).",
|
||||
),
|
||||
arg("boot_language", "Boot source language: 'none', 'python', or 'bash'."),
|
||||
arg("boot_script", "Boot source code body run on launch."),
|
||||
arg("boot_command", "Fallback boot command used when no boot_script is set."),
|
||||
arg("restart_policy", "never, always, on-failure, or unless-stopped."),
|
||||
arg(
|
||||
"start_on_boot",
|
||||
"Force running on container-service start ('true' or 'false').",
|
||||
),
|
||||
arg("cpu_limit", "CPU limit, e.g. 1 or 1.5."),
|
||||
arg("mem_limit", "Memory limit, e.g. 512m or 1g."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_logs",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="Read the recent logs of a running instance",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg("tail", "Number of log lines (default 200).", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_exec",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Run a one-shot command inside a running instance and return its output. The command runs in /app (the project workspace) by default, so never prefix it with 'cd /app'",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"command",
|
||||
"Command to run, e.g. 'git clone ... && ls'. Runs in /app already; do not prepend 'cd /app'.",
|
||||
required=True,
|
||||
),
|
||||
arg(
|
||||
"confirm",
|
||||
"Required only when the command is destructive (rm, dd, truncate, drop, etc.): set "
|
||||
"true ONLY after the user has explicitly confirmed. Leave unset otherwise; a "
|
||||
"destructive command is refused until you pass confirm=true.",
|
||||
kind="boolean",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_stats",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="Get aggregated resource and runtime statistics for an instance",
|
||||
params=(SLUG, arg("instance", "Instance name, slug, or uid.", required=True)),
|
||||
),
|
||||
Action(
|
||||
name="container_schedule",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Schedule a start or stop of an instance (cron, interval, or one-time)",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg("action", "start or stop.", required=True),
|
||||
arg("kind", "once, interval, or cron.", required=True),
|
||||
arg("cron", "Cron expression for kind=cron, e.g. '0 2 * * *'."),
|
||||
arg("run_at", "ISO time for kind=once, e.g. 2026-06-15T02:00:00."),
|
||||
arg("every_seconds", "Interval seconds for kind=interval.", kind="integer"),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,313 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from types import TracebackType
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.acquisition.domcapture import (
|
||||
DOM_EXTRACT_SCRIPT,
|
||||
DOM_INIT_SCRIPT,
|
||||
DomSnapshot,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
CONSOLE_SAMPLE_CAP,
|
||||
DOM_CAPTURE_TIMEOUT_MS,
|
||||
HEADER_VALUE_CAP,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSER_USER_AGENT: str = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
BROWSER_VIEWPORT: dict[str, int] = {"width": 1440, "height": 900}
|
||||
BROWSER_LOCALE: str = "en-US"
|
||||
BROWSER_TIMEZONE: str = "America/New_York"
|
||||
|
||||
LAUNCH_ARGS: list[str] = [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-gpu",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-extensions",
|
||||
]
|
||||
|
||||
NAVIGATION_TIMEOUT_MS: int = 30000
|
||||
RENDER_SETTLE_MS: int = 1200
|
||||
FETCH_TIMEOUT_MS: int = 25000
|
||||
BROWSER_CLOSE_TIMEOUT_S: float = 15.0
|
||||
LAUNCH_TIMEOUT_S: float = 60.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RenderResult:
|
||||
url: str
|
||||
final_url: str
|
||||
status: int
|
||||
html: str
|
||||
content_type: str
|
||||
ok: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
url: str
|
||||
status: int
|
||||
body: bytes
|
||||
content_type: str
|
||||
ok: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def browser_available() -> bool:
|
||||
try:
|
||||
import playwright.async_api # noqa: F401
|
||||
import playwright_stealth # noqa: F401
|
||||
except ImportError as error:
|
||||
logger.info("Stealth browser unavailable, will use HTTP client: %s", error)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class StealthBrowser:
|
||||
def __init__(self, tab_concurrency: int = 4) -> None:
|
||||
self._tab_concurrency = max(1, tab_concurrency)
|
||||
self._stealth_manager = None
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._tab_semaphore = asyncio.Semaphore(self._tab_concurrency)
|
||||
|
||||
async def __aenter__(self) -> "StealthBrowser":
|
||||
await self._start()
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: Optional[type[BaseException]],
|
||||
exc: Optional[BaseException],
|
||||
traceback: Optional[TracebackType],
|
||||
) -> None:
|
||||
await self.aclose()
|
||||
|
||||
async def _start(self) -> None:
|
||||
from playwright.async_api import async_playwright
|
||||
from playwright_stealth import Stealth
|
||||
|
||||
stealth = Stealth()
|
||||
self._stealth_manager = stealth.use_async(async_playwright())
|
||||
self._playwright = await asyncio.wait_for(self._stealth_manager.__aenter__(), timeout=LAUNCH_TIMEOUT_S)
|
||||
self._browser = await self._launch_browser()
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent=BROWSER_USER_AGENT,
|
||||
viewport=BROWSER_VIEWPORT,
|
||||
locale=BROWSER_LOCALE,
|
||||
timezone_id=BROWSER_TIMEZONE,
|
||||
ignore_https_errors=True,
|
||||
java_script_enabled=True,
|
||||
)
|
||||
self._context.set_default_navigation_timeout(NAVIGATION_TIMEOUT_MS)
|
||||
self._context.set_default_timeout(NAVIGATION_TIMEOUT_MS)
|
||||
logger.info("Stealth browser ready (tab concurrency %d)", self._tab_concurrency)
|
||||
|
||||
async def _launch_browser(self):
|
||||
errors: list[str] = []
|
||||
for channel in ("chrome", None):
|
||||
try:
|
||||
kwargs: dict[str, object] = {"headless": True, "args": LAUNCH_ARGS}
|
||||
if channel:
|
||||
kwargs["channel"] = channel
|
||||
browser = await asyncio.wait_for(
|
||||
self._playwright.chromium.launch(**kwargs), timeout=LAUNCH_TIMEOUT_S
|
||||
)
|
||||
logger.info("Chromium launched (channel=%s)", channel or "bundled")
|
||||
return browser
|
||||
except Exception as error: # noqa: BLE001 - launch failures are heterogeneous, we retry the next channel
|
||||
errors.append(f"{channel or 'bundled'}: {error}")
|
||||
logger.warning("Browser launch failed on channel %s: %s", channel or "bundled", error)
|
||||
raise RuntimeError(f"Could not launch any Chromium build: {'; '.join(errors)}")
|
||||
|
||||
async def _goto_and_settle(self, page: object, url: str) -> object:
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=NAVIGATION_TIMEOUT_MS)
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=RENDER_SETTLE_MS * 3)
|
||||
except PlaywrightTimeout:
|
||||
await page.wait_for_timeout(RENDER_SETTLE_MS)
|
||||
return response
|
||||
|
||||
async def _build_render_result(self, page: object, response: object, url: str) -> RenderResult:
|
||||
html = await page.content()
|
||||
status = response.status if response else 0
|
||||
content_type = ""
|
||||
if response is not None:
|
||||
headers = await response.all_headers()
|
||||
content_type = headers.get("content-type", "")
|
||||
final_url = page.url
|
||||
return RenderResult(
|
||||
url=url,
|
||||
final_url=final_url,
|
||||
status=status,
|
||||
html=html,
|
||||
content_type=content_type or "text/html",
|
||||
ok=bool(html) and status < 400,
|
||||
)
|
||||
|
||||
async def render(self, url: str) -> RenderResult:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
async with self._tab_semaphore:
|
||||
page = None
|
||||
try:
|
||||
page = await self._context.new_page()
|
||||
response = await self._goto_and_settle(page, url)
|
||||
return await self._build_render_result(page, response, url)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Render failed for %s: %s", url, error)
|
||||
return RenderResult(url, url, 0, "", "", False, str(error))
|
||||
finally:
|
||||
if page is not None:
|
||||
try:
|
||||
await page.close()
|
||||
except Exception as error: # noqa: BLE001 - a failed tab close must never abort the crawl
|
||||
logger.debug("Tab close error for %s: %s", url, error)
|
||||
|
||||
async def _evaluate_dom_and_screenshot(self, page: object) -> tuple[dict, Optional[bytes]]:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
dom: dict = {}
|
||||
try:
|
||||
dom = await page.evaluate(DOM_EXTRACT_SCRIPT)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("DOM evaluate failed: %s", error)
|
||||
dom = {}
|
||||
screenshot_bytes: Optional[bytes] = None
|
||||
try:
|
||||
screenshot_bytes = await page.screenshot(full_page=False)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Screenshot capture failed: %s", error)
|
||||
screenshot_bytes = None
|
||||
return dom if isinstance(dom, dict) else {}, screenshot_bytes
|
||||
|
||||
async def capture(self, url: str) -> DomSnapshot:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
async with self._tab_semaphore:
|
||||
page = None
|
||||
try:
|
||||
page = await self._context.new_page()
|
||||
console_warnings: list[str] = []
|
||||
console_errors: list[str] = []
|
||||
|
||||
def _on_console(message: object) -> None:
|
||||
try:
|
||||
kind = message.type
|
||||
text = message.text[:HEADER_VALUE_CAP]
|
||||
except Exception as error:
|
||||
logger.debug("Console handler error: %s", error)
|
||||
return
|
||||
if kind == "warning" and len(console_warnings) < CONSOLE_SAMPLE_CAP:
|
||||
console_warnings.append(text)
|
||||
elif kind == "error" and len(console_errors) < CONSOLE_SAMPLE_CAP:
|
||||
console_errors.append(text)
|
||||
|
||||
page.on("console", _on_console)
|
||||
|
||||
response_headers: dict[str, str] = {}
|
||||
resource_hosts: list[str] = []
|
||||
seen_hosts: set[str] = set()
|
||||
captured_main = False
|
||||
|
||||
def _on_response(response: object) -> None:
|
||||
nonlocal captured_main
|
||||
try:
|
||||
hostname = urlparse(response.url).hostname
|
||||
if hostname and hostname not in seen_hosts:
|
||||
seen_hosts.add(hostname)
|
||||
resource_hosts.append(hostname)
|
||||
if not captured_main and response.frame == page.main_frame:
|
||||
captured_main = True
|
||||
for key, value in response.headers.items():
|
||||
response_headers[key] = str(value)[:HEADER_VALUE_CAP]
|
||||
except Exception as error:
|
||||
logger.debug("Response handler error: %s", error)
|
||||
|
||||
page.on("response", _on_response)
|
||||
await page.add_init_script(DOM_INIT_SCRIPT)
|
||||
response = await self._goto_and_settle(page, url)
|
||||
render = await self._build_render_result(page, response, url)
|
||||
dom: dict = {}
|
||||
screenshot_bytes: Optional[bytes] = None
|
||||
try:
|
||||
dom, screenshot_bytes = await asyncio.wait_for(
|
||||
self._evaluate_dom_and_screenshot(page),
|
||||
timeout=DOM_CAPTURE_TIMEOUT_MS / 1000,
|
||||
)
|
||||
except asyncio.TimeoutError as error:
|
||||
logger.warning("DOM capture timed out for %s: %s", url, error)
|
||||
return DomSnapshot(
|
||||
render=render,
|
||||
dom=dom,
|
||||
console_warnings=console_warnings,
|
||||
console_errors=console_errors,
|
||||
response_headers=response_headers,
|
||||
resource_hosts=resource_hosts,
|
||||
screenshot_bytes=screenshot_bytes,
|
||||
)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Capture failed for %s: %s", url, error)
|
||||
return DomSnapshot(render=RenderResult(url, url, 0, "", "", False, str(error)))
|
||||
finally:
|
||||
if page is not None:
|
||||
try:
|
||||
await page.close()
|
||||
except Exception as error:
|
||||
logger.debug("Tab close error for %s: %s", url, error)
|
||||
|
||||
async def fetch(self, url: str, max_bytes: int) -> FetchResult:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
|
||||
try:
|
||||
response = await self._context.request.get(url, timeout=FETCH_TIMEOUT_MS)
|
||||
body = await response.body()
|
||||
if len(body) > max_bytes:
|
||||
body = body[:max_bytes]
|
||||
content_type = response.headers.get("content-type", "")
|
||||
return FetchResult(url, response.status, body, content_type, response.status < 400)
|
||||
except (PlaywrightError, asyncio.TimeoutError) as error:
|
||||
logger.warning("Fetch failed for %s: %s", url, error)
|
||||
return FetchResult(url, 0, b"", "", False, str(error))
|
||||
|
||||
async def aclose(self) -> None:
|
||||
for closer, label in (
|
||||
(self._context, "context"),
|
||||
(self._browser, "browser"),
|
||||
):
|
||||
if closer is not None:
|
||||
try:
|
||||
await asyncio.wait_for(closer.close(), timeout=BROWSER_CLOSE_TIMEOUT_S)
|
||||
except Exception as error: # noqa: BLE001 - cleanup must be exhaustive
|
||||
logger.warning("Error closing %s: %s", label, error)
|
||||
self._context = None
|
||||
self._browser = None
|
||||
if self._stealth_manager is not None:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stealth_manager.__aexit__(None, None, None), timeout=BROWSER_CLOSE_TIMEOUT_S
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - playwright teardown must not leak
|
||||
logger.warning("Error stopping playwright: %s", error)
|
||||
self._stealth_manager = None
|
||||
self._playwright = None
|
||||
logger.info("Stealth browser closed")
|
||||
@@ -0,0 +1,212 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import RenderResult
|
||||
|
||||
DOM_INIT_SCRIPT = """
|
||||
window.__isslop = {};
|
||||
"""
|
||||
|
||||
DOM_EXTRACT_SCRIPT = r"""
|
||||
() => {
|
||||
const safe = (fn, fallback) => { try { return fn(); } catch (e) { return fallback; } };
|
||||
const metaByName = (name) => safe(() => {
|
||||
const el = document.querySelector(`meta[name="${name}"]`);
|
||||
return el ? (el.getAttribute('content') || '') : '';
|
||||
}, '');
|
||||
const metaByProperty = (name) => safe(() => {
|
||||
const el = document.querySelector(`meta[property="${name}"]`);
|
||||
return el ? (el.getAttribute('content') || '') : '';
|
||||
}, '');
|
||||
const linkHref = (rel) => safe(() => {
|
||||
const el = document.querySelector(`link[rel="${rel}"]`);
|
||||
return el ? (el.getAttribute('href') || '') : '';
|
||||
}, '');
|
||||
const meta = {
|
||||
title: safe(() => (document.title || '').trim(), ''),
|
||||
generator: metaByName('generator'),
|
||||
description: metaByName('description'),
|
||||
ogImage: metaByProperty('og:image'),
|
||||
canonical: linkHref('canonical'),
|
||||
htmlLang: safe(() => document.documentElement.getAttribute('lang') || '', ''),
|
||||
favicon: safe(() => linkHref('icon') || linkHref('shortcut icon'), ''),
|
||||
};
|
||||
const badgeSelectors = [
|
||||
'#lovable-badge',
|
||||
'.replit-badge',
|
||||
'#base44-badge',
|
||||
'#__framer-badge-container',
|
||||
'[data-radix-root]',
|
||||
'iframe[src*="claudeusercontent.com"]',
|
||||
];
|
||||
const badgeHits = {};
|
||||
badgeSelectors.forEach((sel) => {
|
||||
badgeHits[sel] = safe(() => !!document.querySelector(sel), false);
|
||||
});
|
||||
const headings = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('h1,h2,h3,h4,h5,h6').forEach((h) => {
|
||||
if (headings.length >= 60) return;
|
||||
headings.push({
|
||||
level: parseInt(h.tagName.substring(1), 10),
|
||||
text: (h.textContent || '').trim().slice(0, 160),
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const landmarks = {
|
||||
main: safe(() => document.querySelectorAll('main').length, 0),
|
||||
nav: safe(() => document.querySelectorAll('nav').length, 0),
|
||||
header: safe(() => document.querySelectorAll('header').length, 0),
|
||||
footer: safe(() => document.querySelectorAll('footer').length, 0),
|
||||
article: safe(() => document.querySelectorAll('article').length, 0),
|
||||
};
|
||||
const images = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('img').forEach((img) => {
|
||||
if (images.length >= 100) return;
|
||||
images.push({
|
||||
src: img.getAttribute('src') || '',
|
||||
alt: img.getAttribute('alt') || '',
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const classCounts = {};
|
||||
safe(() => {
|
||||
document.querySelectorAll('[class]').forEach((el) => {
|
||||
const raw = el.getAttribute('class') || '';
|
||||
raw.split(/\s+/).filter(Boolean).forEach((token) => {
|
||||
classCounts[token] = (classCounts[token] || 0) + 1;
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const classNames = {};
|
||||
Object.keys(classCounts)
|
||||
.sort((a, b) => classCounts[b] - classCounts[a])
|
||||
.slice(0, 40)
|
||||
.forEach((token) => { classNames[token] = classCounts[token]; });
|
||||
const colorSample = (tag, el) => safe(() => {
|
||||
const style = getComputedStyle(el);
|
||||
return {
|
||||
tag: tag,
|
||||
backgroundImage: style.backgroundImage || '',
|
||||
backgroundColor: style.backgroundColor || '',
|
||||
color: style.color || '',
|
||||
boxShadow: style.boxShadow || '',
|
||||
};
|
||||
}, null);
|
||||
const colors = [];
|
||||
safe(() => {
|
||||
if (document.body) {
|
||||
const sample = colorSample('body', document.body);
|
||||
if (sample) colors.push(sample);
|
||||
}
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('h1,h2')).slice(0, 6).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('button, a.btn, [class*="btn"]')).slice(0, 10).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('nav, header')).slice(0, 4).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('[class*="hero"]')).slice(0, 4).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
const fontSample = (tag, el) => safe(() => {
|
||||
const style = getComputedStyle(el);
|
||||
return {
|
||||
tag: tag,
|
||||
fontFamily: style.fontFamily || '',
|
||||
fontWeight: style.fontWeight || '',
|
||||
letterSpacing: style.letterSpacing || '',
|
||||
fontSize: style.fontSize || '',
|
||||
lineHeight: style.lineHeight || '',
|
||||
};
|
||||
}, null);
|
||||
const fonts = [];
|
||||
safe(() => {
|
||||
if (document.body) {
|
||||
const sample = fontSample('body', document.body);
|
||||
if (sample) fonts.push(sample);
|
||||
}
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('h1,h2,h3')).slice(0, 10).forEach((el) => {
|
||||
const sample = fontSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) fonts.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('p, button')).slice(0, 8).forEach((el) => {
|
||||
if (fonts.length >= 20) return;
|
||||
const sample = fontSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) fonts.push(sample);
|
||||
});
|
||||
}, null);
|
||||
const scripts = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('script[src]').forEach((s) => {
|
||||
if (scripts.length >= 60) return;
|
||||
scripts.push(s.getAttribute('src') || '');
|
||||
});
|
||||
}, null);
|
||||
const links = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('link[href]').forEach((l) => {
|
||||
if (links.length >= 60) return;
|
||||
links.push(l.getAttribute('href') || '');
|
||||
});
|
||||
}, null);
|
||||
const wordCount = safe(() => {
|
||||
const text = document.body ? (document.body.innerText || '') : '';
|
||||
return text.split(/\s+/).filter(Boolean).length;
|
||||
}, 0);
|
||||
const deadAnchorCount = safe(() => document.querySelectorAll('a[href="#"]').length, 0);
|
||||
const jsonldCount = safe(() => document.querySelectorAll('script[type="application/ld+json"]').length, 0);
|
||||
const robotsMeta = metaByName('robots');
|
||||
return {
|
||||
meta: meta,
|
||||
badgeHits: badgeHits,
|
||||
headings: headings.slice(0, 60),
|
||||
landmarks: landmarks,
|
||||
images: images.slice(0, 100),
|
||||
classNames: classNames,
|
||||
colors: colors.slice(0, 30),
|
||||
fonts: fonts.slice(0, 20),
|
||||
jsonld: jsonldCount,
|
||||
scripts: scripts.slice(0, 60),
|
||||
links: links.slice(0, 60),
|
||||
wordCount: wordCount,
|
||||
deadAnchorCount: deadAnchorCount,
|
||||
robotsMeta: robotsMeta,
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomSnapshot:
|
||||
render: RenderResult
|
||||
dom: dict = field(default_factory=dict)
|
||||
console_warnings: list[str] = field(default_factory=list)
|
||||
console_errors: list[str] = field(default_factory=list)
|
||||
response_headers: dict[str, str] = field(default_factory=dict)
|
||||
resource_hosts: list[str] = field(default_factory=list)
|
||||
screenshot_bytes: bytes | None = None
|
||||
@@ -0,0 +1,163 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import directory_size_bytes
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
GIT_CLONE_TIMEOUT_SECONDS,
|
||||
GIT_SIZE_LIMIT_BYTES,
|
||||
GIT_SIZE_POLL_SECONDS,
|
||||
WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RepositoryTooLargeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CloneFailedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SizePreflight:
|
||||
known: bool
|
||||
size_bytes: int
|
||||
origin: str
|
||||
|
||||
|
||||
def _owner_repo(url: str) -> Optional[tuple[str, str, str]]:
|
||||
match = re.match(r"^git@([\w.-]+):(.+)$", url)
|
||||
if match:
|
||||
host, path = match.group(1), match.group(2)
|
||||
else:
|
||||
parsed = urlparse(url)
|
||||
host, path = parsed.hostname or "", parsed.path
|
||||
parts = [part for part in path.strip("/").removesuffix(".git").split("/") if part]
|
||||
if len(parts) < 2 or not host:
|
||||
return None
|
||||
return host, parts[0], parts[1]
|
||||
|
||||
|
||||
async def preflight_size(url: str) -> SizePreflight:
|
||||
located = _owner_repo(url)
|
||||
if located is None:
|
||||
return SizePreflight(known=False, size_bytes=0, origin="unparseable")
|
||||
host, owner, repo = located
|
||||
candidates: list[tuple[str, str]] = []
|
||||
if host == "github.com":
|
||||
candidates.append((f"https://api.github.com/repos/{owner}/{repo}", "github"))
|
||||
else:
|
||||
candidates.append((f"https://{host}/api/v1/repos/{owner}/{repo}", "gitea"))
|
||||
async with stealth_async_client(timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS, follow_redirects=True) as client:
|
||||
for api_url, origin in candidates:
|
||||
try:
|
||||
response = await client.get(api_url, headers={"accept": "application/json"})
|
||||
except httpx.HTTPError as error:
|
||||
logger.info("Size preflight unavailable via %s: %s", origin, error)
|
||||
continue
|
||||
if response.status_code != 200:
|
||||
logger.info("Size preflight %s returned HTTP %d", origin, response.status_code)
|
||||
continue
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
continue
|
||||
size_kb = body.get("size")
|
||||
if isinstance(size_kb, (int, float)) and size_kb > 0:
|
||||
size_bytes = int(size_kb) * 1024
|
||||
logger.info("Preflight size via %s: %d bytes", origin, size_bytes)
|
||||
return SizePreflight(known=True, size_bytes=size_bytes, origin=origin)
|
||||
return SizePreflight(known=False, size_bytes=0, origin="unknown")
|
||||
|
||||
|
||||
async def clone_repository(url: str, workspace: Path, size_limit: int = GIT_SIZE_LIMIT_BYTES) -> AsyncIterator[str]:
|
||||
preflight = await preflight_size(url)
|
||||
if preflight.known:
|
||||
yield f"Repository size reported by {preflight.origin}: {preflight.size_bytes / (1024 * 1024):.1f} MB"
|
||||
if preflight.size_bytes > size_limit:
|
||||
raise RepositoryTooLargeError(
|
||||
f"Repository is {preflight.size_bytes / (1024 ** 3):.2f} GB which exceeds the 3 GB limit"
|
||||
)
|
||||
else:
|
||||
yield "Repository size not determinable up front, monitoring during clone"
|
||||
|
||||
argv = [
|
||||
"git",
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--single-branch",
|
||||
"--no-tags",
|
||||
url,
|
||||
str(workspace),
|
||||
]
|
||||
yield f"Cloning with depth 1: {url}"
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
env={"GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "true", "PATH": "/usr/bin:/bin:/usr/local/bin"},
|
||||
)
|
||||
|
||||
aborted_reason: list[str] = []
|
||||
|
||||
async def monitor() -> None:
|
||||
while proc.returncode is None:
|
||||
await asyncio.sleep(GIT_SIZE_POLL_SECONDS)
|
||||
if not workspace.exists():
|
||||
continue
|
||||
size = await asyncio.to_thread(directory_size_bytes, workspace)
|
||||
logger.debug("Clone size check: %d bytes", size)
|
||||
if size > size_limit:
|
||||
aborted_reason.append(f"Clone exceeded 3 GB limit at {size / (1024 ** 3):.2f} GB, cancelled")
|
||||
proc.kill()
|
||||
return
|
||||
|
||||
monitor_task = asyncio.create_task(monitor())
|
||||
output_lines: list[str] = []
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
while True:
|
||||
try:
|
||||
line = await asyncio.wait_for(proc.stdout.readline(), timeout=GIT_CLONE_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
raise CloneFailedError("Clone timed out")
|
||||
if not line:
|
||||
break
|
||||
text = line.decode("utf-8", errors="replace").strip()
|
||||
if text:
|
||||
output_lines.append(text)
|
||||
yield f"git: {text}"
|
||||
await proc.wait()
|
||||
finally:
|
||||
monitor_task.cancel()
|
||||
try:
|
||||
await monitor_task
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("Clone size monitor stopped")
|
||||
|
||||
if aborted_reason:
|
||||
raise RepositoryTooLargeError(aborted_reason[0])
|
||||
if proc.returncode != 0:
|
||||
tail = " | ".join(output_lines[-4:])
|
||||
raise CloneFailedError(f"git clone failed with code {proc.returncode}: {tail}")
|
||||
final_size = await asyncio.to_thread(directory_size_bytes, workspace)
|
||||
if final_size > size_limit:
|
||||
raise RepositoryTooLargeError(f"Repository is {final_size / (1024 ** 3):.2f} GB which exceeds the 3 GB limit")
|
||||
yield f"Clone complete: {final_size / (1024 * 1024):.1f} MB on disk"
|
||||
@@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.config import GIT_PROBE_TIMEOUT_SECONDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND_GIT: str = "git"
|
||||
KIND_WEBSITE: str = "website"
|
||||
|
||||
GIT_URL_PATTERN = re.compile(r"(\.git$|^git://|^ssh://git@|^git@)", re.IGNORECASE)
|
||||
GIT_HOST_HINTS: tuple[str, ...] = ("github.com", "gitlab.com", "bitbucket.org", "codeberg.org", "gitea.")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceResolution:
|
||||
url: str
|
||||
kind: str
|
||||
probe_output: str
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
url = url.strip()
|
||||
if re.match(r"^git@[\w.-]+:", url):
|
||||
return url
|
||||
if not re.match(r"^[a-z]+://", url, re.IGNORECASE):
|
||||
url = f"https://{url}"
|
||||
return url
|
||||
|
||||
|
||||
def is_private_host(url: str) -> bool:
|
||||
parsed = urlparse(url if "://" in url else f"ssh://{url}")
|
||||
host = parsed.hostname or ""
|
||||
if not host:
|
||||
return True
|
||||
if host in ("localhost",):
|
||||
return True
|
||||
try:
|
||||
address = ipaddress.ip_address(host)
|
||||
return address.is_private or address.is_loopback or address.is_link_local or address.is_reserved
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
resolved = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
for entry in resolved:
|
||||
candidate = ipaddress.ip_address(entry[4][0])
|
||||
if candidate.is_private or candidate.is_loopback or candidate.is_link_local:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def probe_git(url: str) -> tuple[bool, str]:
|
||||
argv = ["git", "ls-remote", "--heads", "--exit-code", url]
|
||||
logger.info("Probing for git repository: %s", url)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env={"GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "true", "PATH": "/usr/bin:/bin:/usr/local/bin"},
|
||||
)
|
||||
except OSError as error:
|
||||
logger.error("git probe spawn failed: %s", error)
|
||||
return False, str(error)
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=GIT_PROBE_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
logger.warning("git probe timed out for %s", url)
|
||||
return False, "probe timeout"
|
||||
output = stdout.decode("utf-8", errors="replace")
|
||||
errors = stderr.decode("utf-8", errors="replace")
|
||||
is_git = proc.returncode == 0 and bool(output.strip())
|
||||
logger.info("git probe %s: is_git=%s refs=%d", url, is_git, len(output.splitlines()))
|
||||
return is_git, output if is_git else errors[:500]
|
||||
|
||||
|
||||
async def resolve_source(url: str) -> SourceResolution:
|
||||
normalized = normalize_url(url)
|
||||
hinted = bool(GIT_URL_PATTERN.search(normalized)) or any(hint in normalized.lower() for hint in GIT_HOST_HINTS)
|
||||
is_git, probe_output = await probe_git(normalized)
|
||||
if is_git:
|
||||
return SourceResolution(url=normalized, kind=KIND_GIT, probe_output=probe_output)
|
||||
if hinted:
|
||||
logger.info("URL hinted git but probe failed, treating as website: %s", normalized)
|
||||
return SourceResolution(url=normalized, kind=KIND_WEBSITE, probe_output=probe_output)
|
||||
@@ -0,0 +1,315 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
from urllib.parse import urldefrag, urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.net_guard import BlockedAddressError, guard_public_url
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
from devplacepy.services.jobs.isslop.acquisition.domcapture import DomSnapshot
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import safe_relative_path
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
DOM_ANALYSIS_MAX_PAGES,
|
||||
WEBSITE_MAX_DEPTH,
|
||||
WEBSITE_MAX_FILE_BYTES,
|
||||
WEBSITE_MAX_FILES,
|
||||
WEBSITE_MAX_PAGES,
|
||||
WEBSITE_MAX_TOTAL_BYTES,
|
||||
WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
||||
TEXT_CONTENT_HINTS: tuple[str, ...] = ("text/", "javascript", "json", "xml", "svg", "css")
|
||||
SKIP_SCHEMES: tuple[str, ...] = ("mailto:", "tel:", "javascript:", "data:", "#")
|
||||
|
||||
|
||||
class LinkExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.page_links: list[str] = []
|
||||
self.asset_links: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
mapping = dict(attrs)
|
||||
if tag == "a" and mapping.get("href"):
|
||||
self.page_links.append(str(mapping["href"]))
|
||||
if tag in ("script", "img", "source", "iframe") and mapping.get("src"):
|
||||
self.asset_links.append(str(mapping["src"]))
|
||||
if tag == "link" and mapping.get("href"):
|
||||
self.asset_links.append(str(mapping["href"]))
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlState:
|
||||
pages_fetched: int = 0
|
||||
files_saved: int = 0
|
||||
bytes_saved: int = 0
|
||||
visited: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
def _clean_url(base: str, href: str) -> str:
|
||||
joined = urljoin(base, href)
|
||||
return urldefrag(joined).url
|
||||
|
||||
|
||||
def _local_path_for(workspace: Path, url: str) -> Path:
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or "/"
|
||||
if path.endswith("/"):
|
||||
path += "index.html"
|
||||
name = path.lstrip("/") or "index.html"
|
||||
if "." not in name.split("/")[-1]:
|
||||
name += ".html"
|
||||
if parsed.query:
|
||||
name += "-" + "".join(ch for ch in parsed.query if ch.isalnum())[:40]
|
||||
return safe_relative_path(workspace, name)
|
||||
|
||||
|
||||
def _is_text_response(content_type: str) -> bool:
|
||||
lowered = content_type.lower()
|
||||
return any(hint in lowered for hint in TEXT_CONTENT_HINTS)
|
||||
|
||||
|
||||
def _same_site(candidate: str, root_host: str | None) -> bool:
|
||||
host = urlparse(candidate).hostname
|
||||
if host is None or root_host is None:
|
||||
return False
|
||||
return host == root_host or host.endswith("." + root_host) or root_host.endswith("." + host)
|
||||
|
||||
|
||||
def _extract_links(base: str, html: str, root_host: str | None) -> tuple[list[str], list[str]]:
|
||||
extractor = LinkExtractor()
|
||||
try:
|
||||
extractor.feed(html)
|
||||
except ValueError as error:
|
||||
logger.debug("HTML parse issue on %s: %s", base, error)
|
||||
pages: list[str] = []
|
||||
assets: list[str] = []
|
||||
for asset in extractor.asset_links:
|
||||
cleaned = _clean_url(base, asset)
|
||||
if _same_site(cleaned, root_host):
|
||||
assets.append(cleaned)
|
||||
for link in extractor.page_links:
|
||||
cleaned = _clean_url(base, link)
|
||||
if _same_site(cleaned, root_host):
|
||||
pages.append(cleaned)
|
||||
return pages, assets
|
||||
|
||||
|
||||
def _save_bytes(workspace: Path, url: str, body: bytes) -> Path | None:
|
||||
try:
|
||||
target = _local_path_for(workspace, url)
|
||||
except ValueError as error:
|
||||
logger.warning("Rejected path for %s: %s", url, error)
|
||||
return None
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
return target
|
||||
|
||||
|
||||
async def crawl_website(
|
||||
url: str,
|
||||
workspace: Path,
|
||||
*,
|
||||
dom_sink: list[DomSnapshot] | None = None,
|
||||
allow_private: bool = False,
|
||||
) -> AsyncIterator[str]:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import browser_available
|
||||
|
||||
if browser_available():
|
||||
try:
|
||||
async for progress in _crawl_with_browser(
|
||||
url, workspace, dom_sink=dom_sink, allow_private=allow_private
|
||||
):
|
||||
yield progress
|
||||
return
|
||||
except RuntimeError as error:
|
||||
logger.warning("Stealth browser crawl failed, falling back to HTTP client: %s", error)
|
||||
yield f"Stealth browser unavailable ({error}); falling back to HTTP client"
|
||||
async for progress in _crawl_with_http(url, workspace):
|
||||
yield progress
|
||||
|
||||
|
||||
async def _crawl_with_browser(
|
||||
url: str,
|
||||
workspace: Path,
|
||||
*,
|
||||
dom_sink: list[DomSnapshot] | None = None,
|
||||
allow_private: bool = False,
|
||||
) -> AsyncIterator[str]:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import StealthBrowser
|
||||
|
||||
root_host = urlparse(url).hostname
|
||||
state = CrawlState()
|
||||
yield f"Launching stealth browser for {url} (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
|
||||
async with StealthBrowser(tab_concurrency=4) as browser:
|
||||
yield "Stealth browser ready; rendering pages with an undetectable Chromium fingerprint"
|
||||
current_level: list[str] = [url]
|
||||
for depth in range(WEBSITE_MAX_DEPTH + 1):
|
||||
pending = [
|
||||
candidate
|
||||
for candidate in current_level
|
||||
if candidate not in state.visited and _same_site(candidate, root_host)
|
||||
and not any(candidate.lower().startswith(scheme) for scheme in SKIP_SCHEMES)
|
||||
]
|
||||
if not pending or state.files_saved >= WEBSITE_MAX_FILES:
|
||||
break
|
||||
pending = pending[: max(0, WEBSITE_MAX_PAGES - state.pages_fetched)]
|
||||
safe_candidates: list[str] = []
|
||||
for candidate in pending:
|
||||
try:
|
||||
await guard_public_url(candidate, allow_private=allow_private)
|
||||
except BlockedAddressError:
|
||||
state.visited.add(candidate)
|
||||
yield f"Blocked private/local address: {candidate}"
|
||||
continue
|
||||
safe_candidates.append(candidate)
|
||||
for candidate in safe_candidates:
|
||||
state.visited.add(candidate)
|
||||
|
||||
async def _render_or_capture(index: int, candidate: str) -> object:
|
||||
if depth == 0 and dom_sink is not None and index < DOM_ANALYSIS_MAX_PAGES:
|
||||
snapshot = await browser.capture(candidate)
|
||||
dom_sink.append(snapshot)
|
||||
return snapshot.render
|
||||
return await browser.render(candidate)
|
||||
|
||||
renders = await asyncio.gather(
|
||||
*(_render_or_capture(index, candidate) for index, candidate in enumerate(safe_candidates))
|
||||
)
|
||||
asset_urls: set[str] = set()
|
||||
next_level: list[str] = []
|
||||
for render in renders:
|
||||
if not render.ok:
|
||||
yield f"Render failed: {render.url} ({render.error or 'no content'})"
|
||||
continue
|
||||
body = render.html.encode("utf-8", errors="replace")[:WEBSITE_MAX_FILE_BYTES]
|
||||
target = _save_bytes(workspace, render.final_url, body)
|
||||
if target is None:
|
||||
continue
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(body)
|
||||
state.pages_fetched += 1
|
||||
yield f"Rendered {target.relative_to(workspace)} ({len(body)} bytes, DOM after JS execution)"
|
||||
pages, assets = _extract_links(render.final_url, render.html, root_host)
|
||||
for asset in assets:
|
||||
if asset not in state.visited:
|
||||
asset_urls.add(asset)
|
||||
next_level.extend(pages)
|
||||
async for progress in _download_assets(browser, workspace, sorted(asset_urls), state):
|
||||
yield progress
|
||||
current_level = next_level
|
||||
if state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
|
||||
yield f"Byte budget reached: {state.bytes_saved / 1024:.1f} KB"
|
||||
break
|
||||
if state.files_saved == 0:
|
||||
raise RuntimeError(f"Stealth crawl produced no files for {url}")
|
||||
yield (
|
||||
f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, "
|
||||
f"{state.pages_fetched} rendered pages"
|
||||
)
|
||||
|
||||
|
||||
async def _download_assets(
|
||||
browser: object,
|
||||
workspace: Path,
|
||||
asset_urls: list[str],
|
||||
state: CrawlState,
|
||||
) -> AsyncIterator[str]:
|
||||
budget = min(len(asset_urls), max(0, WEBSITE_MAX_FILES - state.files_saved))
|
||||
if budget <= 0:
|
||||
return
|
||||
targets = [asset for asset in asset_urls if asset not in state.visited][:budget]
|
||||
for asset in targets:
|
||||
state.visited.add(asset)
|
||||
results = await asyncio.gather(*(browser.fetch(asset, WEBSITE_MAX_FILE_BYTES) for asset in targets))
|
||||
saved = 0
|
||||
for result in results:
|
||||
if not result.ok or not result.body:
|
||||
continue
|
||||
target = _save_bytes(workspace, result.url, result.body)
|
||||
if target is None:
|
||||
continue
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(result.body)
|
||||
saved += 1
|
||||
if saved:
|
||||
yield f"Downloaded {saved} assets concurrently (scripts, styles, sources)"
|
||||
|
||||
|
||||
async def _crawl_with_http(url: str, workspace: Path) -> AsyncIterator[str]:
|
||||
root = urlparse(url)
|
||||
state = CrawlState()
|
||||
queue: list[tuple[str, int]] = [(url, 0)]
|
||||
headers = {"user-agent": USER_AGENT, "accept": "*/*"}
|
||||
yield f"Crawling website {url} with HTTP client (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
|
||||
async with stealth_async_client(
|
||||
timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
) as client:
|
||||
while queue:
|
||||
if state.files_saved >= WEBSITE_MAX_FILES or state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
|
||||
yield f"Crawl limits reached: {state.files_saved} files, {state.bytes_saved} bytes"
|
||||
break
|
||||
current, depth = queue.pop(0)
|
||||
if current in state.visited:
|
||||
continue
|
||||
state.visited.add(current)
|
||||
parsed = urlparse(current)
|
||||
if parsed.hostname != root.hostname:
|
||||
continue
|
||||
if any(current.lower().startswith(scheme) for scheme in SKIP_SCHEMES):
|
||||
continue
|
||||
try:
|
||||
response = await client.get(current)
|
||||
except httpx.HTTPError as error:
|
||||
logger.warning("Fetch failed %s: %s", current, error)
|
||||
yield f"Fetch failed: {current} ({error})"
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
yield f"Skipped {current}: HTTP {response.status_code}"
|
||||
continue
|
||||
content_type = response.headers.get("content-type", "")
|
||||
body = response.content[:WEBSITE_MAX_FILE_BYTES]
|
||||
try:
|
||||
target = _local_path_for(workspace, current)
|
||||
except ValueError as error:
|
||||
logger.warning("Rejected path for %s: %s", current, error)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(body)
|
||||
yield f"Saved {target.relative_to(workspace)} ({len(body)} bytes, {content_type.split(';')[0] or 'unknown type'})"
|
||||
|
||||
is_page = "text/html" in content_type.lower()
|
||||
if is_page:
|
||||
state.pages_fetched += 1
|
||||
if depth < WEBSITE_MAX_DEPTH and state.pages_fetched < WEBSITE_MAX_PAGES:
|
||||
extractor = LinkExtractor()
|
||||
try:
|
||||
extractor.feed(body.decode("utf-8", errors="replace"))
|
||||
except ValueError as error:
|
||||
logger.debug("HTML parse issue on %s: %s", current, error)
|
||||
for asset in extractor.asset_links:
|
||||
cleaned = _clean_url(current, asset)
|
||||
if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
|
||||
queue.append((cleaned, depth + 1))
|
||||
for link in extractor.page_links:
|
||||
cleaned = _clean_url(current, link)
|
||||
if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
|
||||
queue.append((cleaned, depth + 1))
|
||||
if state.files_saved == 0:
|
||||
raise RuntimeError(f"Website download produced no files for {url}")
|
||||
yield f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, {state.pages_fetched} pages"
|
||||
@@ -0,0 +1,85 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SLUG_MAX_LENGTH: int = 80
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
value = re.sub(r"^[a-z]+://", "", value.strip().lower())
|
||||
value = re.sub(r"[^a-z0-9._-]+", "-", value).strip("-._")
|
||||
if not value:
|
||||
value = "source"
|
||||
return value[:SLUG_MAX_LENGTH]
|
||||
|
||||
|
||||
def workspace_for(workspaces_dir: Path, source_url: str, analysis_uid: str) -> Path:
|
||||
digest = hashlib.sha256(source_url.encode("utf-8")).hexdigest()[:10]
|
||||
slug = slugify(source_url)
|
||||
workspace = (workspaces_dir / f"{slug}-{digest}-{analysis_uid}").resolve()
|
||||
if workspaces_dir.resolve() != workspace.parent:
|
||||
raise ValueError(f"Workspace path escapes workspaces directory: {workspace}")
|
||||
logger.info("Workspace resolved: %s", workspace)
|
||||
return workspace
|
||||
|
||||
|
||||
def reset_workspace(workspace: Path) -> Path:
|
||||
if workspace.exists():
|
||||
logger.info("Removing existing workspace %s", workspace)
|
||||
shutil.rmtree(workspace)
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
return workspace
|
||||
|
||||
|
||||
def safe_relative_path(workspace: Path, candidate: str) -> Path:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._/-]+", "-", candidate).strip("/")
|
||||
cleaned = re.sub(r"\.\.+", ".", cleaned)
|
||||
target = (workspace / cleaned).resolve()
|
||||
if workspace.resolve() not in target.parents and target != workspace.resolve():
|
||||
raise ValueError(f"Refusing path traversal: {candidate}")
|
||||
return target
|
||||
|
||||
|
||||
def remove_workspace(workspace: Path) -> bool:
|
||||
if not workspace.exists():
|
||||
return False
|
||||
try:
|
||||
shutil.rmtree(workspace)
|
||||
except OSError as error:
|
||||
logger.error("Workspace removal failed for %s: %s", workspace, error)
|
||||
return False
|
||||
logger.info("Workspace removed: %s", workspace)
|
||||
return True
|
||||
|
||||
|
||||
def directory_size_bytes(path: Path) -> int:
|
||||
total = 0
|
||||
for entry in path.rglob("*"):
|
||||
if entry.is_file() and not entry.is_symlink():
|
||||
try:
|
||||
total += entry.stat().st_size
|
||||
except OSError as error:
|
||||
logger.debug("Size stat failed for %s: %s", entry, error)
|
||||
return total
|
||||
|
||||
|
||||
def content_hash(workspace: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
entries: list[tuple[str, Path]] = []
|
||||
for entry in workspace.rglob("*"):
|
||||
if entry.is_file() and not entry.is_symlink() and ".git" not in entry.parts:
|
||||
entries.append((str(entry.relative_to(workspace)), entry))
|
||||
for relative, entry in sorted(entries):
|
||||
digest.update(relative.encode("utf-8"))
|
||||
try:
|
||||
digest.update(hashlib.sha256(entry.read_bytes()).digest())
|
||||
except OSError as error:
|
||||
logger.debug("Hash read failed for %s: %s", entry, error)
|
||||
return digest.hexdigest()
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,111 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError, extract_json_object
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import (
|
||||
CATEGORY_HUMAN_CLEAN,
|
||||
CATEGORY_HUMAN_MESSY,
|
||||
CATEGORY_SLOP,
|
||||
CATEGORY_SOPHISTICATED,
|
||||
CATEGORY_UNCERTAIN,
|
||||
FileScore,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_STRONG
|
||||
from devplacepy.services.jobs.isslop.config import AI_EXCERPT_CHARS, AI_SAMPLE_LIMIT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_CATEGORIES: frozenset[str] = frozenset(
|
||||
{CATEGORY_SLOP, CATEGORY_SOPHISTICATED, CATEGORY_HUMAN_CLEAN, CATEGORY_HUMAN_MESSY, CATEGORY_UNCERTAIN}
|
||||
)
|
||||
|
||||
SYSTEM_PROMPT: str = (
|
||||
"You are a deterministic source-code provenance and quality auditor. "
|
||||
"You classify code on two independent axes: origin (was it AI-generated) and "
|
||||
"curation quality (was it reviewed, integrated and production-worthy). "
|
||||
"The categories are: ai-slop (AI origin, low curation: the model's defaults shipped as-is), "
|
||||
"sophisticated-ai (AI origin, high curation: the model typed, the engineer decided), "
|
||||
"human-clean and human-messy (code no model would write that way), uncertain. "
|
||||
"Human messiness is not slop. Clean AI-assisted code is not slop. "
|
||||
"Judge only from the evidence given. Respond with a single JSON object and nothing else, using exactly these keys: "
|
||||
'{"origin_score": <0-100 integer, likelihood of AI generation>, '
|
||||
'"quality_deficit": <0-100 integer, density of anti-patterns and missing curation>, '
|
||||
'"ai_probability": <0-100 integer>, '
|
||||
'"category": <one of ai-slop|sophisticated-ai|human-clean|human-messy|uncertain>, '
|
||||
'"reasoning": <two to four factual sentences citing concrete evidence>, '
|
||||
'"notable_signals": [<up to five short strings>]}'
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AiVerdict:
|
||||
path: str
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
ai_probability: float
|
||||
category: str
|
||||
reasoning: str
|
||||
notable_signals: list[str]
|
||||
|
||||
|
||||
def select_samples(scores: list[FileScore], limit: int = AI_SAMPLE_LIMIT) -> list[FileScore]:
|
||||
def interest(entry: FileScore) -> tuple[float, str]:
|
||||
strong = sum(1 for signal in entry.signals if signal.severity == SEVERITY_STRONG)
|
||||
return (-(entry.sloc * (1.0 + strong) * entry.criticality), entry.relative)
|
||||
|
||||
ranked = sorted((entry for entry in scores if entry.sloc >= 10), key=interest)
|
||||
return ranked[:limit]
|
||||
|
||||
|
||||
def _clamp(value: object, fallback: float) -> float:
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return float(min(100.0, max(0.0, value)))
|
||||
return fallback
|
||||
|
||||
|
||||
async def classify_file(
|
||||
llm: LlmClient,
|
||||
score: FileScore,
|
||||
excerpt: str,
|
||||
) -> Optional[AiVerdict]:
|
||||
signal_lines = "\n".join(
|
||||
f"- [{signal.severity}] {signal.code} line {signal.line}: {signal.title}"
|
||||
for signal in score.signals[:15]
|
||||
) or "- none detected"
|
||||
user = (
|
||||
f"File: {score.relative}\n"
|
||||
f"Language: {score.language}\n"
|
||||
f"Static origin score: {score.origin_score}\n"
|
||||
f"Static quality deficit: {score.quality_deficit}\n"
|
||||
f"Static signals:\n{signal_lines}\n\n"
|
||||
f"Source excerpt (may be truncated):\n```\n{excerpt[:AI_EXCERPT_CHARS]}\n```"
|
||||
)
|
||||
try:
|
||||
answer = await llm.complete(SYSTEM_PROMPT, user)
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("AI classification unavailable for %s: %s", score.relative, error)
|
||||
return None
|
||||
parsed = extract_json_object(answer)
|
||||
if parsed is None:
|
||||
logger.warning("AI classification returned unparseable output for %s", score.relative)
|
||||
return None
|
||||
category = str(parsed.get("category", CATEGORY_UNCERTAIN)).strip().lower()
|
||||
if category not in VALID_CATEGORIES:
|
||||
category = CATEGORY_UNCERTAIN
|
||||
signals_field = parsed.get("notable_signals")
|
||||
notable = [str(item)[:120] for item in signals_field[:5]] if isinstance(signals_field, list) else []
|
||||
verdict = AiVerdict(
|
||||
path=score.relative,
|
||||
origin_score=_clamp(parsed.get("origin_score"), score.origin_score),
|
||||
quality_deficit=_clamp(parsed.get("quality_deficit"), score.quality_deficit),
|
||||
ai_probability=_clamp(parsed.get("ai_probability"), score.origin_score),
|
||||
category=category,
|
||||
reasoning=str(parsed.get("reasoning", ""))[:1200],
|
||||
notable_signals=notable,
|
||||
)
|
||||
logger.info("AI verdict for %s: %s (ai %d%%)", verdict.path, verdict.category, int(verdict.ai_probability))
|
||||
return verdict
|
||||
@@ -0,0 +1,159 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
LLM_MAX_RETRIES,
|
||||
LLM_RETRY_BACKOFF_SECONDS,
|
||||
LLM_TEMPERATURE,
|
||||
LLM_TIMEOUT_SECONDS,
|
||||
WorkerSettings,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LlmUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LlmClient:
|
||||
def __init__(self, settings: WorkerSettings) -> None:
|
||||
self._endpoint = settings.llm_endpoint
|
||||
self._model = settings.llm_model
|
||||
self._api_key = settings.llm_api_key
|
||||
self._review_enabled = settings.ai_review_enabled
|
||||
self._vision_enabled = settings.image_review_enabled
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def active_backend_name(self) -> str:
|
||||
return self._model
|
||||
|
||||
@property
|
||||
def review_available(self) -> bool:
|
||||
return self._review_enabled and bool(self._endpoint and self._api_key)
|
||||
|
||||
@property
|
||||
def vision_available(self) -> bool:
|
||||
return self._vision_enabled and bool(self._endpoint and self._api_key)
|
||||
|
||||
async def _http(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = stealth_async_client(timeout=httpx.Timeout(LLM_TIMEOUT_SECONDS))
|
||||
return self._client
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _call(self, messages: list[dict[str, Any]], timeout: Optional[float] = None) -> str:
|
||||
if not self._endpoint or not self._api_key:
|
||||
raise LlmUnavailableError("AI gateway not configured")
|
||||
client = await self._http()
|
||||
payload = {
|
||||
"model": self._model,
|
||||
"messages": messages,
|
||||
"temperature": LLM_TEMPERATURE,
|
||||
}
|
||||
headers = {"authorization": f"Bearer {self._api_key}", "content-type": "application/json"}
|
||||
request_timeout = httpx.Timeout(timeout) if timeout else None
|
||||
last_error = "unknown"
|
||||
for attempt in range(1, LLM_MAX_RETRIES + 1):
|
||||
try:
|
||||
response = await client.post(
|
||||
self._endpoint, json=payload, headers=headers, timeout=request_timeout
|
||||
)
|
||||
except httpx.HTTPError as error:
|
||||
last_error = f"{type(error).__name__}: {error}"
|
||||
logger.warning("Gateway attempt %d transport failure: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
last_error = f"HTTP {response.status_code}: {response.text[:300]}"
|
||||
logger.warning("Gateway attempt %d failed: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
continue
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
last_error = "invalid JSON envelope"
|
||||
continue
|
||||
choices = body.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
content = choices[0].get("message", {}).get("content")
|
||||
if isinstance(content, str) and content.strip():
|
||||
return content
|
||||
last_error = json.dumps(body)[:300]
|
||||
logger.warning("Gateway attempt %d empty completion: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
raise LlmUnavailableError(f"AI gateway failed after {LLM_MAX_RETRIES} attempts: {last_error}")
|
||||
|
||||
async def complete(self, system: str, user: str) -> str:
|
||||
if not self.review_available:
|
||||
raise LlmUnavailableError("AI review disabled")
|
||||
return await self._call(
|
||||
[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
)
|
||||
|
||||
async def describe_image(self, prompt: str, image_data_url: str, timeout: float) -> str:
|
||||
if not self.vision_available:
|
||||
raise LlmUnavailableError("vision backend unavailable")
|
||||
return await self._call(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": image_data_url}},
|
||||
],
|
||||
}
|
||||
],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def extract_json_object(text: str) -> Optional[dict[str, Any]]:
|
||||
start = text.find("{")
|
||||
while start != -1:
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for position in range(start, len(text)):
|
||||
char = text[position]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
candidate = text[start:position + 1]
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
break
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
break
|
||||
start = text.find("{", start + 1)
|
||||
return None
|
||||
@@ -0,0 +1,246 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.classifier import AiVerdict
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError
|
||||
from devplacepy.services.jobs.isslop.agent.vision import ImageVerdict
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.aggregate import DomEvidence
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import FileScore, RepoScores
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_STRONG
|
||||
from devplacepy.services.jobs.isslop.analysis.templates import TemplateEvidence
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPORT_SYSTEM_PROMPT: str = (
|
||||
"You are a technical auditor writing the final report of a source-code AI usage classification. "
|
||||
"Write factual, professional markdown. No emoji, no marketing language, no hedging filler. "
|
||||
"Structure: '# AI Usage Classification Report' title, then sections 'Verdict', 'Scores', "
|
||||
"'Evidence', 'Notable Files', 'Image Review' (only if image data is present), 'Methodology Notes', 'Caveats'. "
|
||||
"Cite concrete files and signals from the data, and wrap EVERY file path you mention in backticks; "
|
||||
"when citing a specific line, write it as `path:line` (like `src/libs/Env.ts:12`) so the platform can "
|
||||
"link the reference to the exact annotated source line. "
|
||||
"Present the result as calibrated guidance, not accusation. "
|
||||
"Category definitions: ai-slop ships the model's defaults untouched; sophisticated-ai shows AI steered by "
|
||||
"an engineer who knows what they are doing; human is code no model would write that way. "
|
||||
"When template_provenance markers are present, explain that the project ships a starter template's "
|
||||
"defaults and name the template evidence; shipping scaffold defaults counts against authenticity. "
|
||||
"A near-certain unmodified starter template IS ai-slop by definition: the defaults were shipped as-is, "
|
||||
"and clean scaffold code does not change that - the curation belongs to the template author, not the "
|
||||
"presenter. Never call an untouched template sophisticated-ai. "
|
||||
"When dom_evidence.detected_builder is present, lead the Verdict section with a line stating the site was "
|
||||
"detected as built with that no-code builder, treating the rendered-page fingerprint match as near-certain "
|
||||
"evidence. "
|
||||
"Keep it under 900 words. Output only markdown."
|
||||
)
|
||||
|
||||
|
||||
def _summary_payload(
|
||||
source_url: str,
|
||||
source_kind: str,
|
||||
scores: RepoScores,
|
||||
files: list[FileScore],
|
||||
verdicts: list[AiVerdict],
|
||||
excluded_count: int,
|
||||
image_verdicts: list[ImageVerdict],
|
||||
image_stats: dict[str, Any],
|
||||
template: TemplateEvidence,
|
||||
dom_evidence: DomEvidence | None,
|
||||
) -> dict[str, Any]:
|
||||
worst = sorted(files, key=lambda entry: (-entry.quality_deficit, entry.relative))[:10]
|
||||
strongest = [
|
||||
{
|
||||
"path": entry.relative,
|
||||
"origin": entry.origin_score,
|
||||
"quality_deficit": entry.quality_deficit,
|
||||
"category": entry.category,
|
||||
"signals": [signal.to_dict() for signal in entry.signals if signal.severity == SEVERITY_STRONG][:4],
|
||||
}
|
||||
for entry in worst
|
||||
]
|
||||
return {
|
||||
"source_url": source_url,
|
||||
"source_kind": source_kind,
|
||||
"repo_scores": {
|
||||
"grade": scores.grade,
|
||||
"slop_score": scores.slop_score,
|
||||
"origin_score": scores.origin_score,
|
||||
"quality_deficit": scores.quality_deficit,
|
||||
"category": scores.category,
|
||||
"human_percent": scores.human_percent,
|
||||
"ai_percent": scores.ai_percent,
|
||||
"confidence": scores.confidence,
|
||||
"files_scored": scores.files_scored,
|
||||
"excluded_files": excluded_count,
|
||||
"strong_signals": scores.strong_signal_count,
|
||||
"medium_signals": scores.medium_signal_count,
|
||||
},
|
||||
"worst_files": strongest,
|
||||
"ai_verdicts": [
|
||||
{
|
||||
"path": verdict.path,
|
||||
"category": verdict.category,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"reasoning": verdict.reasoning,
|
||||
"notable_signals": verdict.notable_signals,
|
||||
}
|
||||
for verdict in verdicts
|
||||
],
|
||||
"template_provenance": {
|
||||
"score": template.score,
|
||||
"markers": template.markers,
|
||||
},
|
||||
"image_review": image_stats,
|
||||
"image_verdicts": [
|
||||
{
|
||||
"path": verdict.relative,
|
||||
"grade": verdict.grade,
|
||||
"verdict": verdict.verdict,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"image_kind": verdict.image_kind,
|
||||
"tells": verdict.tells,
|
||||
"description": verdict.description,
|
||||
}
|
||||
for verdict in sorted(image_verdicts, key=lambda item: -item.ai_probability)[:12]
|
||||
],
|
||||
"dom_evidence": (
|
||||
{
|
||||
"detected_builder": dom_evidence.detected_builder,
|
||||
"builder_confidence": dom_evidence.builder_confidence,
|
||||
"dom_slop_score": dom_evidence.score,
|
||||
"bucket": dom_evidence.bucket,
|
||||
"top_signals": [signal.to_dict() for signal in dom_evidence.signals[:6]],
|
||||
}
|
||||
if dom_evidence is not None
|
||||
else {"detected_builder": None, "builder_confidence": 0.0, "dom_slop_score": 0.0, "bucket": "none", "top_signals": []}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def fallback_report(payload: dict[str, Any]) -> str:
|
||||
repo = payload["repo_scores"]
|
||||
lines: list[str] = [
|
||||
"# AI Usage Classification Report",
|
||||
"",
|
||||
f"Source: `{payload['source_url']}` ({payload['source_kind']})",
|
||||
"",
|
||||
"## Verdict",
|
||||
"",
|
||||
f"Category **{repo['category']}** with grade **{repo['grade']}** "
|
||||
f"(slop score {repo['slop_score']}/100, confidence {repo['confidence']}).",
|
||||
]
|
||||
dom_evidence = payload.get("dom_evidence") or {}
|
||||
if dom_evidence.get("detected_builder"):
|
||||
lines.append(
|
||||
f"Detected: built with {dom_evidence['detected_builder']} (rendered-page fingerprint, "
|
||||
f"confidence {dom_evidence.get('builder_confidence', 0.0):.0%})."
|
||||
)
|
||||
lines.extend([
|
||||
f"Estimated composition: **{repo['human_percent']}% human**, **{repo['ai_percent']}% AI**.",
|
||||
"",
|
||||
"## Scores",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"| --- | --- |",
|
||||
f"| Grade | {repo['grade']} |",
|
||||
f"| Slop score | {repo['slop_score']} / 100 |",
|
||||
f"| Origin score | {repo['origin_score']} / 100 |",
|
||||
f"| Quality deficit | {repo['quality_deficit']} / 100 |",
|
||||
f"| Human share | {repo['human_percent']}% |",
|
||||
f"| AI share | {repo['ai_percent']}% |",
|
||||
f"| Files scored | {repo['files_scored']} |",
|
||||
f"| Files excluded | {repo['excluded_files']} |",
|
||||
f"| Strong signals | {repo['strong_signals']} |",
|
||||
f"| Medium signals | {repo['medium_signals']} |",
|
||||
"",
|
||||
"## Notable Files",
|
||||
"",
|
||||
])
|
||||
for entry in payload["worst_files"]:
|
||||
lines.append(
|
||||
f"- `{entry['path']}`: {entry['category']}, origin {entry['origin']}, quality deficit {entry['quality_deficit']}"
|
||||
)
|
||||
for signal in entry["signals"]:
|
||||
lines.append(f" - [{signal['severity']}] {signal['title']} (line {signal['line']})")
|
||||
template = payload.get("template_provenance", {})
|
||||
if template.get("markers"):
|
||||
lines.extend(["", "## Template Provenance", ""])
|
||||
lines.append(
|
||||
f"Starter template evidence score **{template.get('score')}/100**. The authenticity grade is "
|
||||
"reduced accordingly: an unmodified starter ships its scaffold's defaults rather than original work."
|
||||
)
|
||||
for marker in template["markers"]:
|
||||
lines.append(f"- {marker}")
|
||||
if payload["ai_verdicts"]:
|
||||
lines.extend(["", "## AI Review Findings", ""])
|
||||
for verdict in payload["ai_verdicts"]:
|
||||
lines.append(f"- `{verdict['path']}`: {verdict['category']} ({verdict['ai_probability']}% AI). {verdict['reasoning']}")
|
||||
image_review = payload.get("image_review", {})
|
||||
if payload.get("image_verdicts"):
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Image Review",
|
||||
"",
|
||||
f"Reviewed {image_review.get('count', 0)} images; "
|
||||
f"{image_review.get('ai_generated_count', 0)} appear AI-generated "
|
||||
f"(mean AI likelihood {image_review.get('mean_ai_probability', 0)}%, image grade {image_review.get('grade', 'n/a')}).",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for verdict in payload["image_verdicts"]:
|
||||
tells = f" Tells: {', '.join(verdict['tells'])}." if verdict["tells"] else ""
|
||||
lines.append(
|
||||
f"- `{verdict['path']}` ({verdict['image_kind']}): grade {verdict['grade']}, {verdict['verdict']} "
|
||||
f"({verdict['ai_probability']}% AI). {verdict['description']}{tells}"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Caveats",
|
||||
"",
|
||||
"No detector is definitive. This report combines weighted static signals with an AI review pass "
|
||||
"and should be read as calibrated guidance rather than proof of provenance.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def generate_report(
|
||||
llm: LlmClient,
|
||||
source_url: str,
|
||||
source_kind: str,
|
||||
scores: RepoScores,
|
||||
files: list[FileScore],
|
||||
verdicts: list[AiVerdict],
|
||||
excluded_count: int,
|
||||
image_verdicts: list[ImageVerdict],
|
||||
image_stats: dict[str, Any],
|
||||
template: TemplateEvidence,
|
||||
dom_evidence: DomEvidence | None,
|
||||
) -> tuple[str, str]:
|
||||
payload = _summary_payload(
|
||||
source_url,
|
||||
source_kind,
|
||||
scores,
|
||||
files,
|
||||
verdicts,
|
||||
excluded_count,
|
||||
image_verdicts,
|
||||
image_stats,
|
||||
template,
|
||||
dom_evidence,
|
||||
)
|
||||
user = "Write the final report for this classification data:\n" + json.dumps(payload, indent=2)[:24000]
|
||||
try:
|
||||
markdown = await llm.complete(REPORT_SYSTEM_PROMPT, user)
|
||||
if "# " not in markdown:
|
||||
raise LlmUnavailableError("report missing markdown structure")
|
||||
logger.info("Report generated via %s (%d chars)", llm.active_backend_name, len(markdown))
|
||||
return markdown.strip(), llm.active_backend_name
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("LLM report generation failed, using deterministic fallback: %s", error)
|
||||
return fallback_report(payload), "static-fallback"
|
||||
@@ -0,0 +1,215 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError, extract_json_object
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
IMAGE_CONCURRENCY,
|
||||
IMAGE_EXTENSIONS,
|
||||
IMAGE_MAX_BYTES,
|
||||
IMAGE_MAX_COUNT,
|
||||
IMAGE_MIN_BYTES,
|
||||
IMAGE_MIN_DIMENSION,
|
||||
THUMBNAIL_MAX_DIMENSION,
|
||||
THUMBNAIL_QUALITY,
|
||||
VISION_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VISION_PROMPT: str = (
|
||||
"You are a forensic image analyst deciding whether an image is AI-generated (or heavily AI-edited) or a genuine "
|
||||
"human-made photograph, illustration or screenshot. Examine, in detail: hands and fingers, faces and teeth, skin "
|
||||
"texture (waxy or plastic over-smoothing), eyes and reflections, text and lettering (garbled or nonsensical), "
|
||||
"lighting and shadow direction consistency, background warping and impossible geometry, repeated or cloned patterns, "
|
||||
"over-saturated or airbrushed rendering, and the 'too perfect' mathematically smooth look typical of diffusion models. "
|
||||
"Screenshots, logos, diagrams, icons and real product photos are usually not AI-generated; judge accordingly and do "
|
||||
"not call an ordinary screenshot AI art. Respond with a single JSON object only, using exactly these keys: "
|
||||
'{"ai_probability": <integer 0-100, likelihood the image is AI-generated>, '
|
||||
'"verdict": <one of "ai-generated"|"likely-ai"|"uncertain"|"likely-human"|"human-made">, '
|
||||
'"image_kind": <short phrase, e.g. "photo of a person", "screenshot", "logo", "illustration">, '
|
||||
'"tells": [<up to five short strings naming concrete artifacts or reasons>], '
|
||||
'"description": <two to four factual sentences describing the image and the evidence>}'
|
||||
)
|
||||
|
||||
VERDICT_VALUES: frozenset[str] = frozenset(
|
||||
{"ai-generated", "likely-ai", "uncertain", "likely-human", "human-made"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageVerdict:
|
||||
relative: str
|
||||
ai_probability: float
|
||||
grade: str
|
||||
verdict: str
|
||||
image_kind: str
|
||||
tells: list[str]
|
||||
description: str
|
||||
|
||||
|
||||
def _grade_for(ai_probability: float) -> str:
|
||||
if ai_probability <= 15:
|
||||
return "A"
|
||||
if ai_probability <= 35:
|
||||
return "B"
|
||||
if ai_probability <= 55:
|
||||
return "C"
|
||||
if ai_probability <= 75:
|
||||
return "D"
|
||||
return "F"
|
||||
|
||||
|
||||
def _eligible(path: Path) -> bool:
|
||||
if path.suffix.lower() not in IMAGE_EXTENSIONS:
|
||||
return False
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return False
|
||||
if size < IMAGE_MIN_BYTES or size > IMAGE_MAX_BYTES:
|
||||
return False
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
width, height = image.size
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.debug("Skipping unreadable image %s: %s", path, error)
|
||||
return False
|
||||
if width < IMAGE_MIN_DIMENSION or height < IMAGE_MIN_DIMENSION:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def collect_images(workspace: Path, limit: int = IMAGE_MAX_COUNT) -> list[Path]:
|
||||
candidates = [
|
||||
path
|
||||
for path in sorted(workspace.rglob("*"))
|
||||
if path.is_file() and not path.is_symlink() and _eligible(path)
|
||||
]
|
||||
if len(candidates) <= limit:
|
||||
logger.info("Collected %d images for vision analysis", len(candidates))
|
||||
return candidates
|
||||
ranked = sorted(
|
||||
candidates,
|
||||
key=lambda path: hashlib.sha256(str(path.relative_to(workspace)).encode("utf-8")).hexdigest(),
|
||||
)
|
||||
sample = sorted(ranked[:limit])
|
||||
logger.info("Sampled %d of %d images deterministically for vision analysis", len(sample), len(candidates))
|
||||
return sample
|
||||
|
||||
|
||||
def _to_data_url(path: Path) -> Optional[str]:
|
||||
mime_type = mimetypes.guess_type(path.name)[0]
|
||||
if not mime_type or not mime_type.startswith("image/"):
|
||||
return None
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as error:
|
||||
logger.warning("Image read failed for %s: %s", path, error)
|
||||
return None
|
||||
return f"data:{mime_type};base64,{base64.b64encode(raw).decode('utf-8')}"
|
||||
|
||||
|
||||
async def classify_image(llm: LlmClient, workspace: Path, path: Path) -> Optional[ImageVerdict]:
|
||||
relative = str(path.relative_to(workspace))
|
||||
data_url = await asyncio.to_thread(_to_data_url, path)
|
||||
if data_url is None:
|
||||
return None
|
||||
try:
|
||||
answer = await llm.describe_image(VISION_PROMPT, data_url, VISION_TIMEOUT_SECONDS)
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("Vision unavailable for %s: %s", relative, error)
|
||||
return None
|
||||
parsed = extract_json_object(answer)
|
||||
if parsed is None:
|
||||
logger.warning("Vision returned unparseable output for %s", relative)
|
||||
return None
|
||||
probability = parsed.get("ai_probability")
|
||||
if not isinstance(probability, (int, float)) or isinstance(probability, bool):
|
||||
probability = 50.0
|
||||
probability = float(min(100.0, max(0.0, probability)))
|
||||
verdict = str(parsed.get("verdict", "uncertain")).strip().lower()
|
||||
if verdict not in VERDICT_VALUES:
|
||||
verdict = "uncertain"
|
||||
tells_field = parsed.get("tells")
|
||||
tells = [str(item)[:120] for item in tells_field[:5]] if isinstance(tells_field, list) else []
|
||||
return ImageVerdict(
|
||||
relative=relative,
|
||||
ai_probability=probability,
|
||||
grade=_grade_for(probability),
|
||||
verdict=verdict,
|
||||
image_kind=str(parsed.get("image_kind", "image"))[:80],
|
||||
tells=tells,
|
||||
description=str(parsed.get("description", ""))[:1000],
|
||||
)
|
||||
|
||||
|
||||
async def analyze_images(
|
||||
llm: LlmClient,
|
||||
workspace: Path,
|
||||
images: list[Path],
|
||||
) -> list[ImageVerdict]:
|
||||
semaphore = asyncio.Semaphore(IMAGE_CONCURRENCY)
|
||||
|
||||
async def worker(path: Path) -> Optional[ImageVerdict]:
|
||||
async with semaphore:
|
||||
return await classify_image(llm, workspace, path)
|
||||
|
||||
results = await asyncio.gather(*(worker(path) for path in images))
|
||||
verdicts = [result for result in results if result is not None]
|
||||
logger.info("Vision analysis produced %d verdicts from %d images", len(verdicts), len(images))
|
||||
return verdicts
|
||||
|
||||
|
||||
def make_thumbnail(source: Path, media_dir: Path, relative: str) -> Optional[str]:
|
||||
name = f"{hashlib.sha1(relative.encode('utf-8')).hexdigest()[:16]}.webp"
|
||||
try:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(source) as image:
|
||||
image.thumbnail((THUMBNAIL_MAX_DIMENSION, THUMBNAIL_MAX_DIMENSION))
|
||||
if image.mode not in ("RGB", "RGBA"):
|
||||
image = image.convert("RGBA")
|
||||
image.save(media_dir / name, format="WEBP", quality=THUMBNAIL_QUALITY)
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.warning("Thumbnail failed for %s: %s", relative, error)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def persist_screenshot(raw_bytes: bytes, media_dir: Path, relative: str) -> Optional[str]:
|
||||
name = f"{hashlib.sha1(relative.encode('utf-8')).hexdigest()[:16]}.webp"
|
||||
try:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(io.BytesIO(raw_bytes)) as image:
|
||||
image.thumbnail((THUMBNAIL_MAX_DIMENSION, THUMBNAIL_MAX_DIMENSION))
|
||||
if image.mode not in ("RGB", "RGBA"):
|
||||
image = image.convert("RGBA")
|
||||
image.save(media_dir / name, format="WEBP", quality=THUMBNAIL_QUALITY)
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.warning("Screenshot persist failed for %s: %s", relative, error)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def image_summary(verdicts: list[ImageVerdict]) -> dict[str, float | int | str]:
|
||||
if not verdicts:
|
||||
return {"count": 0, "mean_ai_probability": 0.0, "grade": "n/a", "ai_generated_count": 0}
|
||||
mean = sum(verdict.ai_probability for verdict in verdicts) / len(verdicts)
|
||||
ai_generated = sum(1 for verdict in verdicts if verdict.ai_probability >= 60.0)
|
||||
return {
|
||||
"count": len(verdicts),
|
||||
"mean_ai_probability": round(mean, 1),
|
||||
"grade": _grade_for(mean),
|
||||
"ai_generated_count": ai_generated,
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,74 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_ORIGIN, SEVERITY_MEDIUM, Signal
|
||||
|
||||
MIN_FILES_FOR_BASELINE: int = 8
|
||||
DEVIATION_Z_THRESHOLD: float = 2.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoBaselines:
|
||||
file_count: int
|
||||
indent_variance_mean: float
|
||||
indent_variance_std: float
|
||||
comment_ratio_mean: float
|
||||
comment_ratio_std: float
|
||||
blank_ratio_mean: float
|
||||
blank_ratio_std: float
|
||||
|
||||
|
||||
def _mean_std(values: list[float]) -> tuple[float, float]:
|
||||
if not values:
|
||||
return 0.0, 0.0
|
||||
mean = sum(values) / len(values)
|
||||
variance = sum((value - mean) ** 2 for value in values) / len(values)
|
||||
return mean, math.sqrt(variance)
|
||||
|
||||
|
||||
def compute_baselines(metrics_by_file: dict[str, FileMetrics]) -> RepoBaselines:
|
||||
indent = [metrics.indent_variance for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
comment = [metrics.comment_ratio for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
blank = [metrics.blank_ratio for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
indent_mean, indent_std = _mean_std(indent)
|
||||
comment_mean, comment_std = _mean_std(comment)
|
||||
blank_mean, blank_std = _mean_std(blank)
|
||||
return RepoBaselines(
|
||||
file_count=len(indent),
|
||||
indent_variance_mean=indent_mean,
|
||||
indent_variance_std=indent_std,
|
||||
comment_ratio_mean=comment_mean,
|
||||
comment_ratio_std=comment_std,
|
||||
blank_ratio_mean=blank_mean,
|
||||
blank_ratio_std=blank_std,
|
||||
)
|
||||
|
||||
|
||||
def deviation_signals(relative: str, metrics: FileMetrics, baselines: RepoBaselines) -> list[Signal]:
|
||||
if baselines.file_count < MIN_FILES_FOR_BASELINE or metrics.sloc <= 10:
|
||||
return []
|
||||
scores: list[float] = []
|
||||
if baselines.indent_variance_std > 0.01:
|
||||
scores.append((baselines.indent_variance_mean - metrics.indent_variance) / baselines.indent_variance_std)
|
||||
if baselines.comment_ratio_std > 0.01:
|
||||
scores.append((metrics.comment_ratio - baselines.comment_ratio_mean) / baselines.comment_ratio_std)
|
||||
if not scores:
|
||||
return []
|
||||
composite = sum(scores) / len(scores)
|
||||
if composite >= DEVIATION_Z_THRESHOLD:
|
||||
return [
|
||||
Signal(
|
||||
code="CONVENTION_DEVIATION",
|
||||
title=f"File is markedly cleaner and more regular than repo baseline (z={composite:.1f})",
|
||||
severity=SEVERITY_MEDIUM,
|
||||
axis=AXIS_ORIGIN,
|
||||
weight=3.0,
|
||||
line=1,
|
||||
evidence=f"indent deviation {metrics.indent_variance:.2f} vs repo mean {baselines.indent_variance_mean:.2f}",
|
||||
)
|
||||
]
|
||||
return []
|
||||
@@ -0,0 +1,2 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_QUALITY, SEVERITY_WEAK, Signal
|
||||
|
||||
GENERIC_ALT_PATTERN = re.compile(r"^(image of|photo of|picture of)\b", re.IGNORECASE)
|
||||
DUPLICATE_ALT_THRESHOLD: int = 3
|
||||
EMPTY_ALT_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _detect_generic_alt_text(images: list[dict]) -> Signal | None:
|
||||
empty_count = 0
|
||||
for image in images:
|
||||
alt = str(image.get("alt", "")).strip() if isinstance(image, dict) else ""
|
||||
if not alt:
|
||||
empty_count += 1
|
||||
continue
|
||||
if GENERIC_ALT_PATTERN.match(alt):
|
||||
return Signal(
|
||||
"GENERIC_ALT_TEXT",
|
||||
"Generic placeholder-style alt text on an image",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
alt[:100],
|
||||
)
|
||||
if empty_count >= EMPTY_ALT_THRESHOLD and len(images) >= EMPTY_ALT_THRESHOLD:
|
||||
return Signal(
|
||||
"GENERIC_ALT_TEXT",
|
||||
f"{empty_count} images with empty alt text",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
f"{empty_count} empty alt attributes",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_duplicate_alt_text(images: list[dict]) -> Signal | None:
|
||||
counts: dict[str, int] = {}
|
||||
for image in images:
|
||||
alt = str(image.get("alt", "")).strip() if isinstance(image, dict) else ""
|
||||
if alt:
|
||||
counts[alt] = counts.get(alt, 0) + 1
|
||||
for alt, count in counts.items():
|
||||
if count >= DUPLICATE_ALT_THRESHOLD:
|
||||
return Signal(
|
||||
"DUPLICATE_ALT_TEXT",
|
||||
f"{count} images share the exact same alt text",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
alt[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_skipped_heading_level(headings: list[dict]) -> Signal | None:
|
||||
levels = [
|
||||
heading.get("level")
|
||||
for heading in headings
|
||||
if isinstance(heading, dict) and isinstance(heading.get("level"), int)
|
||||
]
|
||||
if len(levels) < 2 or levels[0] != 1:
|
||||
return None
|
||||
next_level = levels[1]
|
||||
if isinstance(next_level, int) and next_level > 2:
|
||||
return Signal(
|
||||
"SKIPPED_HEADING_LEVEL",
|
||||
f"H1 followed directly by H{next_level} with no H2 in between",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
0.5,
|
||||
0,
|
||||
f"H1 -> H{next_level}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_accessibility_signals(page: DomPageContext) -> list[Signal]:
|
||||
images = page.dom.get("images", []) or []
|
||||
headings = page.dom.get("headings", []) or []
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_generic_alt_text(images),
|
||||
_detect_duplicate_alt_text(images),
|
||||
_detect_skipped_heading_level(headings),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@@ -0,0 +1,58 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, DomSiteContext
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.builders import detected_builder
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.registry import run_dom_checks, run_dom_site_checks
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_FACTORS, Signal
|
||||
|
||||
DOM_SATURATION: float = 16.0
|
||||
DOM_FINDINGS_CAP: int = 14
|
||||
|
||||
BUCKET_NONE: str = "none"
|
||||
BUCKET_LIGHT: str = "light"
|
||||
BUCKET_MODERATE: str = "moderate"
|
||||
BUCKET_HEAVY: str = "heavy"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomEvidence:
|
||||
score: float
|
||||
bucket: str
|
||||
signals: list[Signal] = field(default_factory=list)
|
||||
detected_builder: str | None = None
|
||||
builder_confidence: float = 0.0
|
||||
|
||||
|
||||
def _bucket_for(score: float) -> str:
|
||||
if score < 10.0:
|
||||
return BUCKET_NONE
|
||||
if score < 35.0:
|
||||
return BUCKET_LIGHT
|
||||
if score < 65.0:
|
||||
return BUCKET_MODERATE
|
||||
return BUCKET_HEAVY
|
||||
|
||||
|
||||
def aggregate_dom_evidence(pages: list[DomPageContext]) -> DomEvidence:
|
||||
if not pages:
|
||||
return DomEvidence(score=0.0, bucket=BUCKET_NONE, signals=[], detected_builder=None, builder_confidence=0.0)
|
||||
signals: list[Signal] = []
|
||||
for page in pages:
|
||||
signals.extend(run_dom_checks(page)[:DOM_FINDINGS_CAP])
|
||||
site = DomSiteContext(pages=pages, root_host=urlparse(pages[0].url).hostname or "")
|
||||
signals.extend(run_dom_site_checks(site))
|
||||
weighted = sum(signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3) for signal in signals)
|
||||
score = round(100.0 * (1.0 - math.exp(-weighted / DOM_SATURATION)), 1)
|
||||
builder_name, builder_confidence = detected_builder(pages)
|
||||
return DomEvidence(
|
||||
score=score,
|
||||
bucket=_bucket_for(score),
|
||||
signals=signals,
|
||||
detected_builder=builder_name,
|
||||
builder_confidence=builder_confidence,
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Signal
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomPageContext:
|
||||
url: str
|
||||
dom: dict = field(default_factory=dict)
|
||||
console_warnings: list[str] = field(default_factory=list)
|
||||
console_errors: list[str] = field(default_factory=list)
|
||||
response_headers: dict[str, str] = field(default_factory=dict)
|
||||
resource_hosts: list[str] = field(default_factory=list)
|
||||
screenshot_bytes: bytes | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomSiteContext:
|
||||
pages: list[DomPageContext] = field(default_factory=list)
|
||||
root_host: str = ""
|
||||
|
||||
|
||||
DomPageDetector = Callable[[DomPageContext], list[Signal]]
|
||||
DomSiteDetector = Callable[[DomSiteContext], list[Signal]]
|
||||
|
||||
DOM_CHECKS: list[DomPageDetector] = []
|
||||
DOM_SITE_CHECKS: list[DomSiteDetector] = []
|
||||
|
||||
|
||||
def dom_check(func: DomPageDetector) -> DomPageDetector:
|
||||
DOM_CHECKS.append(func)
|
||||
return func
|
||||
|
||||
|
||||
def dom_site_check(func: DomSiteDetector) -> DomSiteDetector:
|
||||
DOM_SITE_CHECKS.append(func)
|
||||
return func
|
||||
@@ -0,0 +1,204 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CLAUDE_ARTIFACT_BADGE_KEY: str = 'iframe[src*="claudeusercontent.com"]'
|
||||
|
||||
FRAMER_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"events.framer.com", "framerusercontent.com", "framercdn.com", "framercanvas.com"}
|
||||
)
|
||||
WIX_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"static.wixstatic.com", "static.parastorage.com", "siteassets.parastorage.com"}
|
||||
)
|
||||
WEBFLOW_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"assets-global.website-files.com", "uploads-ssl.webflow.com", "assets.website-files.com"}
|
||||
)
|
||||
SQUARESPACE_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"static1.squarespace.com", "static.squarespace.com", "images.squarespace-cdn.com"}
|
||||
)
|
||||
HOSTING_HEADER_KEYS: frozenset[str] = frozenset({"x-vercel-id", "x-nf-request-id"})
|
||||
|
||||
|
||||
def _host_of(url: str) -> str:
|
||||
return (urlparse(url).hostname or "").lower()
|
||||
|
||||
|
||||
def _resource_text(page: DomPageContext) -> str:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
links = page.dom.get("links", []) or []
|
||||
hosts = page.resource_hosts or []
|
||||
return " ".join(str(item) for item in (*scripts, *links, *hosts)).lower()
|
||||
|
||||
|
||||
def _generator(page: DomPageContext) -> str:
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
return str(meta.get("generator") or "")
|
||||
|
||||
|
||||
def _class_names(page: DomPageContext) -> list[str]:
|
||||
return list((page.dom.get("classNames") or {}).keys())
|
||||
|
||||
|
||||
def _badge(page: DomPageContext, selector: str) -> bool:
|
||||
return bool((page.dom.get("badgeHits") or {}).get(selector))
|
||||
|
||||
|
||||
def _detect_lovable(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, "#lovable-badge")
|
||||
resource_hit = "cdn.gpteng.co" in _resource_text(page) or "gptengineer.js" in _resource_text(page)
|
||||
generator_hit = "lovable" in _generator(page).lower()
|
||||
host_hit = _host_of(page.url).endswith(".lovable.app")
|
||||
if not (badge or resource_hit or generator_hit or host_hit):
|
||||
return None
|
||||
evidence = "lovable-badge" if badge else "cdn.gpteng.co" if resource_hit else "generator meta" if generator_hit else "lovable.app host"
|
||||
return Signal("AI_BUILDER_LOVABLE", "Lovable / GPT Engineer builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_bolt(page: DomPageContext) -> Signal | None:
|
||||
haystack = _resource_text(page) + " " + page.url.lower()
|
||||
if "bolt.new" not in haystack:
|
||||
return None
|
||||
return Signal("AI_BUILDER_BOLT", "Bolt.new builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "bolt.new")
|
||||
|
||||
|
||||
def _detect_replit(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, ".replit-badge")
|
||||
host = _host_of(page.url)
|
||||
host_hit = host.endswith(".replit.app") or host.endswith(".repl.co")
|
||||
if not (badge or host_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_REPLIT", "Replit builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "replit-badge" if badge else host)
|
||||
|
||||
|
||||
def _detect_base44(page: DomPageContext) -> Signal | None:
|
||||
if not _badge(page, "#base44-badge"):
|
||||
return None
|
||||
return Signal("AI_BUILDER_BASE44", "Base44 builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "base44-badge")
|
||||
|
||||
|
||||
def _detect_framer(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, "#__framer-badge-container")
|
||||
generator_hit = "framer" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & FRAMER_RESOURCE_HOSTS)
|
||||
host_hit = _host_of(page.url).endswith(".framer.website")
|
||||
if not (badge or generator_hit or resource_hit or host_hit):
|
||||
return None
|
||||
evidence = "framer-badge" if badge else "generator meta" if generator_hit else "framer resource host" if resource_hit else "framer.website host"
|
||||
return Signal("AI_BUILDER_FRAMER", "Framer builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_wix(page: DomPageContext) -> Signal | None:
|
||||
generator_hit = "wix.com website builder" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & WIX_RESOURCE_HOSTS)
|
||||
class_hit = any("wix-bolt" in name.lower() for name in _class_names(page))
|
||||
if not (generator_hit or resource_hit or class_hit):
|
||||
return None
|
||||
evidence = "generator meta" if generator_hit else "wix resource host" if resource_hit else "wix-bolt class"
|
||||
return Signal("AI_BUILDER_WIX", "Wix builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_webflow(page: DomPageContext) -> Signal | None:
|
||||
generator_hit = "webflow" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & WEBFLOW_RESOURCE_HOSTS)
|
||||
if not (generator_hit or resource_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_WEBFLOW", "Webflow builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "generator meta" if generator_hit else "webflow resource host")
|
||||
|
||||
|
||||
def _detect_godaddy(page: DomPageContext) -> Signal | None:
|
||||
if "go daddy website builder" not in _generator(page).lower():
|
||||
return None
|
||||
return Signal("AI_BUILDER_GODADDY", "GoDaddy Website Builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "generator meta")
|
||||
|
||||
|
||||
def _detect_squarespace(page: DomPageContext) -> Signal | None:
|
||||
resource_hit = bool(set(page.resource_hosts or []) & SQUARESPACE_RESOURCE_HOSTS)
|
||||
class_hit = any(name.lower().startswith("sqs-block") or name.lower().startswith("yui3-") for name in _class_names(page))
|
||||
if not (resource_hit or class_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_SQUARESPACE", "Squarespace builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "squarespace resource host" if resource_hit else "sqs-block class")
|
||||
|
||||
|
||||
def _detect_claude_artifact(page: DomPageContext) -> Signal | None:
|
||||
if not _badge(page, CLAUDE_ARTIFACT_BADGE_KEY):
|
||||
return None
|
||||
return Signal("AI_BUILDER_CLAUDE_ARTIFACT", "Claude Artifacts iframe embed detected", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "claudeusercontent.com iframe")
|
||||
|
||||
|
||||
def _detect_shadcn_radix_cluster(page: DomPageContext) -> Signal | None:
|
||||
indicators: list[str] = []
|
||||
if _badge(page, "[data-radix-root]"):
|
||||
indicators.append("data-radix-root")
|
||||
if any("radix" in name.lower() for name in _class_names(page)):
|
||||
indicators.append("radix class token")
|
||||
if "lucide" in _resource_text(page):
|
||||
indicators.append("lucide icons")
|
||||
if len(indicators) < 2:
|
||||
return None
|
||||
severity = SEVERITY_STRONG if len(indicators) >= 3 else SEVERITY_MEDIUM
|
||||
return Signal("SHADCN_RADIX_CLUSTER", f"shadcn/ui and Radix primitives cluster ({len(indicators)} indicators)", severity, AXIS_ORIGIN, 3.0, 0, ", ".join(indicators))
|
||||
|
||||
|
||||
def _detect_hosting_subdomain(page: DomPageContext) -> Signal | None:
|
||||
host = _host_of(page.url)
|
||||
host_hit = host.endswith(".vercel.app") or host.endswith(".netlify.app") or host.endswith(".databutton.app")
|
||||
headers = {key.lower() for key in (page.response_headers or {}).keys()}
|
||||
header_hit = bool(headers & HOSTING_HEADER_KEYS)
|
||||
if not (host_hit or header_hit):
|
||||
return None
|
||||
evidence = host if host_hit else ", ".join(sorted(headers & HOSTING_HEADER_KEYS))
|
||||
return Signal("BUILDER_HOSTING_SUBDOMAIN", "Hosted on a default builder/PaaS subdomain", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, 0, evidence)
|
||||
|
||||
|
||||
_VENDOR_DETECTORS: tuple[tuple[Callable[[DomPageContext], Signal | None], str], ...] = (
|
||||
(_detect_lovable, "Lovable"),
|
||||
(_detect_bolt, "Bolt.new"),
|
||||
(_detect_replit, "Replit"),
|
||||
(_detect_base44, "Base44"),
|
||||
(_detect_framer, "Framer"),
|
||||
(_detect_wix, "Wix"),
|
||||
(_detect_webflow, "Webflow"),
|
||||
(_detect_godaddy, "GoDaddy"),
|
||||
(_detect_squarespace, "Squarespace"),
|
||||
(_detect_claude_artifact, "Claude Artifacts"),
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_builders(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for detector, _label in _VENDOR_DETECTORS:
|
||||
signal = detector(page)
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
cluster = _detect_shadcn_radix_cluster(page)
|
||||
if cluster is not None:
|
||||
findings.append(cluster)
|
||||
hosting = _detect_hosting_subdomain(page)
|
||||
if hosting is not None:
|
||||
findings.append(hosting)
|
||||
return findings
|
||||
|
||||
|
||||
def detected_builder(pages: list[DomPageContext]) -> tuple[str | None, float]:
|
||||
cluster_hit = False
|
||||
for page in pages:
|
||||
for detector, label in _VENDOR_DETECTORS:
|
||||
if detector(page) is not None:
|
||||
return label, 1.0
|
||||
if _detect_shadcn_radix_cluster(page) is not None:
|
||||
cluster_hit = True
|
||||
if cluster_hit:
|
||||
return "shadcn/ui + Radix", 0.5
|
||||
return None, 0.0
|
||||
@@ -0,0 +1,185 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HASHED_SEGMENT_PATTERN = re.compile(r"[.-][0-9a-f]{6,}\.js", re.IGNORECASE)
|
||||
UNHASHED_BUNDLE_PATTERN = re.compile(r"/(main|bundle|index)\.js(\?|$)", re.IGNORECASE)
|
||||
PLACEHOLDER_TEXT_PATTERN = re.compile(
|
||||
r"lorem ipsum|coming soon|your headline here|add your description|placeholder", re.IGNORECASE
|
||||
)
|
||||
PLACEHOLDER_IMAGE_HOSTS: tuple[str, ...] = ("via.placeholder.com", "placehold.co", "picsum.photos")
|
||||
TAILWIND_PRODUCTION_WARNING: str = "cdn.tailwindcss.com should not be used in production"
|
||||
REACT_DEV_WARNING_MARKERS: tuple[str, ...] = ("development build of React", "Each child in a list should have a unique")
|
||||
|
||||
|
||||
def _text_blob(page: DomPageContext) -> str:
|
||||
parts = [
|
||||
str(heading.get("text", ""))
|
||||
for heading in (page.dom.get("headings") or [])
|
||||
if isinstance(heading, dict) and heading.get("text")
|
||||
]
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
description = meta.get("description") if isinstance(meta, dict) else ""
|
||||
if description:
|
||||
parts.append(str(description))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _looks_unhashed(src: str) -> bool:
|
||||
if not src:
|
||||
return False
|
||||
if HASHED_SEGMENT_PATTERN.search(src):
|
||||
return False
|
||||
return bool(UNHASHED_BUNDLE_PATTERN.search(src))
|
||||
|
||||
|
||||
def _detect_tailwind_cdn(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
if not any("cdn.tailwindcss.com" in str(src) for src in scripts):
|
||||
return None
|
||||
console_text = " ".join(page.console_warnings + page.console_errors)
|
||||
if TAILWIND_PRODUCTION_WARNING in console_text:
|
||||
return Signal(
|
||||
"TAILWIND_CDN_DOM",
|
||||
"Tailwind CDN build flagged unsuitable for production by the browser console",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.5,
|
||||
0,
|
||||
"cdn.tailwindcss.com + production warning",
|
||||
)
|
||||
return Signal(
|
||||
"TAILWIND_CDN_DOM",
|
||||
"Tailwind loaded from the CDN build in the rendered page",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
"cdn.tailwindcss.com",
|
||||
)
|
||||
|
||||
|
||||
def _detect_cdn_react(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
for src in scripts:
|
||||
text = str(src)
|
||||
if "unpkg.com/react" in text or "esm.sh/react" in text or "esm.sh/tsx" in text:
|
||||
return Signal(
|
||||
"CDN_REACT_UNBUNDLED",
|
||||
"React loaded unbundled directly from a CDN",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
text[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_react_dev_warning(page: DomPageContext) -> Signal | None:
|
||||
console_text = " ".join(page.console_warnings + page.console_errors)
|
||||
if not any(marker in console_text for marker in REACT_DEV_WARNING_MARKERS):
|
||||
return None
|
||||
return Signal(
|
||||
"REACT_DEV_BUILD_WARNING",
|
||||
"React development-build or missing-key warnings present in the console",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
console_text[:150],
|
||||
)
|
||||
|
||||
|
||||
def _detect_unhashed_bundle(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
for src in scripts:
|
||||
if _looks_unhashed(str(src)):
|
||||
return Signal(
|
||||
"UNHASHED_BUNDLE_FILENAME",
|
||||
"Script bundle filename with no content-hash segment",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
0.5,
|
||||
0,
|
||||
str(src),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_placeholder_content(page: DomPageContext) -> Signal | None:
|
||||
match = PLACEHOLDER_TEXT_PATTERN.search(_text_blob(page))
|
||||
if not match:
|
||||
return None
|
||||
return Signal(
|
||||
"PLACEHOLDER_CONTENT_DOM",
|
||||
"Placeholder or unedited template copy left in the rendered page",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
0,
|
||||
match.group(0),
|
||||
)
|
||||
|
||||
|
||||
def _detect_placeholder_image_host(page: DomPageContext) -> Signal | None:
|
||||
images = page.dom.get("images", []) or []
|
||||
for image in images:
|
||||
if not isinstance(image, dict):
|
||||
continue
|
||||
src = str(image.get("src", ""))
|
||||
if any(host in src for host in PLACEHOLDER_IMAGE_HOSTS):
|
||||
return Signal(
|
||||
"PLACEHOLDER_IMAGE_HOST",
|
||||
"Image served from a generic placeholder image host",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
src,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_default_framework_title(page: DomPageContext) -> Signal | None:
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
title = str(meta.get("title", ""))
|
||||
if title == "Vite + React" or "Create Next App" in title:
|
||||
return Signal(
|
||||
"DEFAULT_FRAMEWORK_TITLE",
|
||||
"Unedited default framework document title",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
title,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_build_signals(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_tailwind_cdn(page),
|
||||
_detect_cdn_react(page),
|
||||
_detect_react_dev_warning(page),
|
||||
_detect_unhashed_bundle(page),
|
||||
_detect_placeholder_content(page),
|
||||
_detect_placeholder_image_host(page),
|
||||
_detect_default_framework_title(page),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@@ -0,0 +1,245 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import colorsys
|
||||
import io
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HEX_COLOR_PATTERN = re.compile(r"#[0-9a-fA-F]{6}\b")
|
||||
RGB_COLOR_PATTERN = re.compile(r"rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)")
|
||||
GRADIENT_PATTERN = re.compile(r"linear-gradient\([^)]*\)", re.IGNORECASE)
|
||||
|
||||
TAILWIND_DEFAULT_ACCENTS: frozenset[str] = frozenset(
|
||||
{"#6366f1", "#4f46e5", "#8b5cf6", "#a855f7", "#7c3aed", "#818cf8"}
|
||||
)
|
||||
SIGNATURE_GRADIENT_PATTERN = re.compile(r"#667eea.{0,80}#764ba2|#764ba2.{0,80}#667eea", re.IGNORECASE | re.DOTALL)
|
||||
OKLCH_PATTERN = re.compile(r"oklch\(\s*([\d.]+)", re.IGNORECASE)
|
||||
BOX_SHADOW_PURPLE_PATTERN = re.compile(
|
||||
r"(rgba?\(\s*(?:99|79|139|168|124|129)\s*,\s*\d+\s*,\s*\d+|#(?:6366f1|4f46e5|8b5cf6|a855f7|7c3aed|818cf8))"
|
||||
r"[\s\S]{0,60}?(\d+)px",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DARK_LUMINANCE_THRESHOLD: int = 40
|
||||
|
||||
|
||||
def _hue_from_hex(hex_code: str) -> float | None:
|
||||
try:
|
||||
r = int(hex_code[1:3], 16) / 255.0
|
||||
g = int(hex_code[3:5], 16) / 255.0
|
||||
b = int(hex_code[5:7], 16) / 255.0
|
||||
except ValueError:
|
||||
return None
|
||||
hue, _lightness, _saturation = colorsys.rgb_to_hls(r, g, b)
|
||||
return hue * 360.0
|
||||
|
||||
|
||||
def _hue_from_rgb(r: int, g: int, b: int) -> float:
|
||||
hue, _lightness, _saturation = colorsys.rgb_to_hls(r / 255.0, g / 255.0, b / 255.0)
|
||||
return hue * 360.0
|
||||
|
||||
|
||||
def _is_blue_or_purple(hue: float) -> bool:
|
||||
return 210.0 <= hue <= 300.0
|
||||
|
||||
|
||||
def _sample_text(sample: dict) -> str:
|
||||
return " ".join(
|
||||
str(sample.get(field, ""))
|
||||
for field in ("backgroundImage", "backgroundColor", "color", "boxShadow")
|
||||
)
|
||||
|
||||
|
||||
def _detect_gradient_hue_combo(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
gradient = GRADIENT_PATTERN.search(str(sample.get("backgroundImage", "")))
|
||||
if not gradient:
|
||||
continue
|
||||
text = gradient.group(0)
|
||||
hues: list[float] = []
|
||||
for match in HEX_COLOR_PATTERN.finditer(text):
|
||||
hue = _hue_from_hex(match.group(0))
|
||||
if hue is not None:
|
||||
hues.append(hue)
|
||||
for match in RGB_COLOR_PATTERN.finditer(text):
|
||||
hues.append(_hue_from_rgb(int(match.group(1)), int(match.group(2)), int(match.group(3))))
|
||||
if sum(1 for hue in hues if _is_blue_or_purple(hue)) >= 2:
|
||||
return Signal(
|
||||
"SIGNATURE_GRADIENT_HUE_COMBO",
|
||||
"Gradient stops both land in the default LLM blue/purple hue range",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
text[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_tailwind_accent(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
text = _sample_text(sample).lower()
|
||||
for accent in TAILWIND_DEFAULT_ACCENTS:
|
||||
if accent in text:
|
||||
return Signal(
|
||||
"TAILWIND_DEFAULT_ACCENT",
|
||||
"Literal Tailwind default accent color used verbatim",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
accent,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_signature_gradient(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = SIGNATURE_GRADIENT_PATTERN.search(_sample_text(sample))
|
||||
if match:
|
||||
return Signal(
|
||||
"SIGNATURE_GRADIENT",
|
||||
"Canonical LLM purple gradient #667eea to #764ba2",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
0,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_oklch_token(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = OKLCH_PATTERN.search(_sample_text(sample))
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
lightness = float(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if 0.55 <= lightness <= 0.75:
|
||||
return Signal(
|
||||
"OKLCH_SHADCN_TOKEN",
|
||||
"oklch() color token in the shadcn/ui default lightness band",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
match.group(0),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_purple_glow_shadow(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = BOX_SHADOW_PURPLE_PATTERN.search(str(sample.get("boxShadow", "")))
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
blur = int(match.group(2))
|
||||
except ValueError:
|
||||
continue
|
||||
if blur >= 40:
|
||||
return Signal(
|
||||
"PURPLE_GLOW_SHADOW",
|
||||
"Large-blur purple/blue glow box-shadow",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_dark_mode_default(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
if sample.get("tag") != "body":
|
||||
continue
|
||||
match = RGB_COLOR_PATTERN.search(str(sample.get("backgroundColor", "")))
|
||||
if not match:
|
||||
continue
|
||||
channels = [int(match.group(1)), int(match.group(2)), int(match.group(3))]
|
||||
if sum(channels) / 3.0 < DARK_LUMINANCE_THRESHOLD:
|
||||
return Signal(
|
||||
"DARK_MODE_DEFAULT",
|
||||
"Dark background color by default",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
str(sample.get("backgroundColor", "")),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _screenshot_hue_buckets(screenshot_bytes: bytes) -> dict[int, int]:
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(io.BytesIO(screenshot_bytes)).convert("RGB")
|
||||
image = image.resize((64, 64))
|
||||
buckets: dict[int, int] = {}
|
||||
for r, g, b in image.getdata():
|
||||
hue, lightness, saturation = colorsys.rgb_to_hls(r / 255.0, g / 255.0, b / 255.0)
|
||||
if saturation < 0.15 or lightness < 0.05 or lightness > 0.95:
|
||||
continue
|
||||
bucket = int((hue * 360.0) // 20) * 20
|
||||
buckets[bucket] = buckets.get(bucket, 0) + 1
|
||||
return buckets
|
||||
|
||||
|
||||
def _detect_screenshot_gradient_hero(screenshot_bytes: bytes | None) -> Signal | None:
|
||||
if not screenshot_bytes:
|
||||
return None
|
||||
try:
|
||||
buckets = _screenshot_hue_buckets(screenshot_bytes)
|
||||
except Exception:
|
||||
return None
|
||||
total = sum(buckets.values())
|
||||
if total <= 0:
|
||||
return None
|
||||
ranked = sorted(buckets.items(), key=lambda item: item[1], reverse=True)[:2]
|
||||
if len(ranked) < 2 or not all(_is_blue_or_purple(bucket) for bucket, _count in ranked):
|
||||
return None
|
||||
coverage = sum(count for _bucket, count in ranked) / total
|
||||
if coverage <= 0.3:
|
||||
return None
|
||||
return Signal(
|
||||
"SCREENSHOT_GRADIENT_HERO",
|
||||
"Screenshot dominated by a blue/purple hero gradient",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
f"{coverage:.0%} of sampled pixels in blue/purple hues",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_color_signals(page: DomPageContext) -> list[Signal]:
|
||||
samples = page.dom.get("colors", []) or []
|
||||
findings: list[Signal] = []
|
||||
for detector in (
|
||||
_detect_gradient_hue_combo,
|
||||
_detect_tailwind_accent,
|
||||
_detect_signature_gradient,
|
||||
_detect_oklch_token,
|
||||
_detect_purple_glow_shadow,
|
||||
_detect_dark_mode_default,
|
||||
):
|
||||
signal = detector(samples)
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
screenshot_signal = _detect_screenshot_gradient_hero(page.screenshot_bytes)
|
||||
if screenshot_signal is not None:
|
||||
findings.append(screenshot_signal)
|
||||
return findings
|
||||
@@ -0,0 +1,157 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CLICHE_PHRASES: tuple[str, ...] = (
|
||||
r"unlock (the )?(power|potential) of",
|
||||
r"unleash the power of",
|
||||
r"take your .{1,40} to the next level",
|
||||
r"elevate your",
|
||||
r"seamlessly integrat(e|es|ion)",
|
||||
r"revolutioniz(e|ing) the way",
|
||||
r"whether you('re| are) a .{1,30} or (a )?.{1,30}",
|
||||
r"delve into",
|
||||
r"dive into the world of",
|
||||
r"embark on a journey",
|
||||
r"push(ing)? the boundaries of",
|
||||
r"at the forefront of",
|
||||
r"game[- ]chang(er|ing)",
|
||||
r"pave the way for",
|
||||
r"bridging the gap between",
|
||||
r"navigate the complexities of",
|
||||
r"foster a culture of",
|
||||
r"harness the power of",
|
||||
r"cutting[- ]edge",
|
||||
r"future[- ]proof",
|
||||
r"world[- ]class",
|
||||
r"enterprise[- ]grade",
|
||||
r"streamline(d)?",
|
||||
r"empower(ing)?",
|
||||
r"supercharg(e|ed|ing)",
|
||||
)
|
||||
CLICHE_PATTERN = re.compile("|".join(CLICHE_PHRASES), re.IGNORECASE)
|
||||
|
||||
BUZZWORDS: tuple[str, ...] = ("delve", "crucial", "intricate", "nuanced", "myriad", "realm", "tapestry", "landscape")
|
||||
BUZZWORD_THRESHOLD: int = 3
|
||||
BUZZWORD_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
||||
re.compile(rf"\b{word}\b", re.IGNORECASE) for word in BUZZWORDS
|
||||
)
|
||||
|
||||
HEADING_EMOJI_PATTERN = re.compile(r"[\U0001F300-\U0001FAFF✅⭐✨\U0001F680]")
|
||||
|
||||
FAQ_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"^what is\b", re.IGNORECASE),
|
||||
re.compile(r"^how does .{1,30} work\??$", re.IGNORECASE),
|
||||
re.compile(r"^is (it|.{1,20}) secure\??$", re.IGNORECASE),
|
||||
re.compile(r"^can i cancel", re.IGNORECASE),
|
||||
re.compile(r"^do you offer a? ?(free trial|refund)", re.IGNORECASE),
|
||||
)
|
||||
FAQ_PATTERN_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _heading_texts(page: DomPageContext) -> list[str]:
|
||||
return [
|
||||
str(heading.get("text", ""))
|
||||
for heading in (page.dom.get("headings") or [])
|
||||
if isinstance(heading, dict) and heading.get("text")
|
||||
]
|
||||
|
||||
|
||||
def _text_blob(page: DomPageContext) -> str:
|
||||
parts = list(_heading_texts(page))
|
||||
for image in page.dom.get("images", []) or []:
|
||||
if isinstance(image, dict) and image.get("alt"):
|
||||
parts.append(str(image["alt"]))
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
description = meta.get("description") if isinstance(meta, dict) else ""
|
||||
if description:
|
||||
parts.append(str(description))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _detect_template_copy(blob: str) -> Signal | None:
|
||||
match = CLICHE_PATTERN.search(blob)
|
||||
if not match:
|
||||
return None
|
||||
return Signal(
|
||||
"TEMPLATE_COPY_DOM",
|
||||
"Stock AI landing-page cliche phrase in rendered copy",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
match.group(0)[:150],
|
||||
)
|
||||
|
||||
|
||||
def _detect_buzzword_cluster(blob: str) -> Signal | None:
|
||||
hits = [pattern.pattern for pattern in BUZZWORD_PATTERNS if pattern.search(blob)]
|
||||
if len(hits) < BUZZWORD_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"AI_BUZZWORD_CLUSTER",
|
||||
f"Elevated buzzword cluster ({len(hits)} distinct terms)",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
", ".join(hits),
|
||||
)
|
||||
|
||||
|
||||
def _detect_emoji_heading(page: DomPageContext) -> Signal | None:
|
||||
for text in _heading_texts(page):
|
||||
if HEADING_EMOJI_PATTERN.search(text):
|
||||
return Signal(
|
||||
"EMOJI_HEADING_DOM",
|
||||
"Emoji embedded inside a rendered heading",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
text[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_generic_faq_template(page: DomPageContext) -> Signal | None:
|
||||
texts = _heading_texts(page)
|
||||
matched = 0
|
||||
for pattern in FAQ_PATTERNS:
|
||||
if any(pattern.search(text.strip()) for text in texts):
|
||||
matched += 1
|
||||
if matched < FAQ_PATTERN_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"GENERIC_FAQ_TEMPLATE",
|
||||
f"Generic templated FAQ question set ({matched} canonical patterns matched)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
f"{matched} FAQ patterns matched",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_copy_signals(page: DomPageContext) -> list[Signal]:
|
||||
blob = _text_blob(page)
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_template_copy(blob),
|
||||
_detect_buzzword_cluster(blob),
|
||||
_detect_emoji_heading(page),
|
||||
_detect_generic_faq_template(page),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@@ -0,0 +1,104 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
LANDING_SECTION_TOKENS: tuple[str, ...] = ("hero", "features", "testimonials", "pricing", "faq", "cta")
|
||||
LANDING_SECTION_THRESHOLD: int = 4
|
||||
DEAD_ANCHOR_THRESHOLD: int = 5
|
||||
REPEATED_CARD_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _detect_landing_template(class_names: dict) -> Signal | None:
|
||||
matched = {
|
||||
token
|
||||
for token in LANDING_SECTION_TOKENS
|
||||
if any(token in name.lower() for name in class_names)
|
||||
}
|
||||
if len(matched) < LANDING_SECTION_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"LANDING_TEMPLATE_DOM",
|
||||
"Canonical hero-features-testimonials-pricing landing structure in the rendered DOM",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
", ".join(sorted(matched)),
|
||||
)
|
||||
|
||||
|
||||
def _detect_glassmorphic_navbar(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
tag = str(sample.get("tag", ""))
|
||||
if tag not in ("nav", "header"):
|
||||
continue
|
||||
blob = " ".join(
|
||||
str(sample.get(field, "")) for field in ("backgroundImage", "backgroundColor", "boxShadow")
|
||||
).lower()
|
||||
if "blur(" not in blob:
|
||||
continue
|
||||
if "rgba(" in blob or "hsla(" in blob:
|
||||
return Signal(
|
||||
"GLASSMORPHIC_NAVBAR",
|
||||
"Glassmorphic translucent-blur navigation bar",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
blob[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_repeated_card_class(class_names: dict) -> Signal | None:
|
||||
for name, count in class_names.items():
|
||||
if count >= REPEATED_CARD_THRESHOLD:
|
||||
return Signal(
|
||||
"REPEATED_CARD_CLASS",
|
||||
f"Class token repeated across {count} elements, a generated card-grid shape",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
f"{name} x{count}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_dead_anchor_links(dead_anchor_count: int) -> Signal | None:
|
||||
if dead_anchor_count < DEAD_ANCHOR_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"DEAD_ANCHOR_LINKS_DOM",
|
||||
f'{dead_anchor_count} rendered links pointing to href="#"',
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
f"{dead_anchor_count} placeholder links",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_layout_signals(page: DomPageContext) -> list[Signal]:
|
||||
class_names = page.dom.get("classNames", {}) or {}
|
||||
colors = page.dom.get("colors", []) or []
|
||||
dead_anchor_count = int(page.dom.get("deadAnchorCount", 0) or 0)
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_landing_template(class_names),
|
||||
_detect_glassmorphic_navbar(colors),
|
||||
_detect_repeated_card_class(class_names),
|
||||
_detect_dead_anchor_links(dead_anchor_count),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@@ -0,0 +1,66 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, DomSiteContext, dom_check, dom_site_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_QUALITY, SEVERITY_MEDIUM, SEVERITY_WEAK, Signal
|
||||
|
||||
|
||||
def _meta(page: DomPageContext) -> dict:
|
||||
meta = page.dom.get("meta", {})
|
||||
return meta if isinstance(meta, dict) else {}
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_metaseo_signals(page: DomPageContext) -> list[Signal]:
|
||||
meta = _meta(page)
|
||||
findings: list[Signal] = []
|
||||
if not str(meta.get("description") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_META_DESCRIPTION", "No meta description present", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 0, "")
|
||||
)
|
||||
if int(page.dom.get("jsonld", 0) or 0) == 0:
|
||||
findings.append(
|
||||
Signal("MISSING_STRUCTURED_DATA", "No JSON-LD structured data present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("ogImage") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_OG_IMAGE", "No og:image meta tag present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not meta.get("favicon"):
|
||||
findings.append(
|
||||
Signal("MISSING_FAVICON", "No favicon link present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("htmlLang") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_HTML_LANG", "No html lang attribute present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("canonical") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_CANONICAL", "No canonical link present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
@dom_site_check
|
||||
def detect_duplicate_meta_description(site: DomSiteContext) -> list[Signal]:
|
||||
if len(site.pages) < 2:
|
||||
return []
|
||||
counts: dict[str, int] = {}
|
||||
for page in site.pages:
|
||||
description = str(_meta(page).get("description") or "").strip()
|
||||
if description:
|
||||
counts[description] = counts.get(description, 0) + 1
|
||||
for description, count in counts.items():
|
||||
if count >= 2:
|
||||
return [
|
||||
Signal(
|
||||
"DUPLICATE_META_DESCRIPTION",
|
||||
"Identical meta description reused across multiple pages",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
description[:150],
|
||||
)
|
||||
]
|
||||
return []
|
||||
@@ -0,0 +1,35 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import (
|
||||
DOM_CHECKS,
|
||||
DOM_SITE_CHECKS,
|
||||
DomPageContext,
|
||||
DomSiteContext,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Signal
|
||||
from . import accessibility, builders, buildsignals, color, copy, layout, metaseo, typography
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_dom_checks(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for func in DOM_CHECKS:
|
||||
try:
|
||||
findings.extend(func(page))
|
||||
except Exception as error:
|
||||
logger.warning("DOM check %s failed for %s: %s", func.__name__, page.url, error)
|
||||
return findings
|
||||
|
||||
|
||||
def run_dom_site_checks(site: DomSiteContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for func in DOM_SITE_CHECKS:
|
||||
try:
|
||||
findings.extend(func(site))
|
||||
except Exception as error:
|
||||
logger.warning("DOM site check %s failed: %s", func.__name__, error)
|
||||
return findings
|
||||
@@ -0,0 +1,128 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DEFAULT_AI_FONTS: tuple[str, ...] = (
|
||||
"Inter",
|
||||
"Poppins",
|
||||
"Manrope",
|
||||
"Geist",
|
||||
"Space Grotesk",
|
||||
"DM Sans",
|
||||
"Plus Jakarta Sans",
|
||||
)
|
||||
DECORATIVE_MONOSPACE_FONTS: tuple[str, ...] = ("JetBrains Mono", "Space Mono")
|
||||
GOOGLE_FONTS_WEIGHTSET_PATTERN = re.compile(
|
||||
r"fonts\.googleapis\.com.*family=(Inter|Poppins|Manrope)[^&]*wght@400;500;600;700",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _detect_default_font(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
for candidate in DEFAULT_AI_FONTS:
|
||||
if candidate.lower() in family.lower():
|
||||
return Signal(
|
||||
"DEFAULT_AI_FONT",
|
||||
f"Default AI-tool font family in use ({candidate})",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_instrument_serif(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
if "instrument serif" in family.lower():
|
||||
return Signal(
|
||||
"INSTRUMENT_SERIF_ACCENT",
|
||||
"Instrument Serif accent font pairing",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_single_font_family(fonts: list[dict]) -> Signal | None:
|
||||
families = {str(sample.get("fontFamily", "")).strip() for sample in fonts if sample.get("fontFamily")}
|
||||
if len(fonts) >= 2 and len(families) == 1:
|
||||
return Signal(
|
||||
"SINGLE_FONT_FAMILY",
|
||||
"Every sampled element shares one exact font-family declaration",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
next(iter(families))[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_google_fonts_weightset(links: list[str]) -> Signal | None:
|
||||
for href in links:
|
||||
match = GOOGLE_FONTS_WEIGHTSET_PATTERN.search(str(href))
|
||||
if match:
|
||||
return Signal(
|
||||
"GOOGLE_FONTS_DEFAULT_WEIGHTSET",
|
||||
"Google Fonts request for the canonical AI-tool weight set (400;500;600;700)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
str(href)[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_decorative_monospace(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
tag = str(sample.get("tag", ""))
|
||||
if tag not in ("h1", "h2", "h3", "button"):
|
||||
continue
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
for candidate in DECORATIVE_MONOSPACE_FONTS:
|
||||
if candidate.lower() in family.lower():
|
||||
return Signal(
|
||||
"DECORATIVE_MONOSPACE",
|
||||
f"Decorative monospace font used outside code ({candidate})",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_typography_signals(page: DomPageContext) -> list[Signal]:
|
||||
fonts = page.dom.get("fonts", []) or []
|
||||
links = page.dom.get("links", []) or []
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_default_font(fonts),
|
||||
_detect_instrument_serif(fonts),
|
||||
_detect_single_font_family(fonts),
|
||||
_detect_google_fonts_weightset(links),
|
||||
_detect_decorative_monospace(fonts),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@@ -0,0 +1,323 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.baselines import RepoBaselines, compute_baselines, deviation_signals
|
||||
from devplacepy.services.jobs.isslop.analysis.exclusion import exclusion_reason, is_excluded_directory, looks_minified
|
||||
from devplacepy.services.jobs.isslop.analysis.languages import CODE_LANGUAGES, DOCUMENT_LANGUAGES, extract_comments, language_for
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics, compute_metrics
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import (
|
||||
FileScore,
|
||||
categorize,
|
||||
criticality_for,
|
||||
fingerprint_origin_score,
|
||||
origin_score,
|
||||
quality_deficit_score,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Detector, FileContext, RepoContext, run_detectors
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.documentation import detect_documentation
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.errors import detect_error_handling
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.hallucination import detect_hallucination
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.infrastructure import detect_infrastructure
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.language import detect_language_tells
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.llmdefaults import detect_llm_defaults
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.naming import detect_naming
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.scaffolds import detect_scaffolds
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.security import detect_security
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.structure import detect_structure
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.textual import detect_textual
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.vibeerrors import detect_vibe_errors
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.webtells import detect_web_tells
|
||||
from devplacepy.services.jobs.isslop.config import ANALYSIS_MAX_FILES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTORS: list[Detector] = [
|
||||
detect_textual,
|
||||
detect_naming,
|
||||
detect_structure,
|
||||
detect_hallucination,
|
||||
detect_error_handling,
|
||||
detect_security,
|
||||
detect_documentation,
|
||||
detect_language_tells,
|
||||
detect_web_tells,
|
||||
detect_llm_defaults,
|
||||
detect_infrastructure,
|
||||
detect_vibe_errors,
|
||||
detect_scaffolds,
|
||||
]
|
||||
|
||||
FINGERPRINT_DETECTORS: list[Detector] = [
|
||||
detect_textual,
|
||||
detect_web_tells,
|
||||
detect_llm_defaults,
|
||||
detect_infrastructure,
|
||||
detect_security,
|
||||
detect_vibe_errors,
|
||||
detect_scaffolds,
|
||||
]
|
||||
|
||||
PYPROJECT_DEPENDENCY_PATTERN = re.compile(r"[\"']([A-Za-z0-9._-]+)\s*(?:[><=!~\[;].*)?[\"']")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InventoryEntry:
|
||||
path: Path
|
||||
relative: str
|
||||
language: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Inventory:
|
||||
analyzable: list[InventoryEntry]
|
||||
excluded: list[tuple[str, str]]
|
||||
repo: RepoContext
|
||||
|
||||
|
||||
def _python_dependencies(root: Path) -> tuple[frozenset[str], bool]:
|
||||
dependencies: set[str] = set()
|
||||
found = False
|
||||
for name in ("requirements.txt", "requirements-dev.txt"):
|
||||
manifest = root / name
|
||||
if manifest.exists():
|
||||
found = True
|
||||
for line in manifest.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
cleaned = line.split("#")[0].strip()
|
||||
if cleaned and not cleaned.startswith("-"):
|
||||
dependencies.add(re.split(r"[><=!~\[;@ ]", cleaned)[0])
|
||||
pyproject = root / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
text = pyproject.read_text(encoding="utf-8", errors="replace")
|
||||
if "dependencies" in text or "[tool.poetry" in text:
|
||||
found = True
|
||||
block = re.search(r"dependencies\s*=\s*\[(.*?)\]", text, re.DOTALL)
|
||||
if block:
|
||||
for match in PYPROJECT_DEPENDENCY_PATTERN.finditer(block.group(1)):
|
||||
dependencies.add(match.group(1))
|
||||
poetry = re.search(r"\[tool\.poetry\.dependencies\](.*?)(?:\n\[|\Z)", text, re.DOTALL)
|
||||
if poetry:
|
||||
for line in poetry.group(1).splitlines():
|
||||
key = line.split("=")[0].strip().strip('"')
|
||||
if key and key != "python":
|
||||
dependencies.add(key)
|
||||
project_name = re.search(r"^\s*name\s*=\s*[\"']([\w.-]+)[\"']", text, re.MULTILINE)
|
||||
if project_name:
|
||||
dependencies.add(project_name.group(1))
|
||||
return frozenset(dependencies), found
|
||||
|
||||
|
||||
def _javascript_dependencies(root: Path) -> tuple[frozenset[str], bool]:
|
||||
manifest = root / "package.json"
|
||||
if not manifest.exists():
|
||||
return frozenset(), False
|
||||
try:
|
||||
body = json.loads(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError) as error:
|
||||
logger.warning("package.json unreadable: %s", error)
|
||||
return frozenset(), False
|
||||
dependencies: set[str] = set()
|
||||
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
||||
section = body.get(key)
|
||||
if isinstance(section, dict):
|
||||
dependencies.update(section.keys())
|
||||
name = body.get("name")
|
||||
if isinstance(name, str):
|
||||
dependencies.add(name)
|
||||
return frozenset(dependencies), True
|
||||
|
||||
|
||||
def _local_python_modules(root: Path) -> frozenset[str]:
|
||||
modules: set[str] = set()
|
||||
for entry in root.iterdir():
|
||||
if entry.is_dir() and not is_excluded_directory(entry.name):
|
||||
modules.add(entry.name)
|
||||
if (entry / "__init__.py").exists():
|
||||
modules.add(entry.name)
|
||||
elif entry.suffix == ".py":
|
||||
modules.add(entry.stem)
|
||||
for src in (root / "src", root / "lib"):
|
||||
if src.is_dir():
|
||||
for entry in src.iterdir():
|
||||
if entry.is_dir() or entry.suffix == ".py":
|
||||
modules.add(entry.stem if entry.is_file() else entry.name)
|
||||
return frozenset(modules)
|
||||
|
||||
|
||||
JSONC_TRAILING_COMMA_PATTERN = re.compile(r",(\s*[}\]])")
|
||||
|
||||
|
||||
def _strip_jsonc_comments(text: str) -> str:
|
||||
out: list[str] = []
|
||||
position = 0
|
||||
length = len(text)
|
||||
in_string = False
|
||||
escaped = False
|
||||
while position < length:
|
||||
char = text[position]
|
||||
if in_string:
|
||||
out.append(char)
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
position += 1
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
out.append(char)
|
||||
position += 1
|
||||
continue
|
||||
if char == "/" and text[position + 1 : position + 2] == "/":
|
||||
while position < length and text[position] != "\n":
|
||||
position += 1
|
||||
continue
|
||||
if char == "/" and text[position + 1 : position + 2] == "*":
|
||||
position += 2
|
||||
while position + 1 < length and not (text[position] == "*" and text[position + 1] == "/"):
|
||||
position += 1
|
||||
position += 2
|
||||
continue
|
||||
out.append(char)
|
||||
position += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _parse_jsonc(text: str) -> dict:
|
||||
cleaned = _strip_jsonc_comments(text)
|
||||
cleaned = JSONC_TRAILING_COMMA_PATTERN.sub(r"\1", cleaned)
|
||||
parsed = json.loads(cleaned)
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _javascript_alias_prefixes(root: Path) -> frozenset[str]:
|
||||
prefixes: set[str] = set()
|
||||
for name in ("tsconfig.json", "jsconfig.json"):
|
||||
manifest = root / name
|
||||
if not manifest.exists():
|
||||
continue
|
||||
try:
|
||||
body = _parse_jsonc(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError) as error:
|
||||
logger.warning("%s unreadable: %s", name, error)
|
||||
continue
|
||||
options = body.get("compilerOptions")
|
||||
paths = options.get("paths") if isinstance(options, dict) else None
|
||||
if not isinstance(paths, dict):
|
||||
continue
|
||||
for alias in paths:
|
||||
prefix = str(alias).rstrip("*")
|
||||
if prefix:
|
||||
prefixes.add(prefix)
|
||||
return frozenset(prefixes)
|
||||
|
||||
|
||||
def build_repo_context(root: Path) -> RepoContext:
|
||||
python_deps, has_python = _python_dependencies(root)
|
||||
javascript_deps, has_javascript = _javascript_dependencies(root)
|
||||
return RepoContext(
|
||||
root=root,
|
||||
python_dependencies=python_deps,
|
||||
javascript_dependencies=javascript_deps,
|
||||
local_python_modules=_local_python_modules(root),
|
||||
has_python_manifest=has_python,
|
||||
has_javascript_manifest=has_javascript,
|
||||
javascript_alias_prefixes=_javascript_alias_prefixes(root),
|
||||
)
|
||||
|
||||
|
||||
def build_inventory(workspace: Path) -> Inventory:
|
||||
analyzable: list[InventoryEntry] = []
|
||||
excluded: list[tuple[str, str]] = []
|
||||
candidates = sorted(
|
||||
entry for entry in workspace.rglob("*") if entry.is_file() and not entry.is_symlink()
|
||||
)
|
||||
for path in candidates:
|
||||
relative = str(path.relative_to(workspace))
|
||||
reason = exclusion_reason(path, workspace)
|
||||
if reason:
|
||||
excluded.append((relative, reason))
|
||||
continue
|
||||
language = language_for(path.name)
|
||||
if language not in CODE_LANGUAGES and language not in DOCUMENT_LANGUAGES:
|
||||
excluded.append((relative, f"unsupported language ({language})"))
|
||||
continue
|
||||
analyzable.append(InventoryEntry(path=path, relative=relative, language=language))
|
||||
if len(analyzable) >= ANALYSIS_MAX_FILES:
|
||||
logger.warning("File cap of %d reached, remaining files skipped", ANALYSIS_MAX_FILES)
|
||||
break
|
||||
repo = build_repo_context(workspace)
|
||||
logger.info("Inventory: %d analyzable, %d excluded", len(analyzable), len(excluded))
|
||||
return Inventory(analyzable=analyzable, excluded=excluded, repo=repo)
|
||||
|
||||
|
||||
def load_context(entry: InventoryEntry, repo: RepoContext) -> FileContext | None:
|
||||
try:
|
||||
text = entry.path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as error:
|
||||
logger.warning("Read failed for %s: %s", entry.relative, error)
|
||||
return None
|
||||
fingerprint_only = looks_minified(text)
|
||||
lines = text.splitlines()
|
||||
context = FileContext(
|
||||
path=entry.path,
|
||||
relative=entry.relative,
|
||||
language=entry.language,
|
||||
text=text,
|
||||
lines=lines,
|
||||
comments=extract_comments(entry.language, lines),
|
||||
repo=repo,
|
||||
fingerprint_only=fingerprint_only,
|
||||
)
|
||||
context.metrics = compute_metrics(entry.language, text)
|
||||
return context
|
||||
|
||||
|
||||
def score_file(context: FileContext, baselines: RepoBaselines) -> FileScore:
|
||||
metrics = context.metrics
|
||||
if metrics is None:
|
||||
metrics = compute_metrics(context.language, context.text)
|
||||
context.metrics = metrics
|
||||
if context.fingerprint_only:
|
||||
signals = run_detectors(context, FINGERPRINT_DETECTORS)
|
||||
origin = fingerprint_origin_score(signals)
|
||||
quality = quality_deficit_score(signals, metrics)
|
||||
effective_sloc = max(metrics.sloc, min(metrics.total_lines, 20))
|
||||
return FileScore(
|
||||
relative=context.relative,
|
||||
language=context.language,
|
||||
sloc=effective_sloc,
|
||||
origin_score=origin,
|
||||
quality_deficit=quality,
|
||||
category=categorize(origin, quality),
|
||||
criticality=criticality_for(context.relative),
|
||||
signals=signals,
|
||||
)
|
||||
signals = run_detectors(context, DETECTORS)
|
||||
signals.extend(deviation_signals(context.relative, metrics, baselines))
|
||||
origin = origin_score(context.text, metrics, signals)
|
||||
quality = quality_deficit_score(signals, metrics)
|
||||
return FileScore(
|
||||
relative=context.relative,
|
||||
language=context.language,
|
||||
sloc=metrics.sloc,
|
||||
origin_score=origin,
|
||||
quality_deficit=quality,
|
||||
category=categorize(origin, quality),
|
||||
criticality=criticality_for(context.relative),
|
||||
signals=signals,
|
||||
)
|
||||
|
||||
|
||||
def compute_repo_baselines(contexts: list[FileContext]) -> RepoBaselines:
|
||||
metrics_by_file: dict[str, FileMetrics] = {
|
||||
context.relative: context.metrics for context in contexts if context.metrics is not None
|
||||
}
|
||||
return compute_baselines(metrics_by_file)
|
||||
@@ -0,0 +1,142 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.services.jobs.isslop.config import ANALYSIS_FILE_CAP_BYTES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EXCLUDED_DIRECTORIES: frozenset[str] = frozenset(
|
||||
{
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
"node_modules",
|
||||
"bower_components",
|
||||
"vendor",
|
||||
"third_party",
|
||||
"thirdparty",
|
||||
"__pycache__",
|
||||
".venv",
|
||||
"venv",
|
||||
"env",
|
||||
".env",
|
||||
"dist",
|
||||
"build",
|
||||
"out",
|
||||
"target",
|
||||
"coverage",
|
||||
".next",
|
||||
".nuxt",
|
||||
".cache",
|
||||
"site-packages",
|
||||
".idea",
|
||||
".vscode",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
".pytest_cache",
|
||||
".terraform",
|
||||
"migrations",
|
||||
}
|
||||
)
|
||||
|
||||
LOCKFILE_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
"package-lock.json",
|
||||
"yarn.lock",
|
||||
"pnpm-lock.yaml",
|
||||
"poetry.lock",
|
||||
"pipfile.lock",
|
||||
"gemfile.lock",
|
||||
"go.sum",
|
||||
"cargo.lock",
|
||||
"composer.lock",
|
||||
"uv.lock",
|
||||
"bun.lockb",
|
||||
}
|
||||
)
|
||||
|
||||
GENERATED_SUFFIXES: tuple[str, ...] = (
|
||||
".min.js",
|
||||
".min.css",
|
||||
".map",
|
||||
".pb.go",
|
||||
"_pb2.py",
|
||||
"_pb2_grpc.py",
|
||||
".bundle.js",
|
||||
".chunk.js",
|
||||
".d.ts",
|
||||
)
|
||||
|
||||
BINARY_EXTENSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".bmp", ".ico", ".tiff", ".svgz",
|
||||
".woff", ".woff2", ".ttf", ".otf", ".eot",
|
||||
".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
|
||||
".mp3", ".mp4", ".wav", ".ogg", ".webm", ".avi", ".mov", ".flac",
|
||||
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||
".exe", ".dll", ".so", ".dylib", ".bin", ".dat", ".db", ".sqlite", ".sqlite3",
|
||||
".pyc", ".pyo", ".class", ".jar", ".war", ".wasm", ".o", ".a",
|
||||
}
|
||||
)
|
||||
|
||||
GENERATED_MARKERS: tuple[str, ...] = (
|
||||
"do not edit",
|
||||
"auto-generated",
|
||||
"autogenerated",
|
||||
"@generated",
|
||||
"code generated by",
|
||||
"this file was generated",
|
||||
)
|
||||
|
||||
MINIFIED_AVG_LINE_LENGTH: int = 300
|
||||
|
||||
|
||||
def is_excluded_directory(part: str) -> bool:
|
||||
return part.lower() in EXCLUDED_DIRECTORIES
|
||||
|
||||
|
||||
def exclusion_reason(path: Path, workspace: Path) -> Optional[str]:
|
||||
relative = path.relative_to(workspace)
|
||||
for part in relative.parts[:-1]:
|
||||
if is_excluded_directory(part):
|
||||
return f"vendored or generated directory: {part}"
|
||||
name = path.name.lower()
|
||||
if name in LOCKFILE_NAMES:
|
||||
return "lockfile"
|
||||
for suffix in GENERATED_SUFFIXES:
|
||||
if name.endswith(suffix):
|
||||
return f"generated artifact ({suffix})"
|
||||
if path.suffix.lower() in BINARY_EXTENSIONS:
|
||||
return "binary file"
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError as error:
|
||||
return f"unreadable: {error}"
|
||||
if size == 0:
|
||||
return "empty file"
|
||||
if size > ANALYSIS_FILE_CAP_BYTES:
|
||||
return f"oversize ({size} bytes)"
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
head = handle.read(8192)
|
||||
except OSError as error:
|
||||
return f"unreadable: {error}"
|
||||
if b"\x00" in head:
|
||||
return "binary content"
|
||||
text_head = head.decode("utf-8", errors="replace").lower()
|
||||
for marker in GENERATED_MARKERS:
|
||||
if marker in text_head[:600]:
|
||||
return f"generated marker: {marker}"
|
||||
return None
|
||||
|
||||
|
||||
def looks_minified(text: str) -> bool:
|
||||
lines = [line for line in text.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return False
|
||||
average = sum(len(line) for line in lines) / len(lines)
|
||||
return average > MINIFIED_AVG_LINE_LENGTH
|
||||
@@ -0,0 +1,135 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
LANGUAGE_BY_EXTENSION: dict[str, str] = {
|
||||
".py": "python",
|
||||
".pyw": "python",
|
||||
".js": "javascript",
|
||||
".mjs": "javascript",
|
||||
".cjs": "javascript",
|
||||
".jsx": "javascript",
|
||||
".ts": "typescript",
|
||||
".tsx": "typescript",
|
||||
".html": "html",
|
||||
".htm": "html",
|
||||
".css": "css",
|
||||
".scss": "css",
|
||||
".less": "css",
|
||||
".php": "php",
|
||||
".go": "go",
|
||||
".java": "java",
|
||||
".kt": "java",
|
||||
".c": "c",
|
||||
".h": "c",
|
||||
".cpp": "cpp",
|
||||
".cc": "cpp",
|
||||
".hpp": "cpp",
|
||||
".cs": "csharp",
|
||||
".rb": "ruby",
|
||||
".rs": "rust",
|
||||
".sh": "shell",
|
||||
".bash": "shell",
|
||||
".lua": "lua",
|
||||
".sql": "sql",
|
||||
".md": "markdown",
|
||||
".rst": "markdown",
|
||||
".txt": "text",
|
||||
".json": "json",
|
||||
".yaml": "yaml",
|
||||
".yml": "yaml",
|
||||
".toml": "toml",
|
||||
".xml": "xml",
|
||||
".vue": "javascript",
|
||||
".svelte": "javascript",
|
||||
}
|
||||
|
||||
CODE_LANGUAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"python",
|
||||
"javascript",
|
||||
"typescript",
|
||||
"php",
|
||||
"go",
|
||||
"java",
|
||||
"c",
|
||||
"cpp",
|
||||
"csharp",
|
||||
"ruby",
|
||||
"rust",
|
||||
"shell",
|
||||
"lua",
|
||||
"sql",
|
||||
"html",
|
||||
"css",
|
||||
}
|
||||
)
|
||||
|
||||
DOCUMENT_LANGUAGES: frozenset[str] = frozenset({"markdown", "text"})
|
||||
|
||||
LINE_COMMENT_PREFIXES: dict[str, tuple[str, ...]] = {
|
||||
"python": ("#",),
|
||||
"shell": ("#",),
|
||||
"ruby": ("#",),
|
||||
"yaml": ("#",),
|
||||
"toml": ("#",),
|
||||
"javascript": ("//",),
|
||||
"typescript": ("//",),
|
||||
"php": ("//", "#"),
|
||||
"go": ("//",),
|
||||
"java": ("//",),
|
||||
"c": ("//",),
|
||||
"cpp": ("//",),
|
||||
"csharp": ("//",),
|
||||
"rust": ("//",),
|
||||
"sql": ("--",),
|
||||
"lua": ("--",),
|
||||
}
|
||||
|
||||
BLOCK_COMMENT_LANGUAGES: frozenset[str] = frozenset(
|
||||
{"javascript", "typescript", "php", "go", "java", "c", "cpp", "csharp", "rust", "css"}
|
||||
)
|
||||
|
||||
|
||||
def language_for(filename: str) -> str:
|
||||
lowered = filename.lower()
|
||||
for extension, language in LANGUAGE_BY_EXTENSION.items():
|
||||
if lowered.endswith(extension):
|
||||
return language
|
||||
return "unknown"
|
||||
|
||||
|
||||
def extract_comments(language: str, lines: list[str]) -> list[tuple[int, str]]:
|
||||
comments: list[tuple[int, str]] = []
|
||||
prefixes = LINE_COMMENT_PREFIXES.get(language, ())
|
||||
in_block = False
|
||||
block_open, block_close = ("/*", "*/") if language in BLOCK_COMMENT_LANGUAGES or language == "css" else ("", "")
|
||||
if language == "html":
|
||||
block_open, block_close = "<!--", "-->"
|
||||
for number, raw in enumerate(lines, start=1):
|
||||
stripped = raw.strip()
|
||||
if in_block:
|
||||
comments.append((number, stripped.replace(block_close, "").strip()))
|
||||
if block_close and block_close in stripped:
|
||||
in_block = False
|
||||
continue
|
||||
if block_open and block_open in stripped:
|
||||
fragment = stripped.split(block_open, 1)[1]
|
||||
comments.append((number, fragment.replace(block_close, "").strip()))
|
||||
if block_close not in fragment:
|
||||
in_block = True
|
||||
continue
|
||||
for prefix in prefixes:
|
||||
if stripped.startswith(prefix):
|
||||
comments.append((number, stripped[len(prefix):].strip()))
|
||||
break
|
||||
marker = f" {prefix}"
|
||||
if marker in raw and not _inside_string(raw, raw.index(marker)):
|
||||
comments.append((number, raw.split(marker, 1)[1].strip()))
|
||||
break
|
||||
return comments
|
||||
|
||||
|
||||
def _inside_string(line: str, position: int) -> bool:
|
||||
double = line.count('"', 0, position) % 2 == 1
|
||||
single = line.count("'", 0, position) % 2 == 1
|
||||
return double or single
|
||||
@@ -0,0 +1,163 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.languages import extract_comments
|
||||
|
||||
COMPLEXITY_TOKENS = re.compile(
|
||||
r"\b(if|elif|else if|for|while|case|when|catch|except|and|or)\b|&&|\|\||\?\s"
|
||||
)
|
||||
FUNCTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||
"python": re.compile(r"^\s*(?:async\s+)?def\s+(\w+)", re.MULTILINE),
|
||||
"javascript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)|^\s*(\w+)\s*\([^)]*\)\s*{", re.MULTILINE),
|
||||
"typescript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)|^\s*(\w+)\s*\([^)]*\)\s*{", re.MULTILINE),
|
||||
"go": re.compile(r"^\s*func\s+(?:\([^)]+\)\s*)?(\w+)", re.MULTILINE),
|
||||
"java": re.compile(r"^\s*(?:public|private|protected|static|\s)+[\w<>\[\]]+\s+(\w+)\s*\(", re.MULTILINE),
|
||||
"php": re.compile(r"^\s*(?:public|private|protected|static|\s)*function\s+(\w+)", re.MULTILINE),
|
||||
"ruby": re.compile(r"^\s*def\s+(\w+)", re.MULTILINE),
|
||||
"rust": re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)", re.MULTILINE),
|
||||
}
|
||||
DOCSTRING_PATTERN = re.compile(r"def\s+\w+[^:]*:\s*\n\s*(?:\"\"\"|''')", re.MULTILINE)
|
||||
TOKEN_PATTERN = re.compile(r"[A-Za-z_]\w*|\d+|[^\sA-Za-z0-9_]")
|
||||
DUPLICATION_WINDOW: int = 6
|
||||
MAINTAINABILITY_CEILING: float = 171.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileMetrics:
|
||||
total_lines: int
|
||||
sloc: int
|
||||
blank_lines: int
|
||||
comment_lines: int
|
||||
blank_ratio: float
|
||||
comment_ratio: float
|
||||
avg_line_length: float
|
||||
max_line_length: int
|
||||
avg_leading_spaces: float
|
||||
avg_leading_tabs: float
|
||||
indent_variance: float
|
||||
function_count: int
|
||||
docstring_count: int
|
||||
max_function_length: int
|
||||
cyclomatic_total: int
|
||||
cyclomatic_max_estimate: int
|
||||
duplication_ratio: float
|
||||
maintainability_index: float
|
||||
import_count: int
|
||||
|
||||
|
||||
def shannon_entropy(value: str) -> float:
|
||||
if not value:
|
||||
return 0.0
|
||||
counts: dict[str, int] = {}
|
||||
for char in value:
|
||||
counts[char] = counts.get(char, 0) + 1
|
||||
total = len(value)
|
||||
return -sum((count / total) * math.log2(count / total) for count in counts.values())
|
||||
|
||||
|
||||
def _leading_whitespace(lines: list[str]) -> tuple[float, float, float]:
|
||||
spaces: list[int] = []
|
||||
tabs: list[int] = []
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
space_count = len(line) - len(line.lstrip(" "))
|
||||
tab_count = len(line) - len(line.lstrip("\t"))
|
||||
spaces.append(space_count)
|
||||
tabs.append(tab_count)
|
||||
if not spaces:
|
||||
return 0.0, 0.0, 0.0
|
||||
mean_spaces = sum(spaces) / len(spaces)
|
||||
mean_tabs = sum(tabs) / len(tabs)
|
||||
variance = sum((value - mean_spaces) ** 2 for value in spaces) / len(spaces)
|
||||
return mean_spaces, mean_tabs, math.sqrt(variance)
|
||||
|
||||
|
||||
def _duplication_ratio(lines: list[str]) -> float:
|
||||
normalized = [re.sub(r"\s+", " ", line.strip()) for line in lines if line.strip()]
|
||||
if len(normalized) < DUPLICATION_WINDOW * 2:
|
||||
return 0.0
|
||||
seen: dict[str, int] = {}
|
||||
duplicated = 0
|
||||
windows = 0
|
||||
for index in range(len(normalized) - DUPLICATION_WINDOW + 1):
|
||||
window = "\n".join(normalized[index:index + DUPLICATION_WINDOW])
|
||||
if len(window) < 60:
|
||||
continue
|
||||
digest = hashlib.sha1(window.encode("utf-8")).hexdigest()
|
||||
windows += 1
|
||||
if digest in seen:
|
||||
duplicated += 1
|
||||
seen[digest] = index
|
||||
return duplicated / windows if windows else 0.0
|
||||
|
||||
|
||||
def _function_lengths(language: str, lines: list[str]) -> tuple[int, int]:
|
||||
pattern = FUNCTION_PATTERNS.get(language)
|
||||
if pattern is None:
|
||||
return 0, 0
|
||||
starts: list[int] = []
|
||||
for number, line in enumerate(lines):
|
||||
if pattern.match(line):
|
||||
starts.append(number)
|
||||
if not starts:
|
||||
return 0, 0
|
||||
lengths: list[int] = []
|
||||
for index, start in enumerate(starts):
|
||||
end = starts[index + 1] if index + 1 < len(starts) else len(lines)
|
||||
lengths.append(end - start)
|
||||
return len(starts), max(lengths)
|
||||
|
||||
|
||||
def _maintainability(sloc: int, cyclomatic: int, tokens: list[str]) -> float:
|
||||
if sloc <= 0 or not tokens:
|
||||
return 100.0
|
||||
unique = len(set(tokens))
|
||||
volume = len(tokens) * math.log2(max(unique, 2))
|
||||
raw = MAINTAINABILITY_CEILING - 5.2 * math.log(max(volume, 1.0)) - 0.23 * cyclomatic - 16.2 * math.log(max(sloc, 1))
|
||||
return max(0.0, raw * 100.0 / MAINTAINABILITY_CEILING)
|
||||
|
||||
|
||||
def compute_metrics(language: str, text: str) -> FileMetrics:
|
||||
lines = text.splitlines()
|
||||
total = len(lines)
|
||||
blank = sum(1 for line in lines if not line.strip())
|
||||
comments = extract_comments(language, lines)
|
||||
comment_lines = len(comments)
|
||||
sloc = max(0, total - blank - comment_lines)
|
||||
lengths = [len(line) for line in lines if line.strip()]
|
||||
avg_length = sum(lengths) / len(lengths) if lengths else 0.0
|
||||
max_length = max(lengths) if lengths else 0
|
||||
mean_spaces, mean_tabs, indent_deviation = _leading_whitespace(lines)
|
||||
function_count, max_function_length = _function_lengths(language, lines)
|
||||
docstring_count = len(DOCSTRING_PATTERN.findall(text)) if language == "python" else 0
|
||||
cyclomatic = len(COMPLEXITY_TOKENS.findall(text))
|
||||
per_function = cyclomatic // function_count if function_count else cyclomatic
|
||||
tokens = TOKEN_PATTERN.findall(text)[:20000]
|
||||
import_count = len(re.findall(r"^\s*(?:import|from|require|use|#include)\b", text, re.MULTILINE))
|
||||
return FileMetrics(
|
||||
total_lines=total,
|
||||
sloc=sloc,
|
||||
blank_lines=blank,
|
||||
comment_lines=comment_lines,
|
||||
blank_ratio=blank / total if total else 0.0,
|
||||
comment_ratio=comment_lines / total if total else 0.0,
|
||||
avg_line_length=avg_length,
|
||||
max_line_length=max_length,
|
||||
avg_leading_spaces=mean_spaces,
|
||||
avg_leading_tabs=mean_tabs,
|
||||
indent_variance=indent_deviation,
|
||||
function_count=function_count,
|
||||
docstring_count=docstring_count,
|
||||
max_function_length=max_function_length,
|
||||
cyclomatic_total=cyclomatic,
|
||||
cyclomatic_max_estimate=per_function,
|
||||
duplication_ratio=_duplication_ratio(lines),
|
||||
maintainability_index=_maintainability(sloc, per_function, tokens),
|
||||
import_count=import_count,
|
||||
)
|
||||
@@ -0,0 +1,302 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.aggregate import DomEvidence
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_FACTORS,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
Signal,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.config import DOM_AI_WEIGHT
|
||||
|
||||
CATEGORY_SLOP: str = "ai-slop"
|
||||
CATEGORY_SOPHISTICATED: str = "sophisticated-ai"
|
||||
CATEGORY_HUMAN_CLEAN: str = "human-clean"
|
||||
CATEGORY_HUMAN_MESSY: str = "human-messy"
|
||||
CATEGORY_UNCERTAIN: str = "uncertain"
|
||||
|
||||
ORIGIN_HIGH_THRESHOLD: float = 55.0
|
||||
ORIGIN_LOW_THRESHOLD: float = 45.0
|
||||
QUALITY_SLOP_THRESHOLD: float = 50.0
|
||||
QUALITY_CLEAN_THRESHOLD: float = 30.0
|
||||
QUALITY_SATURATION: float = 14.0
|
||||
ORIGIN_SIGNAL_CAP: float = 45.0
|
||||
ORIGIN_SIGNAL_SCALE: float = 4.5
|
||||
|
||||
CRITICAL_PATH_PATTERN = re.compile(r"auth|login|password|payment|billing|crypto|security|token|session|admin", re.IGNORECASE)
|
||||
CRITICALITY_WEIGHT: float = 1.5
|
||||
|
||||
GRADE_BANDS: tuple[tuple[float, str], ...] = ((15.0, "A"), (30.0, "B"), (50.0, "C"), (70.0, "D"), (100.0, "F"))
|
||||
|
||||
AI_SCORE_WEIGHT: float = 0.68
|
||||
QUALITY_SCORE_WEIGHT: float = 0.32
|
||||
|
||||
AI_RAMP_LOW: float = 35.0
|
||||
AI_RAMP_HIGH: float = 65.0
|
||||
|
||||
IMAGE_TEXT_AI_WEIGHT: float = 0.75
|
||||
IMAGE_AI_WEIGHT: float = 0.25
|
||||
|
||||
|
||||
def compose_slop_score(ai_percent: float, quality_deficit: float) -> float:
|
||||
return round(AI_SCORE_WEIGHT * ai_percent + QUALITY_SCORE_WEIGHT * quality_deficit, 1)
|
||||
|
||||
|
||||
def ai_fraction(origin: float) -> float:
|
||||
if origin <= AI_RAMP_LOW:
|
||||
return 0.0
|
||||
if origin >= AI_RAMP_HIGH:
|
||||
return 1.0
|
||||
position = (origin - AI_RAMP_LOW) / (AI_RAMP_HIGH - AI_RAMP_LOW)
|
||||
return position * position * (3.0 - 2.0 * position)
|
||||
|
||||
HUMAN_MARKER_PATTERN = re.compile(r"\b(HACK|FIXME|XXX)\b.*[A-Z]{2,}-\d+|\bFIXME\(\w+\)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileScore:
|
||||
relative: str
|
||||
language: str
|
||||
sloc: int
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
category: str
|
||||
criticality: float
|
||||
signals: list[Signal] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoScores:
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
slop_score: float
|
||||
grade: str
|
||||
category: str
|
||||
human_percent: float
|
||||
ai_percent: float
|
||||
confidence: str
|
||||
strong_signal_count: int
|
||||
medium_signal_count: int
|
||||
files_scored: int
|
||||
|
||||
|
||||
def quality_deficit_score(signals: list[Signal], metrics: FileMetrics) -> float:
|
||||
weighted = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis != AXIS_ORIGIN
|
||||
)
|
||||
if metrics.maintainability_index < 40.0:
|
||||
weighted += 2.0
|
||||
elif metrics.maintainability_index < 65.0:
|
||||
weighted += 1.0
|
||||
return round(100.0 * (1.0 - math.exp(-weighted / QUALITY_SATURATION)), 1)
|
||||
|
||||
|
||||
FINGERPRINT_ORIGIN_BASE: float = 50.0
|
||||
FINGERPRINT_ORIGIN_SCALE: float = 7.0
|
||||
|
||||
|
||||
def fingerprint_origin_score(signals: list[Signal]) -> float:
|
||||
origin_weight = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis == AXIS_ORIGIN
|
||||
)
|
||||
if origin_weight <= 0.0:
|
||||
return FINGERPRINT_ORIGIN_BASE
|
||||
score = FINGERPRINT_ORIGIN_BASE + origin_weight * FINGERPRINT_ORIGIN_SCALE
|
||||
if any(signal.code in ("AI_BUILDER_FINGERPRINT", "VIBE_STACK") for signal in signals):
|
||||
score += 12.0
|
||||
return round(min(100.0, score), 1)
|
||||
|
||||
|
||||
def origin_score(text: str, metrics: FileMetrics, signals: list[Signal]) -> float:
|
||||
score = 0.0
|
||||
if metrics.sloc >= 40:
|
||||
if metrics.indent_variance < 0.45:
|
||||
score += 16.0
|
||||
elif metrics.indent_variance < 1.2:
|
||||
score += 8.0
|
||||
elif metrics.indent_variance > 4.0:
|
||||
score -= 14.0
|
||||
if 0.10 <= metrics.blank_ratio <= 0.22 and metrics.total_lines >= 60:
|
||||
score += 6.0
|
||||
if metrics.comment_ratio >= 0.22 and metrics.sloc >= 40:
|
||||
score += 8.0
|
||||
if metrics.function_count >= 4 and metrics.docstring_count >= metrics.function_count:
|
||||
score += 10.0
|
||||
origin_weight = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis == AXIS_ORIGIN
|
||||
)
|
||||
score += min(ORIGIN_SIGNAL_CAP, origin_weight * ORIGIN_SIGNAL_SCALE)
|
||||
if any(signal.code == "AI_SIGNATURE" for signal in signals):
|
||||
score += 22.0
|
||||
if any(signal.code == "NAMING_MIXED" for signal in signals):
|
||||
score -= 8.0
|
||||
if HUMAN_MARKER_PATTERN.search(text):
|
||||
score -= 10.0
|
||||
score += 24.0
|
||||
return round(min(100.0, max(0.0, score)), 1)
|
||||
|
||||
|
||||
def categorize(origin: float, quality: float) -> str:
|
||||
if origin >= ORIGIN_HIGH_THRESHOLD and quality >= QUALITY_SLOP_THRESHOLD:
|
||||
return CATEGORY_SLOP
|
||||
if origin >= ORIGIN_HIGH_THRESHOLD and quality < QUALITY_CLEAN_THRESHOLD:
|
||||
return CATEGORY_SOPHISTICATED
|
||||
if origin < ORIGIN_LOW_THRESHOLD:
|
||||
return CATEGORY_HUMAN_MESSY if quality >= QUALITY_SLOP_THRESHOLD else CATEGORY_HUMAN_CLEAN
|
||||
return CATEGORY_UNCERTAIN
|
||||
|
||||
|
||||
def criticality_for(relative: str) -> float:
|
||||
return CRITICALITY_WEIGHT if CRITICAL_PATH_PATTERN.search(relative) else 1.0
|
||||
|
||||
|
||||
def grade_for(slop: float) -> str:
|
||||
for ceiling, grade in GRADE_BANDS:
|
||||
if slop <= ceiling:
|
||||
return grade
|
||||
return "F"
|
||||
|
||||
|
||||
def _confidence(strong: int, medium: int, files: int) -> str:
|
||||
if files < 3:
|
||||
return "low"
|
||||
if strong >= 3:
|
||||
return "high"
|
||||
if strong >= 1 or medium >= 5:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def aggregate(files: list[FileScore]) -> RepoScores:
|
||||
scored = [entry for entry in files if entry.sloc > 0]
|
||||
if not scored:
|
||||
return RepoScores(0.0, 0.0, 0.0, "A", CATEGORY_UNCERTAIN, 50.0, 50.0, "low", 0, 0, 0)
|
||||
total_weight = sum(entry.sloc * entry.criticality for entry in scored)
|
||||
origin = sum(entry.origin_score * entry.sloc * entry.criticality for entry in scored) / total_weight
|
||||
quality = sum(entry.quality_deficit * entry.sloc * entry.criticality for entry in scored) / total_weight
|
||||
ai_mass = sum(entry.sloc * entry.criticality * ai_fraction(entry.origin_score) for entry in scored)
|
||||
ai_percent = round(100.0 * ai_mass / total_weight, 1)
|
||||
slop = compose_slop_score(ai_percent, quality)
|
||||
strong = sum(1 for entry in scored for signal in entry.signals if signal.severity == SEVERITY_STRONG)
|
||||
medium = sum(1 for entry in scored for signal in entry.signals if signal.severity == SEVERITY_MEDIUM)
|
||||
return RepoScores(
|
||||
origin_score=round(origin, 1),
|
||||
quality_deficit=round(quality, 1),
|
||||
slop_score=round(slop, 1),
|
||||
grade=grade_for(slop),
|
||||
category=categorize(origin, quality),
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
ai_percent=ai_percent,
|
||||
confidence=_confidence(strong, medium, len(scored)),
|
||||
strong_signal_count=strong,
|
||||
medium_signal_count=medium,
|
||||
files_scored=len(scored),
|
||||
)
|
||||
|
||||
|
||||
TEMPLATE_CONFIDENT_SCORE: float = 35.0
|
||||
TEMPLATE_STRONG_SCORE: float = 70.0
|
||||
TEMPLATE_INFLUENCE: float = 0.7
|
||||
TEMPLATE_STRONG_INFLUENCE: float = 0.85
|
||||
|
||||
DOM_BUILDER_CONFIDENT_THRESHOLD: float = 0.75
|
||||
DOM_BUILDER_FLOOR: float = 70.0
|
||||
|
||||
|
||||
def _force_slop_category(scores: RepoScores, floor: float) -> RepoScores:
|
||||
ai_percent = max(scores.ai_percent, floor)
|
||||
origin = max(scores.origin_score, floor)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
origin_score=origin,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
category=CATEGORY_SLOP,
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_template(scores: RepoScores, template_score: float) -> RepoScores:
|
||||
if template_score < TEMPLATE_CONFIDENT_SCORE:
|
||||
return scores
|
||||
if template_score >= TEMPLATE_STRONG_SCORE:
|
||||
floor = round(TEMPLATE_STRONG_INFLUENCE * template_score, 1)
|
||||
return _force_slop_category(scores, floor)
|
||||
floor = round(TEMPLATE_INFLUENCE * template_score, 1)
|
||||
ai_percent = max(scores.ai_percent, floor)
|
||||
origin = max(scores.origin_score, floor)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
category = categorize(origin, scores.quality_deficit)
|
||||
if category in (CATEGORY_HUMAN_CLEAN, CATEGORY_HUMAN_MESSY):
|
||||
category = CATEGORY_UNCERTAIN
|
||||
return replace(
|
||||
scores,
|
||||
origin_score=origin,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
category=category,
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_dom_signals(scores: RepoScores, dom: DomEvidence) -> RepoScores:
|
||||
if dom.detected_builder is not None and dom.builder_confidence >= DOM_BUILDER_CONFIDENT_THRESHOLD:
|
||||
return _force_slop_category(scores, DOM_BUILDER_FLOOR)
|
||||
if dom.score <= 0:
|
||||
return scores
|
||||
ai_percent = round((1.0 - DOM_AI_WEIGHT) * scores.ai_percent + DOM_AI_WEIGHT * dom.score, 1)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_images(scores: RepoScores, mean_image_ai: float) -> RepoScores:
|
||||
ai_percent = round(
|
||||
IMAGE_TEXT_AI_WEIGHT * scores.ai_percent + IMAGE_AI_WEIGHT * mean_image_ai, 1
|
||||
)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
)
|
||||
|
||||
|
||||
def repo_scores_to_dict(scores: RepoScores) -> dict[str, Any]:
|
||||
return {
|
||||
"origin_score": scores.origin_score,
|
||||
"quality_deficit": scores.quality_deficit,
|
||||
"slop_score": scores.slop_score,
|
||||
"grade": scores.grade,
|
||||
"category": scores.category,
|
||||
"human_percent": scores.human_percent,
|
||||
"ai_percent": scores.ai_percent,
|
||||
"confidence": scores.confidence,
|
||||
"strong_signal_count": scores.strong_signal_count,
|
||||
"medium_signal_count": scores.medium_signal_count,
|
||||
"files_scored": scores.files_scored,
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
|
||||
SEVERITY_STRONG: str = "strong"
|
||||
SEVERITY_MEDIUM: str = "medium"
|
||||
SEVERITY_WEAK: str = "weak"
|
||||
|
||||
AXIS_ORIGIN: str = "origin"
|
||||
AXIS_QUALITY: str = "quality"
|
||||
|
||||
SEVERITY_FACTORS: dict[str, float] = {
|
||||
SEVERITY_STRONG: 1.0,
|
||||
SEVERITY_MEDIUM: 0.6,
|
||||
SEVERITY_WEAK: 0.3,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Signal:
|
||||
code: str
|
||||
title: str
|
||||
severity: str
|
||||
axis: str
|
||||
weight: float
|
||||
line: int
|
||||
evidence: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"code": self.code,
|
||||
"title": self.title,
|
||||
"severity": self.severity,
|
||||
"axis": self.axis,
|
||||
"weight": self.weight,
|
||||
"line": self.line,
|
||||
"evidence": self.evidence[:200],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoContext:
|
||||
root: Path
|
||||
python_dependencies: frozenset[str]
|
||||
javascript_dependencies: frozenset[str]
|
||||
local_python_modules: frozenset[str]
|
||||
has_python_manifest: bool
|
||||
has_javascript_manifest: bool
|
||||
javascript_alias_prefixes: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileContext:
|
||||
path: Path
|
||||
relative: str
|
||||
language: str
|
||||
text: str
|
||||
lines: list[str] = field(default_factory=list)
|
||||
comments: list[tuple[int, str]] = field(default_factory=list)
|
||||
metrics: FileMetrics | None = None
|
||||
repo: RepoContext | None = None
|
||||
fingerprint_only: bool = False
|
||||
|
||||
|
||||
Detector = Callable[[FileContext], list[Signal]]
|
||||
|
||||
|
||||
def run_detectors(context: FileContext, detectors: list[Detector]) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for detector in detectors:
|
||||
findings.extend(detector(context))
|
||||
return findings
|
||||
@@ -0,0 +1,114 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.textual import EM_DASH_THRESHOLD, LLM_LEXICON_PATTERN, LLM_LEXICON_THRESHOLD
|
||||
|
||||
EMOJI_HEADER_PATTERN = re.compile(r"^#{1,4}\s*[\U0001F300-\U0001FAFF✨🚀🤝⭐✅❗]")
|
||||
BADGE_PATTERN = re.compile(r"!\[[^\]]*\]\([^)]*(?:badge|shields\.io)[^)]*\)")
|
||||
BOILERPLATE_PHRASES: tuple[str, ...] = (
|
||||
"getting started",
|
||||
"features",
|
||||
"installation",
|
||||
"prerequisites",
|
||||
"contributing",
|
||||
"built with",
|
||||
"tech stack",
|
||||
"acknowledgments",
|
||||
"roadmap",
|
||||
)
|
||||
MARKETING_PATTERN = re.compile(
|
||||
r"blazingly fast|seamlessly|robust and scalable|cutting[- ]edge|revolutioniz|effortlessly|supercharge|powerful and flexible",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
NOT_JUST_PATTERN = re.compile(r"it'?s not just \w+[,;.]? it'?s", re.IGNORECASE)
|
||||
|
||||
EMOJI_HEADER_THRESHOLD: int = 3
|
||||
BADGE_THRESHOLD: int = 6
|
||||
BOILERPLATE_THRESHOLD: int = 5
|
||||
MARKETING_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def detect_documentation(context: FileContext) -> list[Signal]:
|
||||
if context.language != "markdown":
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
emoji_headers = [number for number, line in enumerate(context.lines, start=1) if EMOJI_HEADER_PATTERN.match(line)]
|
||||
if len(emoji_headers) >= EMOJI_HEADER_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"README_EMOJI_STRUCTURE",
|
||||
f"{len(emoji_headers)} emoji-prefixed headers in documentation",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
emoji_headers[0],
|
||||
context.lines[emoji_headers[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
lowered = context.text.lower()
|
||||
phrase_hits = sum(1 for phrase in BOILERPLATE_PHRASES if phrase in lowered)
|
||||
if phrase_hits >= BOILERPLATE_THRESHOLD and emoji_headers:
|
||||
findings.append(
|
||||
Signal(
|
||||
"README_BOILERPLATE",
|
||||
f"Generic AI README structure ({phrase_hits} boilerplate sections with emoji headers)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
", ".join(phrase for phrase in BOILERPLATE_PHRASES if phrase in lowered)[:150],
|
||||
)
|
||||
)
|
||||
badges = BADGE_PATTERN.findall(context.text)
|
||||
if len(badges) >= BADGE_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("README_BADGE_STUFFING", f"{len(badges)} badges stacked in documentation", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, badges[0][:100])
|
||||
)
|
||||
marketing_hits = MARKETING_PATTERN.findall(context.text)
|
||||
if len(marketing_hits) >= MARKETING_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("MARKETING_TONE", f"Marketing tone in documentation ({len(marketing_hits)} phrases)", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, ", ".join(marketing_hits[:4]))
|
||||
)
|
||||
lexicon_hits = {str(hit).lower() for hit in LLM_LEXICON_PATTERN.findall(context.text)}
|
||||
if len(lexicon_hits) >= LLM_LEXICON_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_LEXICON",
|
||||
f"{len(lexicon_hits)} distinct LLM signature words in documentation",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(sorted(lexicon_hits)[:6]),
|
||||
)
|
||||
)
|
||||
em_dashes = context.text.count("\u2014")
|
||||
if em_dashes >= EM_DASH_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"EM_DASH_OVERUSE",
|
||||
f"{em_dashes} em dashes in documentation, a strong LLM prose habit",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
1,
|
||||
f"{em_dashes} em dashes",
|
||||
)
|
||||
)
|
||||
not_just = NOT_JUST_PATTERN.search(context.text)
|
||||
if not_just:
|
||||
line = context.text.count("\n", 0, not_just.start()) + 1
|
||||
findings.append(
|
||||
Signal("AI_PROSE_PATTERN", "Characteristic AI prose construction", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, line, not_just.group(0))
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,138 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
BARE_EXCEPT_PATTERN = re.compile(r"^\s*except\s*:\s*(#.*)?$")
|
||||
BROAD_EXCEPT_PATTERN = re.compile(r"^\s*except\s+(?:Exception|BaseException)\s*(?:as\s+\w+)?\s*:\s*(#.*)?$")
|
||||
PASS_PATTERN = re.compile(r"^\s*pass\s*(#.*)?$")
|
||||
EMPTY_CATCH_PATTERN = re.compile(r"catch\s*(\([^)]*\))?\s*\{\s*\}")
|
||||
LOG_ONLY_CATCH_PATTERN = re.compile(r"catch\s*\(\s*(\w+)\s*\)\s*\{\s*console\.(log|error)\([^)]*\);?\s*\}")
|
||||
DEBUGGER_PATTERN = re.compile(r"^\s*debugger\s*;?\s*$")
|
||||
TODO_REMOVE_PATTERN = re.compile(r"TODO:? remove", re.IGNORECASE)
|
||||
CONSOLE_LOG_PATTERN = re.compile(r"\bconsole\.log\s*\(")
|
||||
PRINT_PATTERN = re.compile(r"^\s*print\s*\(")
|
||||
|
||||
CONSOLE_LOG_THRESHOLD: int = 6
|
||||
PRINT_DENSITY_THRESHOLD: float = 0.06
|
||||
PRINT_MIN_SLOC: int = 60
|
||||
|
||||
|
||||
def _python_swallowed(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for index, line in enumerate(context.lines):
|
||||
bare = BARE_EXCEPT_PATTERN.match(line)
|
||||
broad = BROAD_EXCEPT_PATTERN.match(line)
|
||||
if not bare and not broad:
|
||||
continue
|
||||
follow = ""
|
||||
for candidate in context.lines[index + 1: index + 3]:
|
||||
if candidate.strip():
|
||||
follow = candidate
|
||||
break
|
||||
swallows = bool(PASS_PATTERN.match(follow))
|
||||
if bare:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR" if swallows else "BARE_EXCEPT",
|
||||
"Bare except swallowing all errors" if swallows else "Bare except clause",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
index + 1,
|
||||
line.strip(),
|
||||
)
|
||||
)
|
||||
elif swallows:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR",
|
||||
"except Exception: pass swallows all errors",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
index + 1,
|
||||
f"{line.strip()} / pass",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_error_handling(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
if context.language == "python":
|
||||
findings.extend(_python_swallowed(context))
|
||||
metrics = context.metrics
|
||||
if metrics is not None and metrics.sloc >= PRINT_MIN_SLOC:
|
||||
prints = sum(1 for line in context.lines if PRINT_PATTERN.match(line))
|
||||
if prints / max(metrics.sloc, 1) > PRINT_DENSITY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_LEFTOVERS",
|
||||
f"{prints} print() calls scattered through module",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"{prints} print calls in {metrics.sloc} SLOC",
|
||||
)
|
||||
)
|
||||
if context.language in ("javascript", "typescript", "java", "php", "csharp"):
|
||||
for match in EMPTY_CATCH_PATTERN.finditer(context.text):
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR",
|
||||
"Empty catch block swallows all errors",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
line,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
for match in LOG_ONLY_CATCH_PATTERN.finditer(context.text):
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"LOG_ONLY_CATCH",
|
||||
"Catch block only logs to console",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
line,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
console_hits = len(CONSOLE_LOG_PATTERN.findall(context.text))
|
||||
if console_hits >= CONSOLE_LOG_THRESHOLD and "test" not in context.relative.lower():
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_LEFTOVERS",
|
||||
f"{console_hits} console.log calls left in source",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"{console_hits} console.log calls",
|
||||
)
|
||||
)
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
if DEBUGGER_PATTERN.match(line):
|
||||
findings.append(
|
||||
Signal("DEBUG_LEFTOVERS", "debugger statement left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, line.strip())
|
||||
)
|
||||
for number, comment in context.comments:
|
||||
if TODO_REMOVE_PATTERN.search(comment):
|
||||
findings.append(
|
||||
Signal("DEBUG_LEFTOVERS", "TODO remove marker left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, comment)
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,139 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PYTHON_IMPORT_PATTERN = re.compile(r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))", re.MULTILINE)
|
||||
JS_IMPORT_PATTERN = re.compile(r"""(?:import[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]([^'"]+)['"]""")
|
||||
|
||||
PYTHON_IMPORT_ALIASES: dict[str, str] = {
|
||||
"pil": "pillow",
|
||||
"cv2": "opencv-python",
|
||||
"yaml": "pyyaml",
|
||||
"bs4": "beautifulsoup4",
|
||||
"dotenv": "python-dotenv",
|
||||
"sklearn": "scikit-learn",
|
||||
"dateutil": "python-dateutil",
|
||||
"jose": "python-jose",
|
||||
"jwt": "pyjwt",
|
||||
"magic": "python-magic",
|
||||
"serial": "pyserial",
|
||||
"usb": "pyusb",
|
||||
"redis": "redis",
|
||||
"attr": "attrs",
|
||||
"google": "google",
|
||||
"pkg_resources": "setuptools",
|
||||
"setuptools": "setuptools",
|
||||
}
|
||||
|
||||
NODE_BUILTINS: frozenset[str] = frozenset(
|
||||
{
|
||||
"assert", "async_hooks", "buffer", "child_process", "cluster", "console", "constants",
|
||||
"crypto", "dgram", "diagnostics_channel", "dns", "domain", "events", "fs", "http", "http2",
|
||||
"https", "inspector", "module", "net", "os", "path", "perf_hooks", "process", "punycode",
|
||||
"querystring", "readline", "repl", "sea", "sqlite", "stream", "string_decoder", "test",
|
||||
"timers", "tls", "trace_events", "tty", "url", "util", "v8", "vm", "wasi",
|
||||
"worker_threads", "zlib",
|
||||
}
|
||||
)
|
||||
|
||||
NON_PACKAGE_PREFIXES: tuple[str, ...] = ("@/", "~", "#", "$")
|
||||
|
||||
MAX_FLAGS_PER_FILE: int = 5
|
||||
|
||||
|
||||
def _normalize_package(name: str) -> str:
|
||||
return name.lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
|
||||
def _python_findings(context: FileContext) -> list[Signal]:
|
||||
repo = context.repo
|
||||
if repo is None or not repo.has_python_manifest:
|
||||
return []
|
||||
known = {_normalize_package(dep) for dep in repo.python_dependencies}
|
||||
local = {_normalize_package(module) for module in repo.local_python_modules}
|
||||
findings: list[Signal] = []
|
||||
seen: set[str] = set()
|
||||
for match in PYTHON_IMPORT_PATTERN.finditer(context.text):
|
||||
module = (match.group(1) or match.group(2) or "").split(".")[0]
|
||||
if not module or module in seen:
|
||||
continue
|
||||
seen.add(module)
|
||||
normalized = _normalize_package(module)
|
||||
aliased = _normalize_package(PYTHON_IMPORT_ALIASES.get(normalized, normalized))
|
||||
if normalized in sys.stdlib_module_names or module in sys.stdlib_module_names:
|
||||
continue
|
||||
if normalized in known or aliased in known or normalized in local:
|
||||
continue
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEP_UNRESOLVED",
|
||||
f"Import '{module}' resolves to no manifest dependency, stdlib or local module",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
line,
|
||||
match.group(0).strip(),
|
||||
)
|
||||
)
|
||||
return findings[:MAX_FLAGS_PER_FILE]
|
||||
|
||||
|
||||
def _javascript_findings(context: FileContext) -> list[Signal]:
|
||||
repo = context.repo
|
||||
if repo is None or not repo.has_javascript_manifest:
|
||||
return []
|
||||
known = {dep.lower() for dep in repo.javascript_dependencies}
|
||||
findings: list[Signal] = []
|
||||
seen: set[str] = set()
|
||||
aliases = tuple(repo.javascript_alias_prefixes)
|
||||
for match in JS_IMPORT_PATTERN.finditer(context.text):
|
||||
target = match.group(1)
|
||||
if target.startswith((".", "/", "http:", "https:")):
|
||||
continue
|
||||
if target.startswith(NON_PACKAGE_PREFIXES):
|
||||
continue
|
||||
if aliases and target.startswith(aliases):
|
||||
continue
|
||||
stripped = target.removeprefix("node:")
|
||||
parts = stripped.split("/")
|
||||
package = "/".join(parts[:2]) if stripped.startswith("@") and len(parts) >= 2 else parts[0]
|
||||
if package in seen:
|
||||
continue
|
||||
seen.add(package)
|
||||
if ":" in package:
|
||||
continue
|
||||
if package.lower() in NODE_BUILTINS or target.startswith("node:"):
|
||||
continue
|
||||
if package.lower() in known:
|
||||
continue
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEP_UNRESOLVED",
|
||||
f"Import '{package}' is not declared in package.json",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
line,
|
||||
match.group(0).strip()[:120],
|
||||
)
|
||||
)
|
||||
return findings[:MAX_FLAGS_PER_FILE]
|
||||
|
||||
|
||||
def detect_hallucination(context: FileContext) -> list[Signal]:
|
||||
if context.language == "python":
|
||||
return _python_findings(context)
|
||||
if context.language in ("javascript", "typescript"):
|
||||
return _javascript_findings(context)
|
||||
return []
|
||||
@@ -0,0 +1,96 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PATH_FRAMEWORK_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(re.compile(r"(^|/)_next/", re.IGNORECASE), "Next.js build output (_next/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)_nuxt/", re.IGNORECASE), "Nuxt build output (_nuxt/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)_astro/", re.IGNORECASE), "Astro build output (_astro/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)\.svelte-kit/", re.IGNORECASE), "SvelteKit build output", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)webpack-[0-9a-f]{8,}\.js$", re.IGNORECASE), "Webpack runtime chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)polyfills-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Bundler polyfills chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)main-app-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Next.js app-router bundle", SEVERITY_MEDIUM),
|
||||
(re.compile(r"(^|/)chunks/app/(page|layout|loading|error|not-found)-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Next.js app-router page bundle", SEVERITY_MEDIUM),
|
||||
(re.compile(r"(^|/)framework-[0-9a-f]{8,}\.js$", re.IGNORECASE), "Next.js framework chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)assets/index-[0-9A-Za-z_-]{8,}\.(js|css)$"), "Vite hashed asset bundle", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)gptengineer\.js$|(^|/)cdn\.gpteng\.co", re.IGNORECASE), "GPT Engineer / Lovable runtime", SEVERITY_STRONG),
|
||||
(re.compile(r"(^|/)lovable-uploads/", re.IGNORECASE), "Lovable uploads directory", SEVERITY_STRONG),
|
||||
(re.compile(r"(^|/)gatsby-", re.IGNORECASE), "Gatsby build artifact", SEVERITY_WEAK),
|
||||
)
|
||||
|
||||
VITE_HASH_ASSET = re.compile(r"index-[0-9A-Za-z_-]{8,}\.(js|css)$")
|
||||
|
||||
CONTENT_FRAMEWORK_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"self\.__next_f|__NEXT_DATA__|next/dist/", re.IGNORECASE), "next.js runtime"),
|
||||
(re.compile(r"__nuxt|nuxt\.config", re.IGNORECASE), "nuxt runtime"),
|
||||
(re.compile(r"astro-island|astro:", re.IGNORECASE), "astro runtime"),
|
||||
(re.compile(r"data-radix|@radix-ui|radix-ui", re.IGNORECASE), "radix primitives"),
|
||||
(re.compile(r"lucide", re.IGNORECASE), "lucide icons"),
|
||||
(re.compile(r"shadcn|class-variance-authority|\bcn\(", re.IGNORECASE), "shadcn/ui"),
|
||||
(re.compile(r"tailwind|--tw-", re.IGNORECASE), "tailwind"),
|
||||
(re.compile(r"framer-motion", re.IGNORECASE), "framer motion"),
|
||||
(re.compile(r"createClient\([^)]*supabase|supabase\.co", re.IGNORECASE), "supabase backend"),
|
||||
(re.compile(r"clerk\.|@clerk/", re.IGNORECASE), "clerk auth"),
|
||||
)
|
||||
|
||||
CONTENT_STRONG_MARKERS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"gpteng|lovable|data-lov-id", re.IGNORECASE), "Lovable/GPT-Engineer marker"),
|
||||
(re.compile(r"generated by (v0|bolt|base44|lovable)", re.IGNORECASE), "AI builder attribution"),
|
||||
)
|
||||
|
||||
VIBE_COMBO_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def detect_infrastructure(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
relative = context.relative
|
||||
|
||||
for pattern, title, severity in PATH_FRAMEWORK_PATTERNS:
|
||||
if pattern.search(relative):
|
||||
weight = 3.0 if severity == SEVERITY_STRONG else (1.0 if severity == SEVERITY_MEDIUM else 0.5)
|
||||
findings.append(
|
||||
Signal("BUILD_ARTIFACT_PATH", f"Framework build artifact path: {title}", severity, AXIS_ORIGIN, weight, 1, relative)
|
||||
)
|
||||
break
|
||||
|
||||
text = context.text
|
||||
for pattern, marker in CONTENT_STRONG_MARKERS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal(
|
||||
"AI_BUILDER_FINGERPRINT",
|
||||
f"AI builder marker in bundle: {marker}",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
4.0,
|
||||
text.count("\n", 0, match.start()) + 1,
|
||||
match.group(0)[:100],
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
stack_hits = [label for pattern, label in CONTENT_FRAMEWORK_PATTERNS if pattern.search(text)]
|
||||
if len(stack_hits) >= VIBE_COMBO_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"VIBE_STACK",
|
||||
f"Default AI frontend stack in bundle ({len(stack_hits)} libraries)",
|
||||
SEVERITY_STRONG if len(stack_hits) >= 5 else SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(stack_hits[:8]),
|
||||
)
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
VAR_PATTERN = re.compile(r"^\s*var\s+\w+")
|
||||
MODERN_JS_PATTERN = re.compile(r"\b(?:const|let|=>|class\s+\w+|import\s)")
|
||||
USE_EFFECT_FETCH_PATTERN = re.compile(r"useEffect\s*\(\s*(?:async\s*)?\(\s*\)\s*=>\s*\{[^}]{0,400}\bfetch\s*\(", re.DOTALL)
|
||||
ANY_TYPE_PATTERN = re.compile(r":\s*any\b")
|
||||
OPEN_WITHOUT_WITH_PATTERN = re.compile(r"^\s*\w+\s*=\s*open\s*\(")
|
||||
BROAD_EXCEPT_COUNT_PATTERN = re.compile(r"^\s*except\s+Exception\b", re.MULTILINE)
|
||||
INLINE_STYLE_PATTERN = re.compile(r"\sstyle\s*=\s*\"")
|
||||
|
||||
VAR_THRESHOLD: int = 3
|
||||
ANY_THRESHOLD: int = 5
|
||||
OPEN_THRESHOLD: int = 2
|
||||
BROAD_EXCEPT_THRESHOLD: int = 3
|
||||
INLINE_STYLE_THRESHOLD: int = 10
|
||||
|
||||
|
||||
def detect_language_tells(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
if context.language in ("javascript", "typescript"):
|
||||
var_lines = [number for number, line in enumerate(context.lines, start=1) if VAR_PATTERN.match(line)]
|
||||
if len(var_lines) >= VAR_THRESHOLD and MODERN_JS_PATTERN.search(context.text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"LEGACY_VAR",
|
||||
f"{len(var_lines)} var declarations mixed into modern code",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
var_lines[0],
|
||||
context.lines[var_lines[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
effect_match = USE_EFFECT_FETCH_PATTERN.search(context.text)
|
||||
if effect_match:
|
||||
line = context.text.count("\n", 0, effect_match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"USEEFFECT_FETCH",
|
||||
"Data fetching inside useEffect without caching or retry",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
line,
|
||||
effect_match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
if context.language == "typescript":
|
||||
any_hits = ANY_TYPE_PATTERN.findall(context.text)
|
||||
if len(any_hits) >= ANY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("ANY_TYPES", f"{len(any_hits)} explicit any types", SEVERITY_MEDIUM, AXIS_QUALITY, 1.0, 1, f"{len(any_hits)} occurrences of : any")
|
||||
)
|
||||
if context.language == "python":
|
||||
open_lines = [number for number, line in enumerate(context.lines, start=1) if OPEN_WITHOUT_WITH_PATTERN.match(line)]
|
||||
if len(open_lines) >= OPEN_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"NO_CONTEXT_MANAGER",
|
||||
f"{len(open_lines)} open() calls without context manager",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
open_lines[0],
|
||||
context.lines[open_lines[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
broad = BROAD_EXCEPT_COUNT_PATTERN.findall(context.text)
|
||||
if len(broad) >= BROAD_EXCEPT_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_EXCEPTIONS",
|
||||
f"{len(broad)} undifferentiated except Exception handlers",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(broad)} broad handlers",
|
||||
)
|
||||
)
|
||||
if context.language == "html":
|
||||
inline_styles = INLINE_STYLE_PATTERN.findall(context.text)
|
||||
if len(inline_styles) >= INLINE_STYLE_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("INLINE_STYLES", f"{len(inline_styles)} inline style attributes", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, f"{len(inline_styles)} inline styles")
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,191 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CODE_LANGUAGES_FOR_DEFAULTS: frozenset[str] = frozenset(
|
||||
{"python", "javascript", "typescript", "php", "go", "java", "ruby", "csharp", "yaml", "json", "html"}
|
||||
)
|
||||
|
||||
DEFAULT_SECRET_PATTERN = re.compile(
|
||||
r"(secret|jwt[_-]?secret|secret[_-]?key|signing[_-]?key|session[_-]?secret)\w*\s*[:=]\s*"
|
||||
r"[\"'](your[_-]secret[_-]key(?:[_-]here)?|your[_-]super[_-]secret|supersecret\w*|super[_-]secret"
|
||||
r"|my[_-]?secret(?:[_-]?key)?|secret(?:123)?|secret[_-]key|change[_-]?(this|me)\w*|keyboard cat)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
WILDCARD_CORS_CREDENTIALS_PATTERN = re.compile(
|
||||
r"allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\][\s\S]{0,200}allow_credentials\s*=\s*True"
|
||||
r"|allow_credentials\s*=\s*True[\s\S]{0,200}allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\]"
|
||||
r"|credentials\s*:\s*true[\s\S]{0,120}origin\s*:\s*[\"']\*[\"']"
|
||||
r"|origin\s*:\s*[\"']\*[\"'][\s\S]{0,120}credentials\s*:\s*true",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
WILDCARD_CORS_PATTERN = re.compile(
|
||||
r"allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\]"
|
||||
r"|Access-Control-Allow-Origin[\"']?\s*[,:]\s*[\"']\*[\"']"
|
||||
r"|app\.use\(\s*cors\(\s*\)\s*\)"
|
||||
r"|origin\s*:\s*[\"']\*[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DEBUG_ENABLED_PATTERN = re.compile(
|
||||
r"\.run\([^)]*debug\s*=\s*True|^\s*DEBUG\s*=\s*True\s*$|debug\s*:\s*true\s*,?\s*//|APP_DEBUG=true",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
SEED_CREDENTIAL_PATTERN = re.compile(
|
||||
r"[\"'](admin123|password123|test123|admin@example\.com|test@test\.com|admin@admin\.com|letmein|qwerty123)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DEFAULT_DATABASE_PATTERN = re.compile(
|
||||
r"mongodb(\+srv)?://localhost(:27017)?/(mydb|myapp|test|mydatabase|app)"
|
||||
r"|(postgres(ql)?|mysql)://(user|root|admin|postgres):(password|pass|root|admin|secret)@localhost",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
TUTORIAL_SERVER_PATTERN = re.compile(
|
||||
r"console\.log\(\s*[\"'`]Server (is )?(running|listening|started) (on|at) (port |http)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
APP_NAME_DEFAULT_PATTERN = re.compile(
|
||||
r"[\"'](my-?app|my-?project|test-?app|awesome-?project|my-?website)[\"']\s*[,:]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
MAX_FINDINGS: int = 10
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def detect_llm_defaults(context: FileContext) -> list[Signal]:
|
||||
if context.language not in CODE_LANGUAGES_FOR_DEFAULTS:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
|
||||
for match in DEFAULT_SECRET_PATTERN.finditer(text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_DEFAULT_SECRET",
|
||||
f"Stock LLM placeholder secret '{match.group(2)}' left in configuration",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
credentials_combo = WILDCARD_CORS_CREDENTIALS_PATTERN.search(text)
|
||||
if credentials_combo:
|
||||
findings.append(
|
||||
Signal(
|
||||
"WILDCARD_CORS_CREDENTIALS",
|
||||
"Wildcard CORS origin combined with credentials, the canonical LLM scaffold vulnerability",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
_line_of(text, credentials_combo.start()),
|
||||
credentials_combo.group(0)[:120],
|
||||
)
|
||||
)
|
||||
else:
|
||||
wildcard = WILDCARD_CORS_PATTERN.search(text)
|
||||
if wildcard:
|
||||
findings.append(
|
||||
Signal(
|
||||
"WILDCARD_CORS",
|
||||
"Wildcard CORS configuration, the default LLM scaffold",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, wildcard.start()),
|
||||
wildcard.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
debug = DEBUG_ENABLED_PATTERN.search(text)
|
||||
if debug:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_ENABLED",
|
||||
"Debug mode committed enabled, the tutorial default",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, debug.start()),
|
||||
debug.group(0).strip()[:120],
|
||||
)
|
||||
)
|
||||
|
||||
for match in SEED_CREDENTIAL_PATTERN.finditer(text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"SEED_CREDENTIALS",
|
||||
f"Stock demo credential {match.group(1)} in source",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
database = DEFAULT_DATABASE_PATTERN.search(text)
|
||||
if database:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEFAULT_DATABASE_URL",
|
||||
"Tutorial-default database connection string",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, database.start()),
|
||||
database.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
tutorial = TUTORIAL_SERVER_PATTERN.search(text)
|
||||
if tutorial:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TUTORIAL_SERVER_LOG",
|
||||
"Tutorial-style server startup console.log",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, tutorial.start()),
|
||||
tutorial.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
app_name = APP_NAME_DEFAULT_PATTERN.search(text)
|
||||
if app_name:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEFAULT_PROJECT_NAME",
|
||||
f"Default scaffold project name {app_name.group(1)}",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, app_name.start()),
|
||||
app_name.group(0)[:80],
|
||||
)
|
||||
)
|
||||
|
||||
return findings[:MAX_FINDINGS]
|
||||
@@ -0,0 +1,115 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DECLARATION_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||
"python": (
|
||||
re.compile(r"^\s*(?:async\s+)?def\s+(\w+)"),
|
||||
re.compile(r"^\s*class\s+(\w+)"),
|
||||
re.compile(r"^\s*([a-zA-Z_]\w*)\s*(?::[^=]+)?=\s*[^=]"),
|
||||
),
|
||||
"javascript": (
|
||||
re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)"),
|
||||
re.compile(r"^\s*(?:export\s+)?class\s+(\w+)"),
|
||||
re.compile(r"^\s*(?:const|let|var)\s+(\w+)"),
|
||||
),
|
||||
}
|
||||
DECLARATION_PATTERNS["typescript"] = DECLARATION_PATTERNS["javascript"]
|
||||
|
||||
GENERIC_IDENTIFIERS: frozenset[str] = frozenset(
|
||||
{
|
||||
"data", "result", "res", "temp", "tmp", "item", "items", "value", "values", "val",
|
||||
"obj", "object", "thing", "stuff", "info", "foo", "bar", "baz", "output", "input",
|
||||
"myfunction", "myvar", "mydata", "example", "processdata", "dosomething", "handledata",
|
||||
"handleclick", "handlesubmit", "handlechange", "mycomponent", "untitled", "test1", "func",
|
||||
"arr", "lst", "dct", "retval", "resp", "ret",
|
||||
}
|
||||
)
|
||||
|
||||
VERBOSE_MIN_LENGTH: int = 30
|
||||
VERBOSE_MIN_WORDS: int = 5
|
||||
GENERIC_DENSITY_THRESHOLD: float = 0.30
|
||||
MIN_DECLARATIONS: int = 5
|
||||
|
||||
|
||||
def _split_words(identifier: str) -> list[str]:
|
||||
parts = re.split(r"_+", identifier)
|
||||
words: list[str] = []
|
||||
for part in parts:
|
||||
words.extend(re.findall(r"[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])", part))
|
||||
return [word for word in words if word]
|
||||
|
||||
|
||||
def detect_naming(context: FileContext) -> list[Signal]:
|
||||
patterns = DECLARATION_PATTERNS.get(context.language)
|
||||
if not patterns:
|
||||
return []
|
||||
declarations: list[tuple[int, str]] = []
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern in patterns:
|
||||
match = pattern.match(line)
|
||||
if match:
|
||||
name = next(group for group in match.groups() if group)
|
||||
declarations.append((number, name))
|
||||
break
|
||||
if len(declarations) < MIN_DECLARATIONS:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
generic = [(number, name) for number, name in declarations if name.lower().replace("_", "") in GENERIC_IDENTIFIERS]
|
||||
density = len(generic) / len(declarations)
|
||||
if density >= GENERIC_DENSITY_THRESHOLD:
|
||||
number, name = generic[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_NAMING",
|
||||
f"Generic identifiers in {density:.0%} of {len(declarations)} declarations",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
number,
|
||||
name,
|
||||
)
|
||||
)
|
||||
verbose = [
|
||||
(number, name)
|
||||
for number, name in declarations
|
||||
if len(name) >= VERBOSE_MIN_LENGTH and len(_split_words(name)) >= VERBOSE_MIN_WORDS
|
||||
]
|
||||
if verbose:
|
||||
number, name = verbose[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"TEXTBOOK_NAMING",
|
||||
"Over-verbose textbook identifier",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
number,
|
||||
name,
|
||||
)
|
||||
)
|
||||
snake = sum(1 for _, name in declarations if "_" in name and name.lower() == name)
|
||||
camel = sum(1 for _, name in declarations if "_" not in name and name.lower() != name and name[0].islower())
|
||||
total = len(declarations)
|
||||
if snake >= 3 and camel >= 3 and snake / total >= 0.25 and camel / total >= 0.25:
|
||||
findings.append(
|
||||
Signal(
|
||||
"NAMING_MIXED",
|
||||
f"Mixed snake_case ({snake}) and camelCase ({camel}) declarations",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
declarations[0][0],
|
||||
f"{snake} snake_case vs {camel} camelCase",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HTML_SCAFFOLD_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(re.compile(r"Generated by create next app", re.IGNORECASE), "Untouched create-next-app metadata", SEVERITY_STRONG),
|
||||
(re.compile(r"<title>\s*Create Next App\s*</title>", re.IGNORECASE), "Default Create Next App title", SEVERITY_STRONG),
|
||||
(re.compile(r"<title>\s*(Vite \+ React|Vite App|React App)\s*</title>", re.IGNORECASE), "Default Vite/CRA title", SEVERITY_STRONG),
|
||||
(re.compile(r"You need to enable JavaScript to run this app", re.IGNORECASE), "Create React App noscript boilerplate", SEVERITY_STRONG),
|
||||
(re.compile(r"/vite\.svg|src=[\"']/?vite\.svg", re.IGNORECASE), "Default Vite logo asset", SEVERITY_MEDIUM),
|
||||
(re.compile(r"<div\s+id=[\"']root[\"']\s*>\s*</div>", re.IGNORECASE), "Empty CRA/Vite root mount div", SEVERITY_MEDIUM),
|
||||
(re.compile(r"<div\s+id=[\"']__next[\"']", re.IGNORECASE), "Next.js hydration root", SEVERITY_WEAK),
|
||||
(re.compile(r"src=[\"']/src/main\.(jsx|tsx)[\"']", re.IGNORECASE), "Default Vite entry module", SEVERITY_MEDIUM),
|
||||
(re.compile(r"Edit <code>src/App", re.IGNORECASE), "Untouched CRA starter copy", SEVERITY_STRONG),
|
||||
(re.compile(r"Save to reload", re.IGNORECASE), "Untouched Vite/CRA starter copy", SEVERITY_STRONG),
|
||||
(re.compile(r"content=[\"']Web site created using create-react-app", re.IGNORECASE), "CRA description meta", SEVERITY_STRONG),
|
||||
(re.compile(r"%PUBLIC_URL%", re.IGNORECASE), "Unrendered CRA template token", SEVERITY_STRONG),
|
||||
(re.compile(r"__NEXT_DATA__|self\.__next_f", re.IGNORECASE), "Next.js runtime payload", SEVERITY_WEAK),
|
||||
(re.compile(r"<meta[^>]+name=[\"']generator[\"'][^>]+(gatsby|hugo|jekyll|astro|next\.js|nuxt|docusaurus)", re.IGNORECASE), "Static-site generator meta", SEVERITY_WEAK),
|
||||
)
|
||||
|
||||
PY_DOCSTRING_TRIVIAL_PATTERN = re.compile(
|
||||
r"def\s+\w+\([^)]*\)\s*(?:->[^\n:]+)?:\s*\n\s*(?:\"\"\"|''')[^\n]{5,80}(?:\"\"\"|''')\s*\n\s*return\b",
|
||||
re.MULTILINE,
|
||||
)
|
||||
PY_VERBOSE_NAME_PATTERN = re.compile(r"\b[a-z]+(?:_[a-z]+){4,}\b\s*=")
|
||||
PY_ERROR_PRINT_PATTERN = re.compile(
|
||||
r"except\s+\w*(?:Exception|Error)?\s*(?:as\s+(\w+))?\s*:\s*\n\s*print\(\s*f?[\"'](?:an?\s+)?error", re.IGNORECASE
|
||||
)
|
||||
PY_MAIN_GUARD_PATTERN = re.compile(r"^if\s+__name__\s*==\s*[\"']__main__[\"']\s*:", re.MULTILINE)
|
||||
PY_TYPE_HINT_PATTERN = re.compile(r"def\s+\w+\([^)]*:\s*\w+[^)]*\)\s*->\s*\w+")
|
||||
PY_FULL_DOCSTRING_PATTERN = re.compile(r"def\s+\w+[^:]*:\s*\n\s*(?:\"\"\"|''')")
|
||||
PY_LOGGING_FSTRING_PATTERN = re.compile(r"logging\.(info|debug|warning|error)\(\s*f[\"']")
|
||||
|
||||
VERBOSE_NAME_THRESHOLD: int = 2
|
||||
TRIVIAL_DOCSTRING_THRESHOLD: int = 2
|
||||
TEXTBOOK_MIN_FUNCTIONS: int = 5
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def _html_scaffold_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
for pattern, title, severity in HTML_SCAFFOLD_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
weight = 4.0 if severity == SEVERITY_STRONG else (2.0 if severity == SEVERITY_MEDIUM else 0.5)
|
||||
findings.append(
|
||||
Signal("FRAMEWORK_SCAFFOLD", f"Framework scaffold artifact: {title}", severity, AXIS_ORIGIN, weight, _line_of(text, match.start()), match.group(0)[:100])
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def _python_structure_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
metrics = context.metrics
|
||||
|
||||
trivial = PY_DOCSTRING_TRIVIAL_PATTERN.findall(text)
|
||||
if len(trivial) >= TRIVIAL_DOCSTRING_THRESHOLD:
|
||||
match = PY_DOCSTRING_TRIVIAL_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"TRIVIAL_DOCSTRINGS",
|
||||
f"{len(trivial)} trivial functions with textbook docstrings",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
"over-documented one-line functions",
|
||||
)
|
||||
)
|
||||
|
||||
verbose = PY_VERBOSE_NAME_PATTERN.findall(text)
|
||||
if len(verbose) >= VERBOSE_NAME_THRESHOLD:
|
||||
match = PY_VERBOSE_NAME_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"VERBOSE_TEXTBOOK_NAMES",
|
||||
f"{len(verbose)} extremely verbose textbook variable names",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
match.group(0).strip()[:80] if match else "",
|
||||
)
|
||||
)
|
||||
|
||||
error_print = PY_ERROR_PRINT_PATTERN.search(text)
|
||||
if error_print:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_ERROR_PRINT",
|
||||
"Textbook 'except: print(f\"Error: {e}\")' handling",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, error_print.start()),
|
||||
error_print.group(0).strip()[:80],
|
||||
)
|
||||
)
|
||||
|
||||
if metrics is not None and metrics.function_count >= TEXTBOOK_MIN_FUNCTIONS:
|
||||
typed = len(PY_TYPE_HINT_PATTERN.findall(text))
|
||||
documented = len(PY_FULL_DOCSTRING_PATTERN.findall(text))
|
||||
has_main_guard = bool(PY_MAIN_GUARD_PATTERN.search(text))
|
||||
logging_fstring = bool(PY_LOGGING_FSTRING_PATTERN.search(text))
|
||||
markers = sum(
|
||||
[
|
||||
typed >= max(3, metrics.function_count // 2),
|
||||
documented >= metrics.function_count,
|
||||
has_main_guard,
|
||||
logging_fstring,
|
||||
]
|
||||
)
|
||||
if markers >= 3:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TEXTBOOK_STRUCTURE",
|
||||
"Uniformly textbook structure: full type hints, docstring on every function, main guard",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
f"typed={typed}, docstrings={documented}, main_guard={has_main_guard}",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_scaffolds(context: FileContext) -> list[Signal]:
|
||||
if context.language == "html":
|
||||
return _html_scaffold_signals(context)
|
||||
if context.language == "python":
|
||||
return _python_structure_signals(context)
|
||||
return []
|
||||
@@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import shannon_entropy
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "AWS access key"),
|
||||
(re.compile(r"\bghp_[A-Za-z0-9]{36,}\b"), "GitHub personal token"),
|
||||
(re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}\b"), "GitHub fine-grained token"),
|
||||
(re.compile(r"\bsk_live_[A-Za-z0-9]{16,}\b"), "Stripe live key"),
|
||||
(re.compile(r"\bsk-[A-Za-z0-9]{40,}\b"), "API secret key"),
|
||||
(re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b"), "Google API key"),
|
||||
(re.compile(r"\bxox[baprs]-[A-Za-z0-9\-]{10,}\b"), "Slack token"),
|
||||
(re.compile(r"-----BEGIN[A-Z ]*PRIVATE KEY-----"), "Private key material"),
|
||||
(re.compile(r"\b(?:postgres|postgresql|mysql|mongodb(?:\+srv)?|redis|amqp)://\w+:[^@/\s]{4,}@"), "Credentials in connection string"),
|
||||
)
|
||||
|
||||
ENTROPY_ASSIGNMENT_PATTERN = re.compile(
|
||||
r"(secret|token|passwd|password|api[_-]?key|auth[_-]?key|private[_-]?key)\w*\s*[:=]\s*[\"']([^\"'\s]{16,})[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ENTROPY_THRESHOLD: float = 3.5
|
||||
|
||||
ALLOWLIST_MARKERS: tuple[str, ...] = (
|
||||
"example", "your_", "your-", "changeme", "change_me", "placeholder", "xxxx", "dummy",
|
||||
"sample", "<", "${", "{{", "todo", "insert", "test", "fake", "redacted", "akiaiosfodnn7example",
|
||||
)
|
||||
ALLOWLIST_PATH_MARKERS: tuple[str, ...] = (".example", "fixture", "mock", "sample", "/docs/", "/test", "_test", "spec.")
|
||||
|
||||
INJECTION_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(
|
||||
re.compile(r"[\"'](?:SELECT|INSERT|UPDATE|DELETE|DROP)\b[^\"']*[\"']\s*\+", re.IGNORECASE),
|
||||
"SQL built by string concatenation",
|
||||
SEVERITY_STRONG,
|
||||
),
|
||||
(
|
||||
re.compile(r"f[\"'](?:[^\"']*\b(?:SELECT|INSERT|UPDATE|DELETE)\b[^\"']*\{)", re.IGNORECASE),
|
||||
"SQL built with f-string interpolation",
|
||||
SEVERITY_STRONG,
|
||||
),
|
||||
(re.compile(r"\beval\s*\(\s*[a-zA-Z_$][\w$.]*\s*[,)]"), "eval() on a variable", SEVERITY_STRONG),
|
||||
(re.compile(r"dangerouslySetInnerHTML"), "dangerouslySetInnerHTML usage", SEVERITY_STRONG),
|
||||
(re.compile(r"\.innerHTML\s*=\s*(?![\"'`][^$])"), "innerHTML assigned from dynamic value", SEVERITY_MEDIUM),
|
||||
(re.compile(r"document\.write\s*\("), "document.write usage", SEVERITY_MEDIUM),
|
||||
(re.compile(r"child_process[.\w]*\.exec(?:Sync)?\s*\(\s*[`\"'][^`\"']*[$+]"), "Shell exec with interpolation", SEVERITY_STRONG),
|
||||
(re.compile(r"os\.system\s*\(\s*f?[\"'][^\"']*[{+]"), "os.system with interpolation", SEVERITY_STRONG),
|
||||
(re.compile(r"subprocess\.\w+\([^)]*shell\s*=\s*True[^)]*[+{]"), "subprocess shell=True with dynamic input", SEVERITY_STRONG),
|
||||
)
|
||||
|
||||
|
||||
def _is_allowlisted(line: str, relative_path: str) -> bool:
|
||||
lowered = line.lower()
|
||||
if any(marker in lowered for marker in ALLOWLIST_MARKERS):
|
||||
return True
|
||||
lowered_path = relative_path.lower()
|
||||
return any(marker in lowered_path for marker in ALLOWLIST_PATH_MARKERS)
|
||||
|
||||
|
||||
def detect_security(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern, title in SECRET_PATTERNS:
|
||||
if pattern.search(line) and not _is_allowlisted(line, context.relative):
|
||||
findings.append(
|
||||
Signal("HARDCODED_SECRET", f"Hardcoded secret: {title}", SEVERITY_STRONG, AXIS_QUALITY, 5.0, number, line.strip()[:120])
|
||||
)
|
||||
break
|
||||
entropy_match = ENTROPY_ASSIGNMENT_PATTERN.search(line)
|
||||
if entropy_match and not _is_allowlisted(line, context.relative):
|
||||
candidate = entropy_match.group(2)
|
||||
if shannon_entropy(candidate) >= ENTROPY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"HARDCODED_SECRET",
|
||||
f"High-entropy credential assigned to '{entropy_match.group(1)}'",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
number,
|
||||
line.strip()[:120],
|
||||
)
|
||||
)
|
||||
for pattern, title, severity in INJECTION_PATTERNS:
|
||||
if pattern.search(line):
|
||||
findings.append(
|
||||
Signal("INJECTION_RISK", title, severity, AXIS_QUALITY, 5.0, number, line.strip()[:120])
|
||||
)
|
||||
break
|
||||
return findings[:12]
|
||||
@@ -0,0 +1,140 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DEAD_CODE_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\bif\s*\(\s*false\s*\)"), "if (false) branch"),
|
||||
(re.compile(r"^\s*if\s+False\s*:"), "if False: branch"),
|
||||
(re.compile(r"^\s*if\s+0\s*:"), "if 0: branch"),
|
||||
(re.compile(r"\bwhile\s*\(\s*false\s*\)"), "while (false) loop"),
|
||||
)
|
||||
|
||||
ABSTRACTION_SUFFIX_PATTERN = re.compile(
|
||||
r"class\s+\w*(Factory|Manager|Provider|Strategy|Handler|Wrapper|Helper|Service|Orchestrator|Coordinator|Impl)\b"
|
||||
)
|
||||
CLASS_PATTERN = re.compile(r"^\s*(?:export\s+)?(?:abstract\s+)?class\s+(\w+)", re.MULTILINE)
|
||||
|
||||
UNIFORM_INDENT_DEVIATION: float = 0.45
|
||||
UNIFORM_MIN_SLOC: int = 50
|
||||
GOD_MODULE_SLOC: int = 600
|
||||
GOD_MODULE_IMPORTS: int = 20
|
||||
DUPLICATION_THRESHOLD: float = 0.08
|
||||
LONG_FUNCTION_LINES: int = 120
|
||||
COMPLEXITY_THRESHOLD: int = 15
|
||||
ABSTRACTION_DENSITY: float = 0.5
|
||||
ABSTRACTION_MIN_CLASSES: int = 3
|
||||
|
||||
|
||||
def detect_structure(context: FileContext) -> list[Signal]:
|
||||
metrics = context.metrics
|
||||
if metrics is None:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
if metrics.sloc >= UNIFORM_MIN_SLOC and metrics.indent_variance < UNIFORM_INDENT_DEVIATION:
|
||||
findings.append(
|
||||
Signal(
|
||||
"MACHINE_REGULARITY",
|
||||
f"Unnaturally uniform whitespace (indent deviation {metrics.indent_variance:.2f})",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"sloc={metrics.sloc}",
|
||||
)
|
||||
)
|
||||
if (
|
||||
context.language == "python"
|
||||
and metrics.function_count >= 4
|
||||
and metrics.docstring_count >= metrics.function_count
|
||||
):
|
||||
findings.append(
|
||||
Signal(
|
||||
"DOCSTRING_UNIFORMITY",
|
||||
f"Every one of {metrics.function_count} functions carries a docstring template",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"{metrics.docstring_count}/{metrics.function_count} docstrings",
|
||||
)
|
||||
)
|
||||
if metrics.sloc > GOD_MODULE_SLOC and metrics.import_count > GOD_MODULE_IMPORTS:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GOD_MODULE",
|
||||
f"Module mixes many concerns ({metrics.sloc} SLOC, {metrics.import_count} imports)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
f"sloc={metrics.sloc} imports={metrics.import_count}",
|
||||
)
|
||||
)
|
||||
if metrics.duplication_ratio > DUPLICATION_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DUPLICATED_LOGIC",
|
||||
f"Duplicated block ratio {metrics.duplication_ratio:.1%}",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
f"windows duplicated: {metrics.duplication_ratio:.1%}",
|
||||
)
|
||||
)
|
||||
if metrics.max_function_length > LONG_FUNCTION_LINES:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LONG_FUNCTION",
|
||||
f"Function of {metrics.max_function_length} lines",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"max function length {metrics.max_function_length}",
|
||||
)
|
||||
)
|
||||
if metrics.cyclomatic_max_estimate > COMPLEXITY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"HIGH_COMPLEXITY",
|
||||
f"Estimated cyclomatic complexity {metrics.cyclomatic_max_estimate} per function",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"complexity estimate {metrics.cyclomatic_max_estimate}",
|
||||
)
|
||||
)
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern, title in DEAD_CODE_PATTERNS:
|
||||
if pattern.search(line):
|
||||
findings.append(
|
||||
Signal("DEAD_DEFENSIVE_CODE", f"Dead code: {title}", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, line.strip())
|
||||
)
|
||||
break
|
||||
classes = CLASS_PATTERN.findall(context.text)
|
||||
if len(classes) >= ABSTRACTION_MIN_CLASSES:
|
||||
pattern_classes = ABSTRACTION_SUFFIX_PATTERN.findall(context.text)
|
||||
if len(pattern_classes) / len(classes) >= ABSTRACTION_DENSITY:
|
||||
findings.append(
|
||||
Signal(
|
||||
"OVER_ABSTRACTION",
|
||||
f"{len(pattern_classes)} of {len(classes)} classes are pattern-suffixed abstractions",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(pattern_classes[:5]),
|
||||
)
|
||||
)
|
||||
return findings
|
||||
@@ -0,0 +1,198 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PLACEHOLDER_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"your (code|logic|implementation) here", re.IGNORECASE), "Placeholder body left in source"),
|
||||
(re.compile(r"\.\.\. ?rest of (the )?code", re.IGNORECASE), "Truncated generation marker"),
|
||||
(re.compile(r"\.\.\. ?(existing|other|previous) code", re.IGNORECASE), "Elided-code marker"),
|
||||
(re.compile(r"/\*\s*implementation\s*\*/", re.IGNORECASE), "Empty implementation stub"),
|
||||
(re.compile(r"TODO:? implement( this)?\b", re.IGNORECASE), "Unimplemented TODO stub"),
|
||||
(re.compile(r"TODO:? add (logic|implementation|code)", re.IGNORECASE), "Unimplemented TODO stub"),
|
||||
(re.compile(r"<!--\s*add content here\s*-->", re.IGNORECASE), "Placeholder HTML content"),
|
||||
(re.compile(r"rest omitted", re.IGNORECASE), "Truncated generation marker"),
|
||||
)
|
||||
|
||||
PROMPT_LEAK_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"as an ai (language )?model", re.IGNORECASE),
|
||||
re.compile(r"i (cannot|can't) (assist|help) with", re.IGNORECASE),
|
||||
re.compile(r"here('|’)?s the (updated|complete|revised|full|corrected) code", re.IGNORECASE),
|
||||
re.compile(r"\bcertainly!\s", re.IGNORECASE),
|
||||
re.compile(r"sure, here('s| is)", re.IGNORECASE),
|
||||
re.compile(r"i hope this helps", re.IGNORECASE),
|
||||
re.compile(r"let me know if (you|there)", re.IGNORECASE),
|
||||
re.compile(r"feel free to (adjust|modify|change|customize)", re.IGNORECASE),
|
||||
re.compile(r"note that this is a (simplified|basic) (example|implementation|version)", re.IGNORECASE),
|
||||
re.compile(r"in a (real|production)([- ]world)? (application|scenario|environment|setting)", re.IGNORECASE),
|
||||
re.compile(r"this is a placeholder", re.IGNORECASE),
|
||||
re.compile(r"replace .{1,40} with your (actual|own)", re.IGNORECASE),
|
||||
)
|
||||
|
||||
AI_SIGNATURE_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"generated (by|with) (chatgpt|copilot|claude|cursor|gemini|codeium|v0|bolt|ai)", re.IGNORECASE),
|
||||
re.compile(r"co-authored-by:\s*(claude|copilot)", re.IGNORECASE),
|
||||
re.compile(r"\U0001F916 generated with"),
|
||||
re.compile(r"created with the help of ai", re.IGNORECASE),
|
||||
re.compile(r"\bclaude code\b", re.IGNORECASE),
|
||||
re.compile(r"powered by (chatgpt|gpt-4|claude|gemini)", re.IGNORECASE),
|
||||
)
|
||||
|
||||
EMOJI_PATTERN = re.compile(
|
||||
"[\U0001F300-\U0001FAFF☀-➿⬀-⯿✅❌✨❗⭐]"
|
||||
)
|
||||
|
||||
ENTHUSIASM_PATTERN = re.compile(
|
||||
r"blazingly fast|seamlessly|robust and scalable|cutting[- ]edge|revolutioniz|effortlessly|supercharge|game[- ]chang",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
LLM_LEXICON_PATTERN = re.compile(
|
||||
r"\b(delve|delving|delves|showcas(?:e|es|ing)|pivotal|intricate|meticulous(?:ly)?|realm"
|
||||
r"|bolster(?:s|ing)?|garnered|underpins|seamless(?:ly)?|leverag(?:e|es|ing)|streamlin(?:e|es|ing)"
|
||||
r"|elevat(?:e|es|ing)|holistic|robust and|comprehensive suite|game[- ]chang(?:er|ing)"
|
||||
r"|it'?s (?:important|worth) (?:to note|noting)|keep in mind|as you can see|in this example)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
LLM_LEXICON_THRESHOLD: int = 3
|
||||
EM_DASH_THRESHOLD: int = 3
|
||||
|
||||
WORD_PATTERN = re.compile(r"[a-z][a-z0-9]+")
|
||||
NARRATION_MIN_OVERLAP: float = 0.55
|
||||
NARRATION_MIN_TOKENS: int = 2
|
||||
|
||||
|
||||
def _narration_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
line_lookup = {number: text for number, text in context.comments}
|
||||
for number, comment in context.comments:
|
||||
if number + 1 in line_lookup or number >= len(context.lines):
|
||||
continue
|
||||
comment_tokens = set(WORD_PATTERN.findall(comment.lower()))
|
||||
if len(comment_tokens) < NARRATION_MIN_TOKENS:
|
||||
continue
|
||||
code_line = context.lines[number].lower()
|
||||
code_tokens = set(WORD_PATTERN.findall(code_line))
|
||||
if not code_tokens:
|
||||
continue
|
||||
overlap = len(comment_tokens & code_tokens) / len(comment_tokens)
|
||||
if overlap >= NARRATION_MIN_OVERLAP:
|
||||
findings.append(
|
||||
Signal(
|
||||
code="COMMENT_NARRATION",
|
||||
title="Comment narrates the next line of code",
|
||||
severity=SEVERITY_MEDIUM,
|
||||
axis=AXIS_ORIGIN,
|
||||
weight=2.0,
|
||||
line=number,
|
||||
evidence=comment,
|
||||
)
|
||||
)
|
||||
return findings[:8]
|
||||
|
||||
|
||||
def detect_textual(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for number, raw in enumerate(context.lines, start=1):
|
||||
for pattern, title in PLACEHOLDER_PATTERNS:
|
||||
if pattern.search(raw):
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_COMMENT", title, SEVERITY_STRONG, AXIS_QUALITY, 4.0, number, raw.strip())
|
||||
)
|
||||
break
|
||||
for pattern in AI_SIGNATURE_PATTERNS:
|
||||
if pattern.search(raw):
|
||||
findings.append(
|
||||
Signal(
|
||||
"AI_SIGNATURE",
|
||||
"AI tool self-attribution signature",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
4.0,
|
||||
number,
|
||||
raw.strip(),
|
||||
)
|
||||
)
|
||||
break
|
||||
for number, comment in context.comments:
|
||||
for pattern in PROMPT_LEAK_PATTERNS:
|
||||
if pattern.search(comment):
|
||||
findings.append(
|
||||
Signal(
|
||||
"PROMPT_LEAK",
|
||||
"Assistant conversation leakage in committed source",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
break
|
||||
emoji_hits = [(number, comment) for number, comment in context.comments if EMOJI_PATTERN.search(comment)]
|
||||
if len(emoji_hits) >= 2:
|
||||
number, comment = emoji_hits[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"EMOJI_COMMENTS",
|
||||
f"Emoji used in {len(emoji_hits)} comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
enthusiasm_hits = [(number, comment) for number, comment in context.comments if ENTHUSIASM_PATTERN.search(comment)]
|
||||
if enthusiasm_hits:
|
||||
number, comment = enthusiasm_hits[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"MARKETING_TONE",
|
||||
"Marketing-style enthusiasm in comments",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
comment_text = " ".join(comment for _, comment in context.comments)
|
||||
lexicon_hits = {hit.lower() if isinstance(hit, str) else hit for hit in LLM_LEXICON_PATTERN.findall(comment_text)}
|
||||
if len(lexicon_hits) >= LLM_LEXICON_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_LEXICON",
|
||||
f"{len(lexicon_hits)} distinct LLM signature words in comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
context.comments[0][0] if context.comments else 1,
|
||||
", ".join(sorted(str(hit) for hit in lexicon_hits)[:6]),
|
||||
)
|
||||
)
|
||||
em_dashes = comment_text.count("\u2014")
|
||||
if em_dashes >= EM_DASH_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"EM_DASH_OVERUSE",
|
||||
f"{em_dashes} em dashes in comments, a strong LLM prose habit",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
context.comments[0][0] if context.comments else 1,
|
||||
f"{em_dashes} em dashes",
|
||||
)
|
||||
)
|
||||
findings.extend(_narration_signals(context))
|
||||
return findings
|
||||
@@ -0,0 +1,201 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
JS_LANGUAGES: frozenset[str] = frozenset({"javascript", "typescript"})
|
||||
|
||||
TOKEN_IN_STORAGE_PATTERN = re.compile(
|
||||
r"(?:local|session)Storage\.setItem\(\s*[\"'`][^\"'`]*(token|jwt|auth|access[_-]?token|refresh[_-]?token|api[_-]?key|secret|password|session)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
INSECURE_COOKIE_PATTERN = re.compile(r"(document\.cookie\s*=|res\.cookie\(|response\.set_cookie\(|setCookie\()", re.IGNORECASE)
|
||||
COOKIE_SECURITY_HINT = re.compile(r"httponly|secure|samesite", re.IGNORECASE)
|
||||
|
||||
USEEFFECT_BLOCK_PATTERN = re.compile(r"useEffect\s*\(\s*\(\s*\)\s*=>\s*\{", re.IGNORECASE)
|
||||
EFFECT_RESOURCE_PATTERN = re.compile(r"\b(setInterval|setTimeout|addEventListener|\.subscribe\(|\.on\()", re.IGNORECASE)
|
||||
EFFECT_CLEANUP_PATTERN = re.compile(r"return\s*(\(\s*\)\s*=>|function)")
|
||||
|
||||
CONDITIONAL_HOOK_PATTERN = re.compile(
|
||||
r"^\s*(if|for|while)\s*\([^)]*\)\s*\{[^}]{0,200}\buse(State|Effect|Ref|Memo|Callback|Context|Reducer)\s*\(",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
MIXED_CONTENT_PATTERN = re.compile(r"[\"'(]http://(?!localhost|127\.0\.0\.1|0\.0\.0\.0)[\w.-]+", re.IGNORECASE)
|
||||
JSON_PARSE_UNGUARDED_PATTERN = re.compile(r"JSON\.parse\(\s*(?:await\s+)?\w+(?:\.(?:text|body|data))?\s*\)")
|
||||
|
||||
ENV_ACCESS_PATTERN = re.compile(r"process\.env\.[A-Z0-9_]+")
|
||||
ENV_FALLBACK_HINT = re.compile(r"process\.env\.[A-Z0-9_]+\s*(\|\||\?\?|,|\))")
|
||||
PUBLIC_SECRET_PATTERN = re.compile(
|
||||
r"(NEXT_PUBLIC|VITE|REACT_APP|NUXT_PUBLIC)_[A-Z0-9_]*(SECRET|PRIVATE|SERVICE_ROLE|API_KEY|TOKEN|PASSWORD)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SUPABASE_SERVICE_ROLE_PATTERN = re.compile(r"service_role|SUPABASE_SERVICE_ROLE_KEY", re.IGNORECASE)
|
||||
|
||||
MISSING_ROUTE_GUARD_PATTERN = re.compile(r"(router\.(get|post|put|delete|patch)|app\.(get|post|put|delete|patch))\s*\(", re.IGNORECASE)
|
||||
|
||||
ENV_DENSITY_THRESHOLD: int = 4
|
||||
MAX_FINDINGS: int = 10
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def _effect_cleanup_findings(text: str) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for match in USEEFFECT_BLOCK_PATTERN.finditer(text):
|
||||
start = match.end()
|
||||
depth = 1
|
||||
index = start
|
||||
while index < len(text) and depth > 0:
|
||||
char = text[index]
|
||||
if char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
index += 1
|
||||
body = text[start:index]
|
||||
if EFFECT_RESOURCE_PATTERN.search(body) and not EFFECT_CLEANUP_PATTERN.search(body):
|
||||
findings.append(
|
||||
Signal(
|
||||
"MISSING_EFFECT_CLEANUP",
|
||||
"useEffect registers a timer or listener without a cleanup return",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:80],
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_vibe_errors(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
language = context.language
|
||||
|
||||
public_secret = PUBLIC_SECRET_PATTERN.search(text)
|
||||
if public_secret:
|
||||
findings.append(
|
||||
Signal(
|
||||
"PUBLIC_ENV_SECRET",
|
||||
"Secret exposed through a client-side public env variable",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, public_secret.start()),
|
||||
public_secret.group(0)[:100],
|
||||
)
|
||||
)
|
||||
|
||||
if language in JS_LANGUAGES or language == "html":
|
||||
token_storage = TOKEN_IN_STORAGE_PATTERN.search(text)
|
||||
if token_storage:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TOKEN_IN_WEB_STORAGE",
|
||||
"Auth token stored in localStorage or sessionStorage, exposed to XSS",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
_line_of(text, token_storage.start()),
|
||||
token_storage.group(0)[:100],
|
||||
)
|
||||
)
|
||||
cookie_match = INSECURE_COOKIE_PATTERN.search(text)
|
||||
if cookie_match:
|
||||
window = text[cookie_match.start(): cookie_match.start() + 300]
|
||||
if not COOKIE_SECURITY_HINT.search(window):
|
||||
findings.append(
|
||||
Signal(
|
||||
"INSECURE_COOKIE",
|
||||
"Cookie set without HttpOnly, Secure or SameSite flags",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, cookie_match.start()),
|
||||
cookie_match.group(0)[:80],
|
||||
)
|
||||
)
|
||||
mixed = MIXED_CONTENT_PATTERN.search(text)
|
||||
if mixed:
|
||||
findings.append(
|
||||
Signal(
|
||||
"MIXED_CONTENT",
|
||||
"Insecure http:// resource referenced",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, mixed.start()),
|
||||
mixed.group(0)[:80],
|
||||
)
|
||||
)
|
||||
|
||||
if language in JS_LANGUAGES:
|
||||
conditional_hook = CONDITIONAL_HOOK_PATTERN.search(text)
|
||||
if conditional_hook:
|
||||
findings.append(
|
||||
Signal(
|
||||
"CONDITIONAL_HOOK",
|
||||
"React hook called conditionally, violates rules of hooks",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, conditional_hook.start()),
|
||||
conditional_hook.group(0).strip()[:80],
|
||||
)
|
||||
)
|
||||
findings.extend(_effect_cleanup_findings(text))
|
||||
unguarded = JSON_PARSE_UNGUARDED_PATTERN.search(text)
|
||||
if unguarded and "try" not in text[max(0, unguarded.start() - 120): unguarded.start()]:
|
||||
findings.append(
|
||||
Signal(
|
||||
"UNGUARDED_JSON_PARSE",
|
||||
"JSON.parse on a response without a guard for non-JSON payloads",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, unguarded.start()),
|
||||
unguarded.group(0)[:80],
|
||||
)
|
||||
)
|
||||
if SUPABASE_SERVICE_ROLE_PATTERN.search(text) and PUBLIC_SECRET_PATTERN.search(text) is None:
|
||||
env_exposed = re.search(r"(NEXT_PUBLIC|VITE|REACT_APP)[A-Z0-9_]*[\s\S]{0,60}service_role", text, re.IGNORECASE)
|
||||
if env_exposed:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SUPABASE_SERVICE_ROLE_EXPOSED",
|
||||
"Supabase service_role key reachable from client code",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, env_exposed.start()),
|
||||
env_exposed.group(0)[:100],
|
||||
)
|
||||
)
|
||||
env_accesses = ENV_ACCESS_PATTERN.findall(text)
|
||||
env_fallbacks = ENV_FALLBACK_HINT.findall(text)
|
||||
if len(env_accesses) >= ENV_DENSITY_THRESHOLD and not env_fallbacks:
|
||||
findings.append(
|
||||
Signal(
|
||||
"ENV_NO_VALIDATION",
|
||||
f"{len(env_accesses)} process.env reads with no fallback or validation",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(env_accesses)} unvalidated env reads",
|
||||
)
|
||||
)
|
||||
|
||||
return findings[:MAX_FINDINGS]
|
||||
@@ -0,0 +1,293 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
WEB_LANGUAGES: frozenset[str] = frozenset({"html", "css", "javascript", "typescript", "markdown"})
|
||||
|
||||
BUILDER_FINGERPRINTS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"cdn\.gpteng\.co|gptengineer\.js", re.IGNORECASE), "GPT Engineer runtime script"),
|
||||
(re.compile(r"lovable-uploads|data-lov-id|lovable\.app", re.IGNORECASE), "Lovable builder artifact"),
|
||||
(re.compile(r"bolt\.new", re.IGNORECASE), "Bolt.new builder reference"),
|
||||
(re.compile(r"(made|built|created|generated)\s+with\s+(lovable|bolt|v0|base44|windsurf|cursor|replit agent)", re.IGNORECASE), "AI builder attribution"),
|
||||
(re.compile(r"<meta[^>]+generator[^>]+(lovable|bolt|v0|base44|framer|durable|10web|hostinger ai)", re.IGNORECASE), "AI site generator meta tag"),
|
||||
(re.compile(r"v0\.dev|vercel\.ai/v0", re.IGNORECASE), "Vercel v0 builder reference"),
|
||||
)
|
||||
|
||||
SIGNATURE_GRADIENT_PATTERN = re.compile(r"#667eea.{0,80}#764ba2|#764ba2.{0,80}#667eea", re.IGNORECASE | re.DOTALL)
|
||||
GENERIC_PURPLE_GRADIENT_PATTERN = re.compile(
|
||||
r"linear-gradient\([^)]*(#6366f1|#8b5cf6|#a855f7|#7c3aed|#4f46e5|#818cf8)[^)]*\)", re.IGNORECASE
|
||||
)
|
||||
|
||||
SECTION_COMMENT_PATTERN = re.compile(
|
||||
r"<!--\s*([\w][\w /&'-]{0,40}\s+section|hero|navigation|navbar|footer|header)\s*-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
EDIT_NARRATION_PATTERN = re.compile(
|
||||
r"<!--\s*(updated|added|new|fixed|changed|modified|improved|enhanced|revised)\b[^>]{0,60}-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VIBE_STACK_INDICATORS: tuple[tuple[re.Pattern[str], int, str], ...] = (
|
||||
(re.compile(r"lucide", re.IGNORECASE), 5, "lucide icons"),
|
||||
(re.compile(r"data-radix|radix-ui|\bradix\b", re.IGNORECASE), 5, "radix primitives"),
|
||||
(re.compile(r"shadcn", re.IGNORECASE), 1, "shadcn/ui"),
|
||||
(re.compile(r"__next|/_next/", re.IGNORECASE), 2, "next.js runtime"),
|
||||
(re.compile(r"\bInter\b"), 3, "inter font"),
|
||||
(re.compile(r"framer-motion|data-framer", re.IGNORECASE), 2, "framer motion"),
|
||||
(re.compile(r"class=\"[^\"]*flex[^\"]*items-center[^\"]*justify-center[^\"]*\""), 6, "tailwind utility soup"),
|
||||
(re.compile(r"animate-(fade|pulse|bounce|spin|in)", re.IGNORECASE), 8, "animation utilities"),
|
||||
)
|
||||
INLINE_MONOLITH_STYLE_PATTERN = re.compile(r"<style[^>]*>([\s\S]*?)</style>", re.IGNORECASE)
|
||||
INLINE_MONOLITH_SCRIPT_PATTERN = re.compile(r"<script(?![^>]*src)[^>]*>([\s\S]*?)</script>", re.IGNORECASE)
|
||||
MONOLITH_STYLE_MIN_LINES: int = 80
|
||||
MONOLITH_SCRIPT_MIN_LINES: int = 40
|
||||
LANDING_SECTION_PATTERN = re.compile(
|
||||
r"(?:class|id)\s*=\s*[\"'][^\"']*\b(hero|features|testimonials|pricing|faq|cta)\b", re.IGNORECASE
|
||||
)
|
||||
|
||||
PLACEHOLDER_CONTACT_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\b(john|jane)(\.doe)?@example\.com\b", re.IGNORECASE), "Placeholder person email"),
|
||||
(re.compile(r"\b(info|contact|hello|support|your)@(example|yourcompany|yourdomain|company|yoursite)\.(com|org)\b", re.IGNORECASE), "Placeholder contact email"),
|
||||
(re.compile(r"\(555\)\s?\d{3}[- ]\d{4}|\+1\s?\(?555\)?[- ]\d{3,4}[- ]\d{4}|\b555-01\d{2}\b"), "Placeholder 555 phone number"),
|
||||
(re.compile(r"\b123 Main (Street|St)\b", re.IGNORECASE), "Placeholder street address"),
|
||||
(re.compile(r"\bYour Company( Name)?\b"), "Placeholder company name"),
|
||||
)
|
||||
|
||||
LOREM_PATTERN = re.compile(r"lorem ipsum dolor", re.IGNORECASE)
|
||||
DEAD_LINK_PATTERN = re.compile(r"href\s*=\s*[\"']#[\"']")
|
||||
TAILWIND_CDN_PATTERN = re.compile(r"cdn\.tailwindcss\.com", re.IGNORECASE)
|
||||
FONT_STACK_PATTERN = re.compile(r"fonts\.googleapis\.com[^\"']*(Inter|Poppins)", re.IGNORECASE)
|
||||
ICON_CDN_PATTERN = re.compile(r"(font-?awesome|cdnjs[^\"']*all\.min\.css|lucide)", re.IGNORECASE)
|
||||
|
||||
SMOOTH_SCROLL_PATTERN = re.compile(r"scroll-behavior\s*:\s*smooth|scrollIntoView\s*\(\s*\{\s*behavior\s*:\s*[\"']smooth")
|
||||
OBSERVER_REVEAL_PATTERN = re.compile(
|
||||
r"IntersectionObserver[\s\S]{0,400}classList\.(add|toggle)\s*\(\s*[\"'](visible|show|fade-in|animate|active|in-view)[\"']"
|
||||
)
|
||||
ANCHOR_SCRIPT_PATTERN = re.compile(r"querySelectorAll\s*\(\s*[\"']a\[href\^=[\"'\\]+#")
|
||||
|
||||
ROOT_PALETTE_PATTERN = re.compile(r"--primary(-color)?\s*:[\s\S]{0,300}--secondary(-color)?\s*:", re.IGNORECASE)
|
||||
UNIVERSAL_RESET_PATTERN = re.compile(
|
||||
r"\*\s*\{[^}]*margin\s*:\s*0[^}]*padding\s*:\s*0[^}]*box-sizing\s*:\s*border-box[^}]*\}", re.DOTALL
|
||||
)
|
||||
|
||||
HEADING_EMOJI_PATTERN = re.compile(r"<(h[1-3]|button)[^>]*>[^<]*[\U0001F300-\U0001FAFF✨🚀⭐✅]")
|
||||
STARTUP_COPY_PATTERN = re.compile(
|
||||
r"empower your (team|business)|built for (modern|the) (teams|businesses|ai era|ai age|modern web)"
|
||||
r"|unlock (the power|your potential|productivity)|transform your (business|workflow|ideas)"
|
||||
r"|elevate your|seamless integration|all rights reserved\.\s*(built|made) with",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
SECTION_COMMENT_THRESHOLD: int = 3
|
||||
LANDING_SECTION_THRESHOLD: int = 4
|
||||
DEAD_LINK_THRESHOLD: int = 5
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def detect_web_tells(context: FileContext) -> list[Signal]:
|
||||
if context.language not in WEB_LANGUAGES:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
|
||||
for pattern, title in BUILDER_FINGERPRINTS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal("AI_BUILDER_FINGERPRINT", title, SEVERITY_STRONG, AXIS_ORIGIN, 4.0, _line_of(text, match.start()), match.group(0)[:120])
|
||||
)
|
||||
|
||||
signature = SIGNATURE_GRADIENT_PATTERN.search(text)
|
||||
if signature:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SIGNATURE_GRADIENT",
|
||||
"Canonical LLM purple gradient #667eea to #764ba2",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
_line_of(text, signature.start()),
|
||||
signature.group(0)[:120],
|
||||
)
|
||||
)
|
||||
else:
|
||||
generic = GENERIC_PURPLE_GRADIENT_PATTERN.search(text)
|
||||
if generic:
|
||||
findings.append(
|
||||
Signal(
|
||||
"PURPLE_GRADIENT",
|
||||
"Indigo or violet gradient from default LLM palette",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, generic.start()),
|
||||
generic.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
stack_hits: list[str] = []
|
||||
for pattern, minimum, label in VIBE_STACK_INDICATORS:
|
||||
if len(pattern.findall(text)) >= minimum:
|
||||
stack_hits.append(label)
|
||||
if len(stack_hits) >= 3:
|
||||
findings.append(
|
||||
Signal(
|
||||
"VIBE_STACK",
|
||||
f"Default AI frontend stack fingerprint ({len(stack_hits)} indicators)",
|
||||
SEVERITY_STRONG if len(stack_hits) >= 5 else SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(stack_hits),
|
||||
)
|
||||
)
|
||||
|
||||
if context.language == "html":
|
||||
section_comments = SECTION_COMMENT_PATTERN.findall(text)
|
||||
if len(section_comments) >= SECTION_COMMENT_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SECTION_BANNER_COMMENTS",
|
||||
f"{len(section_comments)} template section banner comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(section_comments[:6]),
|
||||
)
|
||||
)
|
||||
edit_narrations = EDIT_NARRATION_PATTERN.findall(text)
|
||||
if edit_narrations:
|
||||
match = EDIT_NARRATION_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"EDIT_NARRATION_COMMENT",
|
||||
f"{len(edit_narrations)} chat-edit narration comments (Updated/Added/New ...)",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
match.group(0)[:100] if match else "",
|
||||
)
|
||||
)
|
||||
style_blocks = INLINE_MONOLITH_STYLE_PATTERN.findall(text)
|
||||
script_blocks = INLINE_MONOLITH_SCRIPT_PATTERN.findall(text)
|
||||
style_lines = sum(block.count("\n") for block in style_blocks)
|
||||
script_lines = sum(block.count("\n") for block in script_blocks)
|
||||
if style_lines >= MONOLITH_STYLE_MIN_LINES and script_lines >= MONOLITH_SCRIPT_MIN_LINES:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SINGLE_FILE_MONOLITH",
|
||||
f"Single-file page with {style_lines} CSS and {script_lines} JS lines inlined, the chat-output shape",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"inline style {style_lines} lines, inline script {script_lines} lines",
|
||||
)
|
||||
)
|
||||
landing_sections = {match.group(1).lower() for match in LANDING_SECTION_PATTERN.finditer(text)}
|
||||
if len(landing_sections) >= LANDING_SECTION_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LANDING_TEMPLATE",
|
||||
"Canonical hero-features-testimonials-pricing landing structure",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(sorted(landing_sections)),
|
||||
)
|
||||
)
|
||||
dead_links = DEAD_LINK_PATTERN.findall(text)
|
||||
if len(dead_links) >= DEAD_LINK_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEAD_ANCHOR_LINKS",
|
||||
f'{len(dead_links)} links pointing to href="#"',
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(dead_links)} placeholder links",
|
||||
)
|
||||
)
|
||||
if TAILWIND_CDN_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("TAILWIND_CDN", "Tailwind loaded from CDN, the LLM prototype default", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, 1, "cdn.tailwindcss.com")
|
||||
)
|
||||
fonts = FONT_STACK_PATTERN.search(text)
|
||||
if fonts and ICON_CDN_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("DEFAULT_FONT_STACK", f"Default LLM font and icon stack ({fonts.group(1)} plus icon CDN)", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, _line_of(text, fonts.start()), fonts.group(0)[:100])
|
||||
)
|
||||
emoji_heading = HEADING_EMOJI_PATTERN.search(text)
|
||||
if emoji_heading:
|
||||
findings.append(
|
||||
Signal("EMOJI_HEADINGS", "Emoji inside headings or buttons", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, emoji_heading.start()), emoji_heading.group(0)[:100])
|
||||
)
|
||||
|
||||
for pattern, title in PLACEHOLDER_CONTACT_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_CONTENT", title, SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, _line_of(text, match.start()), match.group(0)[:100])
|
||||
)
|
||||
|
||||
lorem = LOREM_PATTERN.search(text)
|
||||
if lorem:
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_CONTENT", "Lorem ipsum filler text left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, _line_of(text, lorem.start()), lorem.group(0))
|
||||
)
|
||||
|
||||
copy_match = STARTUP_COPY_PATTERN.search(text)
|
||||
if copy_match:
|
||||
findings.append(
|
||||
Signal("TEMPLATE_COPY", "Stock AI landing page copy phrase", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, copy_match.start()), copy_match.group(0)[:100])
|
||||
)
|
||||
|
||||
if context.language in ("javascript", "typescript", "html"):
|
||||
observer = OBSERVER_REVEAL_PATTERN.search(text)
|
||||
if observer:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SCROLL_REVEAL_BOILERPLATE",
|
||||
"IntersectionObserver fade-in reveal boilerplate",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, observer.start()),
|
||||
observer.group(0)[:100],
|
||||
)
|
||||
)
|
||||
if SMOOTH_SCROLL_PATTERN.search(text) and ANCHOR_SCRIPT_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("SMOOTH_SCROLL_BOILERPLATE", "Smooth-scroll anchor polyfill boilerplate", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, 1, "smooth scroll + anchor querySelectorAll")
|
||||
)
|
||||
|
||||
if context.language in ("css", "html"):
|
||||
palette = ROOT_PALETTE_PATTERN.search(text)
|
||||
if palette:
|
||||
findings.append(
|
||||
Signal("DEFAULT_CSS_PALETTE", "Generic --primary/--secondary variable palette", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, palette.start()), palette.group(0)[:80])
|
||||
)
|
||||
reset = UNIVERSAL_RESET_PATTERN.search(text)
|
||||
if reset:
|
||||
findings.append(
|
||||
Signal("UNIVERSAL_RESET", "Universal margin/padding/box-sizing reset block", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, _line_of(text, reset.start()), "* { margin:0; padding:0; box-sizing:border-box }")
|
||||
)
|
||||
|
||||
return findings[:14]
|
||||
@@ -0,0 +1,170 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
README_SCAN_BYTES: int = 6144
|
||||
SATURATION: float = 16.0
|
||||
README_CAP: float = 18.0
|
||||
CONSTELLATION_FREEBIES: int = 3
|
||||
CONSTELLATION_STEP: float = 1.2
|
||||
CONSTELLATION_CAP: float = 12.0
|
||||
|
||||
KNOWN_TEMPLATE_SLUGS: tuple[tuple[str, float], ...] = (
|
||||
("next-js-boilerplate", 9.0),
|
||||
("nextjs-boilerplate", 9.0),
|
||||
("next-js-blog-boilerplate", 9.0),
|
||||
("vercel-ai-chatbot", 9.0),
|
||||
("ai-chatbot", 9.0),
|
||||
("create-t3-app", 9.0),
|
||||
("fullstack-nextjs-app-template", 9.0),
|
||||
("my-v0-project", 9.0),
|
||||
("nextjs-starter", 8.0),
|
||||
("saas-starter", 8.0),
|
||||
("astro-boilerplate", 8.0),
|
||||
("vite-project", 7.0),
|
||||
("my-t3-app", 9.0),
|
||||
("my-app", 5.0),
|
||||
("chatbot", 5.0),
|
||||
)
|
||||
|
||||
KNOWN_TEMPLATE_AUTHORS: tuple[str, ...] = (
|
||||
"ixartz",
|
||||
"uiux lab",
|
||||
"shoubhit dash",
|
||||
"vercel",
|
||||
"shadcn",
|
||||
"steven tey",
|
||||
)
|
||||
|
||||
GENERIC_IDENTITY_PATTERN = re.compile(r"\b(boilerplate|starter|template|scaffold)\b", re.IGNORECASE)
|
||||
|
||||
README_MARKERS: tuple[tuple[str, float, str], ...] = (
|
||||
(r"bootstrapped with .{0,40}create", 10.0, "bootstrapped-with"),
|
||||
(r"create[- ]t3[- ]app|\bt3 stack\b", 9.0, "t3-stack"),
|
||||
(r"(free,? )?open[- ]source (template|starter)|\b(app|chatbot|website) template\b|template built with", 9.0, "template-marketing"),
|
||||
(r"\bboilerplate\b", 8.0, "boilerplate-wording"),
|
||||
(r"\bstarter\b", 6.0, "starter-wording"),
|
||||
(r"getting started.{0,160}first,? run the development server", 7.0, "default-readme"),
|
||||
(r"deploy your own|one[- ]click deploy|deploy with vercel", 5.0, "deploy-your-own"),
|
||||
(r"https?://demo\.|/demo/", 4.0, "hosted-demo"),
|
||||
(r"\bsponsors?\b", 3.0, "sponsor-section"),
|
||||
)
|
||||
|
||||
SCAFFOLD_ARTIFACTS: tuple[str, ...] = (
|
||||
".storybook",
|
||||
".husky",
|
||||
".devcontainer",
|
||||
".github/FUNDING.yml",
|
||||
".github/dependabot.yml",
|
||||
".github/workflows",
|
||||
"commitlint.config.js",
|
||||
"commitlint.config.ts",
|
||||
"lint-staged.config.js",
|
||||
".lintstagedrc",
|
||||
".lintstagedrc.js",
|
||||
"renovate.json",
|
||||
"playwright.config.js",
|
||||
"playwright.config.ts",
|
||||
"vitest.config.js",
|
||||
"vitest.config.ts",
|
||||
"drizzle.config.ts",
|
||||
"components.json",
|
||||
"crowdin.yml",
|
||||
".releaserc",
|
||||
".releaserc.json",
|
||||
"docker-compose.yml",
|
||||
"Dockerfile",
|
||||
"sentry.client.config.ts",
|
||||
"sentry.server.config.ts",
|
||||
"CHANGELOG.md",
|
||||
"CODE_OF_CONDUCT.md",
|
||||
"oxlint.config.ts",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TemplateEvidence:
|
||||
score: float = 0.0
|
||||
markers: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _manifest_markers(root: Path) -> tuple[float, list[str]]:
|
||||
manifest = root / "package.json"
|
||||
if not manifest.exists():
|
||||
return 0.0, []
|
||||
try:
|
||||
body = json.loads(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return 0.0, []
|
||||
if not isinstance(body, dict):
|
||||
return 0.0, []
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
name = str(body.get("name") or "").lower()
|
||||
matched_slug = next(
|
||||
((slug, slug_weight) for slug, slug_weight in KNOWN_TEMPLATE_SLUGS if slug in name), None
|
||||
)
|
||||
if matched_slug:
|
||||
weight += matched_slug[1]
|
||||
markers.append(f"package name matches known template '{matched_slug[0]}'")
|
||||
identity = f"{name} {str(body.get('description') or '')}"
|
||||
if not matched_slug and GENERIC_IDENTITY_PATTERN.search(identity):
|
||||
weight += 7.0
|
||||
markers.append("package identity describes itself as a boilerplate/starter/template")
|
||||
author = str(body.get("author") or "").lower()
|
||||
if any(known in author for known in KNOWN_TEMPLATE_AUTHORS):
|
||||
weight += 6.0
|
||||
markers.append("package author is a known template publisher")
|
||||
if "ct3aMetadata" in body:
|
||||
weight += 10.0
|
||||
markers.append("create-t3-app scaffold metadata present")
|
||||
return weight, markers
|
||||
|
||||
|
||||
def _readme_markers(root: Path) -> tuple[float, list[str]]:
|
||||
readme = next((root / name for name in ("README.md", "readme.md", "README") if (root / name).exists()), None)
|
||||
if readme is None:
|
||||
return 0.0, []
|
||||
try:
|
||||
text = readme.read_text(encoding="utf-8", errors="replace")[:README_SCAN_BYTES]
|
||||
except OSError:
|
||||
return 0.0, []
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
for pattern, marker_weight, label in README_MARKERS:
|
||||
if re.search(pattern, text, re.IGNORECASE | re.DOTALL):
|
||||
weight += marker_weight
|
||||
markers.append(f"README carries template marker: {label}")
|
||||
return min(weight, README_CAP), markers
|
||||
|
||||
|
||||
def _constellation_markers(root: Path) -> tuple[float, list[str]]:
|
||||
present = [artifact for artifact in SCAFFOLD_ARTIFACTS if (root / artifact).exists()]
|
||||
extra = max(0, len(present) - CONSTELLATION_FREEBIES)
|
||||
if extra <= 0:
|
||||
return 0.0, []
|
||||
weight = min(extra * CONSTELLATION_STEP, CONSTELLATION_CAP)
|
||||
return weight, [f"kitchen-sink scaffold constellation: {len(present)} standard scaffold artifacts"]
|
||||
|
||||
|
||||
def detect_template(root: Path) -> TemplateEvidence:
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
for probe in (_manifest_markers, _readme_markers, _constellation_markers):
|
||||
try:
|
||||
probe_weight, probe_markers = probe(root)
|
||||
except OSError as error:
|
||||
logger.warning("Template probe failed in %s: %s", root, error)
|
||||
continue
|
||||
weight += probe_weight
|
||||
markers.extend(probe_markers)
|
||||
score = round(100.0 * (1.0 - math.exp(-weight / SATURATION)), 1) if weight > 0 else 0.0
|
||||
return TemplateEvidence(score=score, markers=markers)
|
||||
@@ -0,0 +1,82 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from html import escape
|
||||
|
||||
GRADE_COLORS: dict[str, str] = {
|
||||
"A": "#34a853",
|
||||
"B": "#7cb342",
|
||||
"C": "#fbbc04",
|
||||
"D": "#fb8c00",
|
||||
"F": "#ea4335",
|
||||
}
|
||||
LABEL_COLOR: str = "#555555"
|
||||
PENDING_COLOR: str = "#9e9e9e"
|
||||
CHAR_WIDTH: float = 6.3
|
||||
HORIZONTAL_PADDING: float = 10.0
|
||||
BADGE_HEIGHT: int = 20
|
||||
LABEL_TEXT: str = "authenticity · human"
|
||||
|
||||
|
||||
def _segment_width(text: str) -> float:
|
||||
return len(text) * CHAR_WIDTH + HORIZONTAL_PADDING
|
||||
|
||||
|
||||
def render_badge(human_percent: float | str | None, grade: str | None, report_url: str) -> str:
|
||||
percent: float | None = None
|
||||
if human_percent is not None:
|
||||
try:
|
||||
percent = float(human_percent)
|
||||
except (TypeError, ValueError):
|
||||
percent = None
|
||||
if percent is None or not grade:
|
||||
value_text = "analyzing"
|
||||
value_color = PENDING_COLOR
|
||||
else:
|
||||
value_text = f"{percent:.0f}% · {grade}"
|
||||
value_color = GRADE_COLORS.get(grade, PENDING_COLOR)
|
||||
label_width = _segment_width(LABEL_TEXT)
|
||||
value_width = _segment_width(value_text)
|
||||
total_width = label_width + value_width
|
||||
label_center = label_width / 2.0
|
||||
value_center = label_width + value_width / 2.0
|
||||
safe_url = escape(report_url, quote=True)
|
||||
safe_value = escape(value_text)
|
||||
safe_label = escape(LABEL_TEXT)
|
||||
return (
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" '
|
||||
f'width="{total_width:.0f}" height="{BADGE_HEIGHT}" role="img" aria-label="{safe_label}: {safe_value}">'
|
||||
f'<title>{safe_label}: {safe_value}</title>'
|
||||
f'<a xlink:href="{safe_url}" target="_blank">'
|
||||
f'<linearGradient id="s" x2="0" y2="100%">'
|
||||
f'<stop offset="0" stop-color="#bbbbbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/>'
|
||||
f'</linearGradient>'
|
||||
f'<clipPath id="r"><rect width="{total_width:.0f}" height="{BADGE_HEIGHT}" rx="3" fill="#ffffff"/></clipPath>'
|
||||
f'<g clip-path="url(#r)">'
|
||||
f'<rect width="{label_width:.0f}" height="{BADGE_HEIGHT}" fill="{LABEL_COLOR}"/>'
|
||||
f'<rect x="{label_width:.0f}" width="{value_width:.0f}" height="{BADGE_HEIGHT}" fill="{value_color}"/>'
|
||||
f'<rect width="{total_width:.0f}" height="{BADGE_HEIGHT}" fill="url(#s)"/>'
|
||||
f'</g>'
|
||||
f'<g fill="#ffffff" text-anchor="middle" '
|
||||
f'font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110">'
|
||||
f'<text aria-hidden="true" x="{label_center * 10:.0f}" y="150" fill="#010101" fill-opacity=".3" '
|
||||
f'transform="scale(.1)" textLength="{(label_width - HORIZONTAL_PADDING) * 10:.0f}">{safe_label}</text>'
|
||||
f'<text x="{label_center * 10:.0f}" y="140" transform="scale(.1)" '
|
||||
f'textLength="{(label_width - HORIZONTAL_PADDING) * 10:.0f}">{safe_label}</text>'
|
||||
f'<text aria-hidden="true" x="{value_center * 10:.0f}" y="150" fill="#010101" fill-opacity=".3" '
|
||||
f'transform="scale(.1)" textLength="{(value_width - HORIZONTAL_PADDING) * 10:.0f}">{safe_value}</text>'
|
||||
f'<text x="{value_center * 10:.0f}" y="140" transform="scale(.1)" '
|
||||
f'textLength="{(value_width - HORIZONTAL_PADDING) * 10:.0f}">{safe_value}</text>'
|
||||
f'</g></a></svg>'
|
||||
)
|
||||
|
||||
|
||||
def badge_markdown(badge_url: str, report_url: str) -> str:
|
||||
return f"[]({report_url})"
|
||||
|
||||
|
||||
def badge_html(badge_url: str, report_url: str) -> str:
|
||||
return (
|
||||
f'<a href="{escape(report_url, quote=True)}">'
|
||||
f'<img src="{escape(badge_url, quote=True)}" alt="authenticity human score"/></a>'
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
GIT_SIZE_LIMIT_BYTES: int = 3 * 1024 * 1024 * 1024
|
||||
GIT_CLONE_TIMEOUT_SECONDS: int = 900
|
||||
GIT_PROBE_TIMEOUT_SECONDS: int = 25
|
||||
GIT_SIZE_POLL_SECONDS: float = 2.0
|
||||
|
||||
WEBSITE_MAX_PAGES: int = 40
|
||||
WEBSITE_MAX_FILES: int = 150
|
||||
WEBSITE_MAX_TOTAL_BYTES: int = 80 * 1024 * 1024
|
||||
WEBSITE_MAX_FILE_BYTES: int = 5 * 1024 * 1024
|
||||
WEBSITE_MAX_DEPTH: int = 2
|
||||
WEBSITE_REQUEST_TIMEOUT_SECONDS: float = 30.0
|
||||
|
||||
ANALYSIS_FILE_CAP_BYTES: int = 512 * 1024
|
||||
ANALYSIS_MAX_FILES: int = 2000
|
||||
AI_SAMPLE_LIMIT: int = 12
|
||||
AI_EXCERPT_CHARS: int = 6000
|
||||
|
||||
LLM_MODEL: str = "molodetz"
|
||||
LLM_TIMEOUT_SECONDS: float = 180.0
|
||||
LLM_MAX_RETRIES: int = 3
|
||||
LLM_RETRY_BACKOFF_SECONDS: float = 2.0
|
||||
LLM_TEMPERATURE: float = 0.0
|
||||
|
||||
WORKER_TIMEOUT_SECONDS: int = 3600
|
||||
SOURCE_URL_MAX_LENGTH: int = 2048
|
||||
|
||||
IMAGE_MAX_COUNT: int = 20
|
||||
IMAGE_CONCURRENCY: int = 5
|
||||
IMAGE_MIN_BYTES: int = 6 * 1024
|
||||
IMAGE_MAX_BYTES: int = 12 * 1024 * 1024
|
||||
IMAGE_MIN_DIMENSION: int = 128
|
||||
IMAGE_EXTENSIONS: tuple[str, ...] = (".png", ".jpg", ".jpeg", ".webp", ".avif", ".bmp", ".gif")
|
||||
VISION_TIMEOUT_SECONDS: float = 120.0
|
||||
|
||||
|
||||
THUMBNAIL_MAX_DIMENSION: int = 480
|
||||
THUMBNAIL_QUALITY: int = 80
|
||||
|
||||
DOM_ANALYSIS_MAX_PAGES: int = 1
|
||||
DOM_CAPTURE_TIMEOUT_MS: int = 8000
|
||||
CONSOLE_SAMPLE_CAP: int = 20
|
||||
HEADER_VALUE_CAP: int = 500
|
||||
DOM_AI_WEIGHT: float = 0.12
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerSettings:
|
||||
allow_private_hosts: bool
|
||||
llm_endpoint: str
|
||||
llm_model: str
|
||||
llm_api_key: str
|
||||
ai_review_enabled: bool
|
||||
image_review_enabled: bool
|
||||
image_max_count: int
|
||||
media_dir: str
|
||||
template_detection: bool
|
||||
|
||||
|
||||
def settings_from_payload(payload: dict) -> WorkerSettings:
|
||||
return WorkerSettings(
|
||||
allow_private_hosts=bool(payload.get("allow_private", False)),
|
||||
llm_endpoint=str(payload.get("llm_endpoint", "")),
|
||||
llm_model=str(payload.get("llm_model", LLM_MODEL)),
|
||||
llm_api_key=str(payload.get("api_key", "")),
|
||||
ai_review_enabled=bool(payload.get("ai_review", True)),
|
||||
image_review_enabled=bool(payload.get("image_review", True)),
|
||||
image_max_count=int(payload.get("image_max", IMAGE_MAX_COUNT)),
|
||||
media_dir=str(payload.get("media_dir", "")),
|
||||
template_detection=bool(payload.get("template_detection", True)),
|
||||
)
|
||||
@@ -0,0 +1,59 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND_STAGE: str = "stage"
|
||||
KIND_LOG: str = "log"
|
||||
KIND_FILE: str = "file"
|
||||
KIND_SIGNAL: str = "signal"
|
||||
KIND_AI: str = "ai"
|
||||
KIND_IMAGE: str = "image"
|
||||
KIND_DOM: str = "dom"
|
||||
KIND_PROGRESS: str = "progress"
|
||||
KIND_SCORE: str = "score"
|
||||
KIND_REPORT: str = "report"
|
||||
KIND_ERROR: str = "error"
|
||||
KIND_DONE: str = "done"
|
||||
|
||||
STAGE_RESOLVE: str = "resolve"
|
||||
STAGE_ACQUIRE: str = "acquire"
|
||||
STAGE_INVENTORY: str = "inventory"
|
||||
STAGE_STATIC: str = "static-analysis"
|
||||
STAGE_AI: str = "ai-review"
|
||||
STAGE_IMAGES: str = "image-review"
|
||||
STAGE_DOM: str = "dom-analysis"
|
||||
STAGE_SCORE: str = "scoring"
|
||||
STAGE_REPORT: str = "report"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerEvent:
|
||||
kind: str
|
||||
message: str
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_json(self) -> str:
|
||||
return json.dumps({"kind": self.kind, "message": self.message, "data": self.data}, ensure_ascii=False)
|
||||
|
||||
@staticmethod
|
||||
def parse(line: str) -> Optional["WorkerEvent"]:
|
||||
line = line.strip()
|
||||
if not line or not line.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
logger.debug("Unparseable worker line: %s (%s)", line[:120], error)
|
||||
return None
|
||||
kind = body.get("kind")
|
||||
message = body.get("message")
|
||||
data = body.get("data")
|
||||
if not isinstance(kind, str) or not isinstance(message, str):
|
||||
return None
|
||||
return WorkerEvent(kind=kind, message=message, data=data if isinstance(data, dict) else {})
|
||||
@@ -0,0 +1,130 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.services.jobs.isslop import store
|
||||
from devplacepy.services.jobs.isslop.events import (
|
||||
KIND_DOM,
|
||||
KIND_DONE,
|
||||
KIND_ERROR,
|
||||
KIND_FILE,
|
||||
KIND_IMAGE,
|
||||
KIND_REPORT,
|
||||
WorkerEvent,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCORE_FIELDS: tuple[str, ...] = (
|
||||
"grade",
|
||||
"slop_score",
|
||||
"origin_score",
|
||||
"human_percent",
|
||||
"ai_percent",
|
||||
"category",
|
||||
"confidence",
|
||||
"content_hash",
|
||||
"source_kind",
|
||||
"files_total",
|
||||
"files_analyzed",
|
||||
"detected_builder",
|
||||
)
|
||||
|
||||
ERROR_MESSAGE_CAP = 2000
|
||||
TELLS_CAP = 4000
|
||||
DESCRIPTION_CAP = 2000
|
||||
|
||||
|
||||
class EventPersister:
|
||||
def __init__(self, analysis_uid: str) -> None:
|
||||
self._analysis_uid = analysis_uid
|
||||
self._seq = 0
|
||||
|
||||
@property
|
||||
def seq(self) -> int:
|
||||
return self._seq
|
||||
|
||||
@property
|
||||
def analysis_uid(self) -> str:
|
||||
return self._analysis_uid
|
||||
|
||||
def apply(self, event: WorkerEvent) -> dict[str, Any]:
|
||||
self._seq += 1
|
||||
payload_for_storage = dict(event.data)
|
||||
if event.kind == KIND_REPORT:
|
||||
markdown = str(payload_for_storage.pop("markdown", ""))
|
||||
model_used = str(payload_for_storage.get("model_used", "unknown"))
|
||||
store.save_report(self._analysis_uid, markdown, model_used)
|
||||
store.insert_event(
|
||||
self._analysis_uid,
|
||||
self._seq,
|
||||
event.kind,
|
||||
event.message,
|
||||
json.dumps(payload_for_storage, ensure_ascii=False),
|
||||
)
|
||||
if event.kind == KIND_FILE:
|
||||
store.insert_file_result(
|
||||
self._analysis_uid,
|
||||
{
|
||||
"path": str(event.data.get("path", "")),
|
||||
"language": str(event.data.get("language", "unknown")),
|
||||
"lines": int(event.data.get("sloc", 0)),
|
||||
"origin_score": float(event.data.get("origin_score", 0.0)),
|
||||
"quality_deficit_score": float(event.data.get("quality_deficit", 0.0)),
|
||||
"category": str(event.data.get("category", "uncertain")),
|
||||
"signals": json.dumps(event.data.get("signals", []), ensure_ascii=False)[: store.SIGNALS_CAP],
|
||||
"source": str(event.data.get("source") or ""),
|
||||
},
|
||||
)
|
||||
if event.kind == KIND_IMAGE:
|
||||
store.insert_image_result(
|
||||
self._analysis_uid,
|
||||
{
|
||||
"path": str(event.data.get("path", "")),
|
||||
"ai_probability": float(event.data.get("ai_probability", 0.0)),
|
||||
"grade": str(event.data.get("grade", "n/a")),
|
||||
"verdict": str(event.data.get("verdict", "uncertain")),
|
||||
"image_kind": str(event.data.get("image_kind", "image")),
|
||||
"tells": json.dumps(event.data.get("tells", []), ensure_ascii=False)[:TELLS_CAP],
|
||||
"description": str(event.data.get("description", ""))[:DESCRIPTION_CAP],
|
||||
"thumb": str(event.data.get("thumb") or ""),
|
||||
},
|
||||
)
|
||||
if event.kind == KIND_DOM:
|
||||
store.insert_dom_result(
|
||||
self._analysis_uid,
|
||||
{
|
||||
"url": str(event.data.get("url", "")),
|
||||
"detected_builder": str(event.data.get("detected_builder") or ""),
|
||||
"signal_count": int(event.data.get("signal_count", 0)),
|
||||
"screenshot": str(event.data.get("screenshot") or ""),
|
||||
"signals": json.dumps(event.data.get("signals", []), ensure_ascii=False)[: store.SIGNALS_CAP],
|
||||
},
|
||||
)
|
||||
if event.kind == KIND_DONE:
|
||||
updates: dict[str, Any] = {
|
||||
key: event.data.get(key) for key in SCORE_FIELDS if key in event.data
|
||||
}
|
||||
updates["quality_deficit_score"] = event.data.get("quality_deficit")
|
||||
updates["dom_slop_score"] = event.data.get("dom_slop_score")
|
||||
updates["status"] = "completed"
|
||||
updates["finished_at"] = store.utc_now()
|
||||
store.update_analysis(self._analysis_uid, **updates)
|
||||
if event.kind == KIND_ERROR:
|
||||
store.update_analysis(
|
||||
self._analysis_uid,
|
||||
status="failed",
|
||||
finished_at=store.utc_now(),
|
||||
error_message_text=event.message[:ERROR_MESSAGE_CAP],
|
||||
)
|
||||
return {
|
||||
"analysis": self._analysis_uid,
|
||||
"seq": self._seq,
|
||||
"kind": event.kind,
|
||||
"message": event.message,
|
||||
"data": payload_for_storage if event.kind != KIND_FILE else event.data,
|
||||
"created_at": store.utc_now(),
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
from devplacepy.services.jobs.isslop.acquisition.domcapture import DomSnapshot
|
||||
from devplacepy.services.jobs.isslop.acquisition.git import CloneFailedError, RepositoryTooLargeError, clone_repository
|
||||
from devplacepy.services.jobs.isslop.acquisition.source import KIND_GIT, is_private_host, resolve_source
|
||||
from devplacepy.services.jobs.isslop.acquisition.website import crawl_website
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import content_hash, remove_workspace, reset_workspace
|
||||
from devplacepy.services.jobs.isslop.agent.classifier import AiVerdict, classify_file, select_samples
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient
|
||||
from devplacepy.services.jobs.isslop.agent.reporter import generate_report
|
||||
from devplacepy.services.jobs.isslop.agent.vision import (
|
||||
ImageVerdict,
|
||||
classify_image,
|
||||
collect_images,
|
||||
image_summary,
|
||||
make_thumbnail,
|
||||
persist_screenshot,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.aggregate import aggregate_dom_evidence
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.builders import detected_builder as detected_builder_for
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.registry import run_dom_checks
|
||||
from devplacepy.services.jobs.isslop.analysis.engine import build_inventory, compute_repo_baselines, load_context, score_file
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import (
|
||||
FileScore,
|
||||
adjust_for_dom_signals,
|
||||
adjust_for_images,
|
||||
adjust_for_template,
|
||||
aggregate,
|
||||
categorize,
|
||||
criticality_for,
|
||||
repo_scores_to_dict,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_STRONG
|
||||
from devplacepy.services.jobs.isslop.analysis.templates import TemplateEvidence, detect_template
|
||||
from devplacepy.services.jobs.isslop.config import AI_EXCERPT_CHARS, IMAGE_CONCURRENCY, WorkerSettings
|
||||
from devplacepy.services.jobs.isslop.events import (
|
||||
KIND_AI,
|
||||
KIND_DOM,
|
||||
KIND_DONE,
|
||||
KIND_ERROR,
|
||||
KIND_FILE,
|
||||
KIND_LOG,
|
||||
KIND_PROGRESS,
|
||||
KIND_REPORT,
|
||||
KIND_SCORE,
|
||||
KIND_SIGNAL,
|
||||
KIND_STAGE,
|
||||
STAGE_ACQUIRE,
|
||||
STAGE_AI,
|
||||
STAGE_DOM,
|
||||
STAGE_IMAGES,
|
||||
STAGE_INVENTORY,
|
||||
STAGE_REPORT,
|
||||
STAGE_RESOLVE,
|
||||
STAGE_SCORE,
|
||||
STAGE_STATIC,
|
||||
KIND_IMAGE,
|
||||
WorkerEvent,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROGRESS_EVERY_FILES: int = 5
|
||||
SOURCE_CAP_FILES: int = 60
|
||||
SOURCE_CAP_BYTES: int = 200000
|
||||
STATIC_BLEND_WEIGHT: float = 0.6
|
||||
AI_BLEND_WEIGHT: float = 0.4
|
||||
AI_REVIEW_CONCURRENCY: int = 4
|
||||
|
||||
|
||||
def _persist_source(media_dir: Path | None, relative: str, text: str) -> str | None:
|
||||
if media_dir is None:
|
||||
return None
|
||||
name = f"s{hashlib.sha1(relative.encode('utf-8')).hexdigest()[:16]}.txt"
|
||||
try:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
(media_dir / name).write_text(text[:SOURCE_CAP_BYTES], encoding="utf-8")
|
||||
except OSError as error:
|
||||
logger.warning("Source snippet persist failed for %s: %s", relative, error)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def _stage(stage: str, message: str) -> WorkerEvent:
|
||||
return WorkerEvent(kind=KIND_STAGE, message=message, data={"stage": stage})
|
||||
|
||||
|
||||
def _blend(static_value: float, ai_value: float) -> float:
|
||||
return round(STATIC_BLEND_WEIGHT * static_value + AI_BLEND_WEIGHT * ai_value, 1)
|
||||
|
||||
|
||||
def apply_ai_verdicts(scores: list[FileScore], verdicts: list[AiVerdict]) -> list[FileScore]:
|
||||
by_path = {verdict.path: verdict for verdict in verdicts}
|
||||
adjusted: list[FileScore] = []
|
||||
for score in scores:
|
||||
verdict = by_path.get(score.relative)
|
||||
if verdict is None:
|
||||
adjusted.append(score)
|
||||
continue
|
||||
origin = _blend(score.origin_score, verdict.origin_score)
|
||||
quality = _blend(score.quality_deficit, verdict.quality_deficit)
|
||||
adjusted.append(
|
||||
FileScore(
|
||||
relative=score.relative,
|
||||
language=score.language,
|
||||
sloc=score.sloc,
|
||||
origin_score=origin,
|
||||
quality_deficit=quality,
|
||||
category=categorize(origin, quality),
|
||||
criticality=criticality_for(score.relative),
|
||||
signals=score.signals,
|
||||
)
|
||||
)
|
||||
return adjusted
|
||||
|
||||
|
||||
async def _cleanup_event(workspace: Path, stage: str) -> WorkerEvent:
|
||||
removed = await asyncio.to_thread(remove_workspace, workspace)
|
||||
message = "Workspace deleted, only the persisted report remains" if removed else "Workspace already absent"
|
||||
return WorkerEvent(KIND_LOG, message, {"stage": stage, "workspace_removed": removed})
|
||||
|
||||
|
||||
async def run_pipeline(source_url: str, workspace: Path, config: WorkerSettings) -> AsyncIterator[WorkerEvent]:
|
||||
yield _stage(STAGE_RESOLVE, f"Resolving source type for {source_url}")
|
||||
if not config.allow_private_hosts and is_private_host(source_url):
|
||||
yield WorkerEvent(KIND_ERROR, "Refusing to analyze private or loopback hosts", {"stage": STAGE_RESOLVE})
|
||||
return
|
||||
resolution = await resolve_source(source_url)
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"Source classified as {resolution.kind}",
|
||||
{"stage": STAGE_RESOLVE, "kind_detected": resolution.kind, "url": resolution.url},
|
||||
)
|
||||
|
||||
yield _stage(STAGE_ACQUIRE, f"Acquiring source into workspace ({resolution.kind})")
|
||||
reset_workspace(workspace)
|
||||
dom_snapshots: list[DomSnapshot] = []
|
||||
try:
|
||||
if resolution.kind == KIND_GIT:
|
||||
async for progress in clone_repository(resolution.url, workspace):
|
||||
yield WorkerEvent(KIND_LOG, progress, {"stage": STAGE_ACQUIRE})
|
||||
else:
|
||||
async for progress in crawl_website(
|
||||
resolution.url, workspace, dom_sink=dom_snapshots, allow_private=config.allow_private_hosts
|
||||
):
|
||||
yield WorkerEvent(KIND_LOG, progress, {"stage": STAGE_ACQUIRE})
|
||||
except (RepositoryTooLargeError, CloneFailedError, RuntimeError) as error:
|
||||
yield await _cleanup_event(workspace, STAGE_ACQUIRE)
|
||||
yield WorkerEvent(KIND_ERROR, str(error), {"stage": STAGE_ACQUIRE})
|
||||
return
|
||||
digest = content_hash(workspace)
|
||||
yield WorkerEvent(KIND_LOG, f"Workspace content hash {digest[:16]}", {"stage": STAGE_ACQUIRE, "content_hash": digest})
|
||||
|
||||
yield _stage(STAGE_INVENTORY, "Building file inventory and applying exclusion rules")
|
||||
inventory = build_inventory(workspace)
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"{len(inventory.analyzable)} files analyzable, {len(inventory.excluded)} excluded "
|
||||
f"(vendored, generated, binary, lockfiles, oversize)",
|
||||
{
|
||||
"stage": STAGE_INVENTORY,
|
||||
"analyzable": len(inventory.analyzable),
|
||||
"excluded": len(inventory.excluded),
|
||||
"excluded_samples": [
|
||||
{"path": path, "reason": reason} for path, reason in inventory.excluded[:15]
|
||||
],
|
||||
},
|
||||
)
|
||||
if not inventory.analyzable:
|
||||
yield await _cleanup_event(workspace, STAGE_INVENTORY)
|
||||
yield WorkerEvent(KIND_ERROR, "No analyzable source files found in this source", {"stage": STAGE_INVENTORY})
|
||||
return
|
||||
template = detect_template(workspace) if config.template_detection else TemplateEvidence()
|
||||
if template.markers:
|
||||
yield WorkerEvent(
|
||||
KIND_SIGNAL,
|
||||
f"Starter template provenance detected (score {template.score}/100): "
|
||||
+ "; ".join(template.markers[:4]),
|
||||
{"stage": STAGE_INVENTORY, "template_score": template.score, "markers": template.markers},
|
||||
)
|
||||
|
||||
yield _stage(STAGE_STATIC, f"Running static multi-signal analysis on {len(inventory.analyzable)} files")
|
||||
contexts = []
|
||||
for entry in inventory.analyzable:
|
||||
context = load_context(entry, inventory.repo)
|
||||
if context is not None:
|
||||
contexts.append(context)
|
||||
else:
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"Skipped {entry.relative}: unreadable content",
|
||||
{"stage": STAGE_STATIC, "path": entry.relative, "reason": "unreadable"},
|
||||
)
|
||||
if not contexts:
|
||||
yield await _cleanup_event(workspace, STAGE_STATIC)
|
||||
yield WorkerEvent(
|
||||
KIND_ERROR,
|
||||
"Insufficient analyzable content: every candidate file is minified, generated or unreadable. "
|
||||
"Abstaining rather than guessing, per methodology.",
|
||||
{"stage": STAGE_STATIC, "candidates": len(inventory.analyzable)},
|
||||
)
|
||||
return
|
||||
baselines = compute_repo_baselines(contexts)
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"Repo baselines computed over {baselines.file_count} files "
|
||||
f"(mean indent deviation {baselines.indent_variance_mean:.2f}, mean comment ratio {baselines.comment_ratio_mean:.2f})",
|
||||
{"stage": STAGE_STATIC},
|
||||
)
|
||||
scores: list[FileScore] = []
|
||||
excerpts: dict[str, str] = {}
|
||||
source_media_dir = Path(config.media_dir) if config.media_dir else None
|
||||
sources_persisted = 0
|
||||
for index, context in enumerate(contexts, start=1):
|
||||
score = score_file(context, baselines)
|
||||
scores.append(score)
|
||||
excerpts[score.relative] = context.text[:AI_EXCERPT_CHARS]
|
||||
source_name = None
|
||||
if score.signals and sources_persisted < SOURCE_CAP_FILES:
|
||||
source_name = await asyncio.to_thread(
|
||||
_persist_source, source_media_dir, score.relative, context.text
|
||||
)
|
||||
if source_name:
|
||||
sources_persisted += 1
|
||||
mode = "fingerprint scan" if context.fingerprint_only else "full analysis"
|
||||
yield WorkerEvent(
|
||||
KIND_FILE,
|
||||
f"Checked {score.relative} ({mode}): origin {score.origin_score}, quality deficit {score.quality_deficit}, {score.category}",
|
||||
{
|
||||
"stage": STAGE_STATIC,
|
||||
"path": score.relative,
|
||||
"language": score.language,
|
||||
"sloc": score.sloc,
|
||||
"origin_score": score.origin_score,
|
||||
"quality_deficit": score.quality_deficit,
|
||||
"category": score.category,
|
||||
"fingerprint_only": context.fingerprint_only,
|
||||
"signals": [signal.to_dict() for signal in score.signals[:20]],
|
||||
"source": source_name,
|
||||
},
|
||||
)
|
||||
for signal in score.signals:
|
||||
if signal.severity == SEVERITY_STRONG:
|
||||
yield WorkerEvent(
|
||||
KIND_SIGNAL,
|
||||
f"Strong signal in {score.relative}:{signal.line} - {signal.title}",
|
||||
{"stage": STAGE_STATIC, "path": score.relative, "signal": signal.to_dict()},
|
||||
)
|
||||
if index % PROGRESS_EVERY_FILES == 0 or index == len(contexts):
|
||||
yield WorkerEvent(
|
||||
KIND_PROGRESS,
|
||||
f"Static analysis {index}/{len(contexts)} files",
|
||||
{"stage": STAGE_STATIC, "current": index, "total": len(contexts)},
|
||||
)
|
||||
|
||||
dom_pages = [
|
||||
DomPageContext(
|
||||
url=snapshot.render.final_url,
|
||||
dom=snapshot.dom,
|
||||
console_warnings=snapshot.console_warnings,
|
||||
console_errors=snapshot.console_errors,
|
||||
response_headers=snapshot.response_headers,
|
||||
resource_hosts=snapshot.resource_hosts,
|
||||
screenshot_bytes=snapshot.screenshot_bytes,
|
||||
)
|
||||
for snapshot in dom_snapshots
|
||||
]
|
||||
dom_evidence = aggregate_dom_evidence(dom_pages)
|
||||
dom_media_dir = Path(config.media_dir) if config.media_dir else None
|
||||
if dom_pages:
|
||||
yield _stage(STAGE_DOM, f"Analyzing rendered DOM for AI-builder and AI-slop tells on {len(dom_pages)} page(s)")
|
||||
for page in dom_pages:
|
||||
screenshot_name = None
|
||||
if page.screenshot_bytes is not None and dom_media_dir is not None:
|
||||
screenshot_name = await asyncio.to_thread(
|
||||
persist_screenshot, page.screenshot_bytes, dom_media_dir, page.url
|
||||
)
|
||||
page_signals = run_dom_checks(page)
|
||||
page_builder, page_builder_confidence = detected_builder_for([page])
|
||||
yield WorkerEvent(
|
||||
KIND_DOM,
|
||||
f"DOM analysis of {page.url}: {len(page_signals)} signals, "
|
||||
f"builder {page_builder or 'none'}",
|
||||
{
|
||||
"stage": STAGE_DOM,
|
||||
"url": page.url,
|
||||
"detected_builder": page_builder,
|
||||
"builder_confidence": page_builder_confidence,
|
||||
"signal_count": len(page_signals),
|
||||
"signals": [signal.to_dict() for signal in page_signals[:20]],
|
||||
"screenshot": screenshot_name,
|
||||
},
|
||||
)
|
||||
|
||||
yield _stage(STAGE_AI, "Selecting representative files for AI review")
|
||||
llm = LlmClient(config)
|
||||
samples = select_samples(scores) if llm.review_available else []
|
||||
if not llm.review_available:
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
"AI review disabled or unavailable; static signals remain authoritative",
|
||||
{"stage": STAGE_AI},
|
||||
)
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"AI reviewing {len(samples)} representative files, up to {AI_REVIEW_CONCURRENCY} concurrently "
|
||||
f"(deterministic selection, temperature 0)",
|
||||
{"stage": STAGE_AI, "samples": [sample.relative for sample in samples]},
|
||||
)
|
||||
verdicts: list[AiVerdict] = []
|
||||
review_semaphore = asyncio.Semaphore(AI_REVIEW_CONCURRENCY)
|
||||
|
||||
async def review(sample: FileScore) -> tuple[FileScore, AiVerdict | None]:
|
||||
async with review_semaphore:
|
||||
return sample, await classify_file(llm, sample, excerpts.get(sample.relative, ""))
|
||||
|
||||
review_tasks = [asyncio.create_task(review(sample)) for sample in samples]
|
||||
for index, finished in enumerate(asyncio.as_completed(review_tasks), start=1):
|
||||
sample, verdict = await finished
|
||||
if verdict is None:
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"AI review unavailable for {sample.relative}, static signals remain authoritative",
|
||||
{"stage": STAGE_AI, "path": sample.relative},
|
||||
)
|
||||
else:
|
||||
verdicts.append(verdict)
|
||||
yield WorkerEvent(
|
||||
KIND_AI,
|
||||
f"AI verdict {sample.relative}: {verdict.category} ({verdict.ai_probability:.0f}% AI)",
|
||||
{
|
||||
"stage": STAGE_AI,
|
||||
"path": verdict.path,
|
||||
"category": verdict.category,
|
||||
"origin_score": verdict.origin_score,
|
||||
"quality_deficit": verdict.quality_deficit,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"reasoning": verdict.reasoning,
|
||||
"notable_signals": verdict.notable_signals,
|
||||
},
|
||||
)
|
||||
yield WorkerEvent(
|
||||
KIND_PROGRESS,
|
||||
f"AI review {index}/{len(samples)} files",
|
||||
{"stage": STAGE_AI, "current": index, "total": len(samples)},
|
||||
)
|
||||
|
||||
yield _stage(STAGE_IMAGES, "Reviewing images for AI-generated content")
|
||||
image_verdicts: list[ImageVerdict] = []
|
||||
image_stats: dict[str, object] = {"count": 0, "mean_ai_probability": 0.0, "grade": "n/a", "ai_generated_count": 0}
|
||||
if not llm.vision_available:
|
||||
yield WorkerEvent(KIND_LOG, "Vision backend unavailable; skipping image analysis", {"stage": STAGE_IMAGES})
|
||||
else:
|
||||
images = await asyncio.to_thread(collect_images, workspace, config.image_max_count)
|
||||
if not images:
|
||||
yield WorkerEvent(KIND_LOG, "No qualifying images found to review", {"stage": STAGE_IMAGES})
|
||||
else:
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"Reviewing {len(images)} images with the vision model, up to {IMAGE_CONCURRENCY} concurrently "
|
||||
f"(deterministic sample, capped at {config.image_max_count})",
|
||||
{"stage": STAGE_IMAGES, "images": [str(path.relative_to(workspace)) for path in images]},
|
||||
)
|
||||
media_dir = Path(config.media_dir) if config.media_dir else None
|
||||
thumbs: dict[str, str | None] = {}
|
||||
for image_path in images:
|
||||
relative = str(image_path.relative_to(workspace))
|
||||
thumb = None
|
||||
if media_dir is not None:
|
||||
thumb = await asyncio.to_thread(make_thumbnail, image_path, media_dir, relative)
|
||||
thumbs[relative] = thumb
|
||||
yield WorkerEvent(
|
||||
KIND_LOG,
|
||||
f"Reviewing image {relative}",
|
||||
{"stage": STAGE_IMAGES, "path": relative, "thumb": thumb},
|
||||
)
|
||||
image_semaphore = asyncio.Semaphore(IMAGE_CONCURRENCY)
|
||||
|
||||
async def review_image(image_path: Path) -> ImageVerdict | None:
|
||||
async with image_semaphore:
|
||||
return await classify_image(llm, workspace, image_path)
|
||||
|
||||
image_tasks = [asyncio.create_task(review_image(image_path)) for image_path in images]
|
||||
for index, finished in enumerate(asyncio.as_completed(image_tasks), start=1):
|
||||
verdict = await finished
|
||||
if verdict is None:
|
||||
continue
|
||||
image_verdicts.append(verdict)
|
||||
yield WorkerEvent(
|
||||
KIND_IMAGE,
|
||||
f"Image {verdict.relative}: grade {verdict.grade}, {verdict.verdict} "
|
||||
f"({verdict.ai_probability:.0f}% AI) - {verdict.image_kind}",
|
||||
{
|
||||
"stage": STAGE_IMAGES,
|
||||
"path": verdict.relative,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"grade": verdict.grade,
|
||||
"verdict": verdict.verdict,
|
||||
"image_kind": verdict.image_kind,
|
||||
"tells": verdict.tells,
|
||||
"description": verdict.description,
|
||||
"thumb": thumbs.get(verdict.relative),
|
||||
},
|
||||
)
|
||||
yield WorkerEvent(
|
||||
KIND_PROGRESS,
|
||||
f"Image review {index}/{len(images)}",
|
||||
{"stage": STAGE_IMAGES, "current": index, "total": len(images)},
|
||||
)
|
||||
image_stats = image_summary(image_verdicts)
|
||||
yield WorkerEvent(
|
||||
KIND_PROGRESS,
|
||||
f"Image review complete: {image_stats['count']} images, "
|
||||
f"{image_stats['ai_generated_count']} look AI-generated, grade {image_stats['grade']}",
|
||||
{"stage": STAGE_IMAGES, **image_stats},
|
||||
)
|
||||
|
||||
yield _stage(STAGE_SCORE, "Aggregating per-file scores into repository verdict")
|
||||
static_scores = aggregate(scores)
|
||||
blended = bool(verdicts)
|
||||
adjusted = apply_ai_verdicts(scores, verdicts) if blended else scores
|
||||
final_scores = aggregate(adjusted)
|
||||
image_influenced = False
|
||||
if image_verdicts:
|
||||
final_scores = adjust_for_images(final_scores, float(image_stats["mean_ai_probability"]))
|
||||
image_influenced = True
|
||||
final_scores = adjust_for_dom_signals(final_scores, dom_evidence)
|
||||
final_scores = adjust_for_template(final_scores, template.score)
|
||||
score_payload = {
|
||||
"stage": STAGE_SCORE,
|
||||
"static": repo_scores_to_dict(static_scores),
|
||||
"blended_with_ai": blended,
|
||||
"template_score": template.score,
|
||||
"template_markers": template.markers,
|
||||
**repo_scores_to_dict(final_scores),
|
||||
"files_total": len(inventory.analyzable) + len(inventory.excluded),
|
||||
"files_analyzed": len(scores),
|
||||
"content_hash": digest,
|
||||
"source_kind": resolution.kind,
|
||||
"image_review": image_stats,
|
||||
"image_influenced": image_influenced,
|
||||
"detected_builder": dom_evidence.detected_builder,
|
||||
"dom_slop_score": dom_evidence.score,
|
||||
}
|
||||
yield WorkerEvent(
|
||||
KIND_SCORE,
|
||||
f"Final verdict: grade {final_scores.grade}, {final_scores.category}, "
|
||||
f"{final_scores.human_percent}% human / {final_scores.ai_percent}% AI, confidence {final_scores.confidence}",
|
||||
score_payload,
|
||||
)
|
||||
|
||||
yield _stage(STAGE_REPORT, "Generating final report with findings")
|
||||
markdown, model_used = await generate_report(
|
||||
llm,
|
||||
source_url,
|
||||
resolution.kind,
|
||||
final_scores,
|
||||
adjusted,
|
||||
verdicts,
|
||||
len(inventory.excluded),
|
||||
image_verdicts,
|
||||
image_stats,
|
||||
template,
|
||||
dom_evidence,
|
||||
)
|
||||
await llm.aclose()
|
||||
yield WorkerEvent(
|
||||
KIND_REPORT,
|
||||
f"Report generated via {model_used} ({len(markdown)} chars)",
|
||||
{"stage": STAGE_REPORT, "markdown": markdown, "model_used": model_used},
|
||||
)
|
||||
yield await _cleanup_event(workspace, STAGE_REPORT)
|
||||
yield WorkerEvent(KIND_DONE, "Analysis complete", score_payload)
|
||||
@@ -0,0 +1,237 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import BASE_DIR, ISSLOP_RUNS_DIR, ISSLOP_WORKSPACES_DIR
|
||||
from devplacepy.database import INTERNAL_GATEWAY_URL, get_int_setting, get_setting, internal_gateway_key
|
||||
from devplacepy.services import pubsub
|
||||
from devplacepy.services.base import ConfigField
|
||||
from devplacepy.services.jobs.base import JobService
|
||||
from devplacepy.services.jobs.isslop import store
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import remove_workspace, workspace_for
|
||||
from devplacepy.services.jobs.isslop.config import IMAGE_MAX_COUNT, LLM_MODEL, WORKER_TIMEOUT_SECONDS
|
||||
from devplacepy.services.jobs.isslop.events import KIND_DONE, KIND_ERROR, WorkerEvent
|
||||
from devplacepy.services.jobs.isslop.persistence import EventPersister
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WORKER_MODULE = "devplacepy.services.jobs.isslop.worker"
|
||||
STREAM_LIMIT = 16 * 1024 * 1024
|
||||
TOPIC_PREFIX = "public.isslop."
|
||||
|
||||
|
||||
def topic_for(uid: str) -> str:
|
||||
return f"{TOPIC_PREFIX}{uid}"
|
||||
|
||||
|
||||
class IsslopService(JobService):
|
||||
kind = "isslop"
|
||||
title = "AI Usage Analyzer"
|
||||
description = (
|
||||
"Classifies a git repository or website as AI slop, sophisticated AI-assisted work or "
|
||||
"genuine human work: acquires the source in an isolated subprocess, runs a multi-signal "
|
||||
"static analysis plus AI and vision review passes through the internal gateway, streams "
|
||||
"every step live over pub/sub and persists a shareable report with an authenticity badge."
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="isslop", interval_seconds=2)
|
||||
for field in self.config_fields:
|
||||
if field.key == self.timeout_key:
|
||||
field.default = WORKER_TIMEOUT_SECONDS
|
||||
self.config_fields += [
|
||||
ConfigField(
|
||||
"isslop_allow_private",
|
||||
"Allow private hosts",
|
||||
type="bool",
|
||||
default="0",
|
||||
help="Permit analyzing private, loopback and link-local hosts. Keep off in production (SSRF).",
|
||||
group="Analysis",
|
||||
),
|
||||
ConfigField(
|
||||
"isslop_ai_review",
|
||||
"AI review pass",
|
||||
type="bool",
|
||||
default="1",
|
||||
help="Audit representative files with the AI gateway on top of the static engine.",
|
||||
group="Analysis",
|
||||
),
|
||||
ConfigField(
|
||||
"isslop_image_review",
|
||||
"Image review pass",
|
||||
type="bool",
|
||||
default="1",
|
||||
help="Grade images for AI generation with the vision model.",
|
||||
group="Analysis",
|
||||
),
|
||||
ConfigField(
|
||||
"isslop_template_detection",
|
||||
"Template provenance detection",
|
||||
type="bool",
|
||||
default="1",
|
||||
help="Detect starter-template/boilerplate provenance and count shipped scaffold defaults against the authenticity grade.",
|
||||
group="Analysis",
|
||||
),
|
||||
ConfigField(
|
||||
"isslop_image_max",
|
||||
"Max images per analysis",
|
||||
type="int",
|
||||
default=IMAGE_MAX_COUNT,
|
||||
minimum=1,
|
||||
maximum=100,
|
||||
help="Deterministically sampled cap on images sent to the vision model.",
|
||||
group="Analysis",
|
||||
),
|
||||
]
|
||||
|
||||
def run_dir(self, uid: str) -> Path:
|
||||
return ISSLOP_RUNS_DIR / uid
|
||||
|
||||
def _worker_payload(self, job: dict) -> dict:
|
||||
payload = dict(job.get("payload", {}))
|
||||
payload["llm_endpoint"] = INTERNAL_GATEWAY_URL
|
||||
payload["llm_model"] = LLM_MODEL
|
||||
payload["api_key"] = internal_gateway_key()
|
||||
payload["allow_private"] = get_setting("isslop_allow_private", "0") == "1"
|
||||
payload["ai_review"] = get_setting("isslop_ai_review", "1") == "1"
|
||||
payload["image_review"] = get_setting("isslop_image_review", "1") == "1"
|
||||
payload["image_max"] = max(1, get_int_setting("isslop_image_max", IMAGE_MAX_COUNT))
|
||||
payload["media_dir"] = str(store.media_dir_for(job["uid"]))
|
||||
payload["template_detection"] = get_setting("isslop_template_detection", "1") == "1"
|
||||
return payload
|
||||
|
||||
async def _relay(self, persister: EventPersister, event: WorkerEvent) -> None:
|
||||
enriched = await asyncio.to_thread(persister.apply, event)
|
||||
await pubsub.publish(topic_for(persister.analysis_uid), enriched)
|
||||
|
||||
async def process(self, job: dict) -> dict:
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
uid = job["uid"]
|
||||
source_url = str(job.get("payload", {}).get("url", ""))
|
||||
actor_kind = "user" if job.get("owner_kind") == "user" else (job.get("owner_kind") or "system")
|
||||
actor_uid = job.get("owner_id") if job.get("owner_kind") == "user" else None
|
||||
persister = EventPersister(uid)
|
||||
store.reset_evidence(uid)
|
||||
store.update_analysis(uid, status="running")
|
||||
await pubsub.publish(
|
||||
topic_for(uid),
|
||||
{
|
||||
"analysis": uid,
|
||||
"seq": 0,
|
||||
"kind": "status",
|
||||
"message": "running",
|
||||
"data": {},
|
||||
"created_at": store.utc_now(),
|
||||
},
|
||||
)
|
||||
|
||||
run_dir = self.run_dir(uid)
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
payload_path = run_dir / "payload.json"
|
||||
payload_path.write_text(json.dumps(self._worker_payload(job)), encoding="utf-8")
|
||||
try:
|
||||
workspace = workspace_for(ISSLOP_WORKSPACES_DIR, source_url, uid)
|
||||
except ValueError as error:
|
||||
await self._relay(persister, WorkerEvent(KIND_ERROR, f"Workspace rejected: {error}", {}))
|
||||
shutil.rmtree(run_dir, ignore_errors=True)
|
||||
raise RuntimeError(str(error))
|
||||
|
||||
try:
|
||||
final = await self._run_worker(uid, persister, payload_path, workspace)
|
||||
finally:
|
||||
await asyncio.to_thread(remove_workspace, workspace)
|
||||
shutil.rmtree(run_dir, ignore_errors=True)
|
||||
|
||||
if final.kind == KIND_ERROR:
|
||||
audit.record_system(
|
||||
"isslop.run.failed",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=actor_uid,
|
||||
result="failure",
|
||||
summary=f"AI usage analysis of {source_url} failed",
|
||||
metadata={"target": source_url, "error": final.message[:200]},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
raise RuntimeError(final.message)
|
||||
|
||||
audit.record_system(
|
||||
"isslop.run.complete",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=actor_uid,
|
||||
summary=f"AI usage analysis of {source_url} graded {final.data.get('grade')}",
|
||||
metadata={
|
||||
"target": source_url,
|
||||
"grade": final.data.get("grade"),
|
||||
"category": final.data.get("category"),
|
||||
"human_percent": final.data.get("human_percent"),
|
||||
"files_analyzed": final.data.get("files_analyzed"),
|
||||
},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
return {
|
||||
"target": source_url,
|
||||
"grade": final.data.get("grade"),
|
||||
"category": final.data.get("category"),
|
||||
"human_percent": final.data.get("human_percent"),
|
||||
"ai_percent": final.data.get("ai_percent"),
|
||||
"report_url": f"/tools/isslop/{uid}/report",
|
||||
"bytes_in": 0,
|
||||
"bytes_out": 0,
|
||||
"item_count": int(final.data.get("files_analyzed") or 0),
|
||||
}
|
||||
|
||||
async def _run_worker(
|
||||
self, uid: str, persister: EventPersister, payload_path: Path, workspace: Path
|
||||
) -> WorkerEvent:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
WORKER_MODULE,
|
||||
str(payload_path),
|
||||
str(workspace),
|
||||
cwd=str(BASE_DIR),
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
limit=STREAM_LIMIT,
|
||||
)
|
||||
final: WorkerEvent | None = None
|
||||
stderr_task = asyncio.create_task(proc.stderr.read())
|
||||
try:
|
||||
while True:
|
||||
line = await proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
event = WorkerEvent.parse(line.decode("utf-8", "replace"))
|
||||
if event is None:
|
||||
continue
|
||||
await self._relay(persister, event)
|
||||
if event.kind in (KIND_DONE, KIND_ERROR):
|
||||
final = event
|
||||
await stderr_task
|
||||
await proc.wait()
|
||||
finally:
|
||||
if proc.returncode is None:
|
||||
try:
|
||||
proc.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
await proc.wait()
|
||||
if not stderr_task.done():
|
||||
stderr_task.cancel()
|
||||
if final is None:
|
||||
final = WorkerEvent(
|
||||
KIND_ERROR, f"Analysis worker exited unexpectedly with code {proc.returncode}", {}
|
||||
)
|
||||
await self._relay(persister, final)
|
||||
return final
|
||||
|
||||
def cleanup(self, job: dict) -> None:
|
||||
shutil.rmtree(self.run_dir(job["uid"]), ignore_errors=True)
|
||||
@@ -0,0 +1,193 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from devplacepy.config import ISSLOP_MEDIA_DIR
|
||||
from devplacepy.database import get_table
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TABLE_ANALYSES = "isslop_analyses"
|
||||
TABLE_EVENTS = "isslop_events"
|
||||
TABLE_FILE_RESULTS = "isslop_file_results"
|
||||
TABLE_IMAGE_RESULTS = "isslop_image_results"
|
||||
TABLE_DOM_RESULTS = "isslop_dom_results"
|
||||
TABLE_REPORTS = "isslop_reports"
|
||||
|
||||
EVENT_LIMIT_DEFAULT = 2000
|
||||
EVENT_PAYLOAD_CAP = 60000
|
||||
SIGNALS_CAP = 30000
|
||||
LIST_LIMIT_DEFAULT = 50
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def create_analysis(uid: str, source_url: str, owner_kind: str, owner_id: str) -> dict[str, Any]:
|
||||
row: dict[str, Any] = {
|
||||
"uid": uid,
|
||||
"owner_kind": owner_kind,
|
||||
"owner_id": owner_id,
|
||||
"source_url": source_url,
|
||||
"source_kind": "unknown",
|
||||
"status": "pending",
|
||||
"created_at": utc_now(),
|
||||
"finished_at": None,
|
||||
"content_hash": None,
|
||||
"grade": None,
|
||||
"slop_score": None,
|
||||
"origin_score": None,
|
||||
"quality_deficit_score": None,
|
||||
"human_percent": None,
|
||||
"ai_percent": None,
|
||||
"category": None,
|
||||
"confidence": None,
|
||||
"files_total": 0,
|
||||
"files_analyzed": 0,
|
||||
"error_message_text": None,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
get_table(TABLE_ANALYSES).insert(row)
|
||||
return row
|
||||
|
||||
|
||||
def update_analysis(uid: str, **fields: Any) -> None:
|
||||
fields["uid"] = uid
|
||||
get_table(TABLE_ANALYSES).update(fields, ["uid"])
|
||||
|
||||
|
||||
def get_analysis(uid: str) -> Optional[dict[str, Any]]:
|
||||
row = get_table(TABLE_ANALYSES).find_one(uid=uid, deleted_at=None)
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def list_analyses(owner_kind: str, owner_id: str, limit: int = LIST_LIMIT_DEFAULT) -> list[dict[str, Any]]:
|
||||
rows = get_table(TABLE_ANALYSES).find(
|
||||
owner_kind=owner_kind,
|
||||
owner_id=owner_id,
|
||||
deleted_at=None,
|
||||
order_by=["-created_at"],
|
||||
_limit=limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def claim_guest_analyses(guest_id: str, user_uid: str) -> int:
|
||||
table = get_table(TABLE_ANALYSES)
|
||||
rows = list(table.find(owner_kind="guest", owner_id=guest_id, deleted_at=None))
|
||||
for row in rows:
|
||||
table.update({"uid": row["uid"], "owner_kind": "user", "owner_id": user_uid}, ["uid"])
|
||||
if rows:
|
||||
logger.info("Claimed %d guest isslop analyses for user %s", len(rows), user_uid)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def insert_event(analysis_uid: str, seq: int, kind: str, message: str, payload: str) -> None:
|
||||
get_table(TABLE_EVENTS).insert(
|
||||
{
|
||||
"analysis_uid": analysis_uid,
|
||||
"seq": seq,
|
||||
"kind": kind,
|
||||
"message": message,
|
||||
"payload": payload[:EVENT_PAYLOAD_CAP],
|
||||
"created_at": utc_now(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def events_for(analysis_uid: str, after_seq: int = 0, limit: int = EVENT_LIMIT_DEFAULT) -> list[dict[str, Any]]:
|
||||
rows = get_table(TABLE_EVENTS).find(
|
||||
analysis_uid=analysis_uid,
|
||||
seq={">": after_seq},
|
||||
order_by=["seq"],
|
||||
_limit=limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def insert_file_result(analysis_uid: str, record: dict[str, Any]) -> None:
|
||||
record["analysis_uid"] = analysis_uid
|
||||
get_table(TABLE_FILE_RESULTS).insert(record)
|
||||
|
||||
|
||||
def file_result_for(analysis_uid: str, path: str) -> Optional[dict[str, Any]]:
|
||||
row = get_table(TABLE_FILE_RESULTS).find_one(analysis_uid=analysis_uid, path=path)
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def file_results_for(analysis_uid: str) -> list[dict[str, Any]]:
|
||||
rows = get_table(TABLE_FILE_RESULTS).find(analysis_uid=analysis_uid, order_by=["path"])
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def insert_image_result(analysis_uid: str, record: dict[str, Any]) -> None:
|
||||
record["analysis_uid"] = analysis_uid
|
||||
get_table(TABLE_IMAGE_RESULTS).insert(record)
|
||||
|
||||
|
||||
def image_results_for(analysis_uid: str) -> list[dict[str, Any]]:
|
||||
rows = get_table(TABLE_IMAGE_RESULTS).find(analysis_uid=analysis_uid, order_by=["-ai_probability"])
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def insert_dom_result(analysis_uid: str, record: dict[str, Any]) -> None:
|
||||
record["analysis_uid"] = analysis_uid
|
||||
get_table(TABLE_DOM_RESULTS).insert(record)
|
||||
|
||||
|
||||
def dom_results_for(analysis_uid: str) -> list[dict[str, Any]]:
|
||||
rows = get_table(TABLE_DOM_RESULTS).find(analysis_uid=analysis_uid, order_by=["-signal_count"])
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def save_report(analysis_uid: str, markdown: str, model_used: str) -> None:
|
||||
get_table(TABLE_REPORTS).upsert(
|
||||
{
|
||||
"analysis_uid": analysis_uid,
|
||||
"markdown": markdown,
|
||||
"model_used": model_used,
|
||||
"generated_at": utc_now(),
|
||||
},
|
||||
["analysis_uid"],
|
||||
)
|
||||
|
||||
|
||||
def get_report(analysis_uid: str) -> Optional[dict[str, Any]]:
|
||||
row = get_table(TABLE_REPORTS).find_one(analysis_uid=analysis_uid)
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def decode_json(field: Any, fallback: Any) -> Any:
|
||||
if not field:
|
||||
return fallback
|
||||
try:
|
||||
parsed = json.loads(field)
|
||||
except (ValueError, TypeError):
|
||||
return fallback
|
||||
return parsed if isinstance(parsed, type(fallback)) else fallback
|
||||
|
||||
|
||||
def media_dir_for(uid: str) -> Path:
|
||||
return ISSLOP_MEDIA_DIR / uid
|
||||
|
||||
|
||||
def reset_evidence(uid: str) -> None:
|
||||
get_table(TABLE_EVENTS).delete(analysis_uid=uid)
|
||||
get_table(TABLE_FILE_RESULTS).delete(analysis_uid=uid)
|
||||
get_table(TABLE_IMAGE_RESULTS).delete(analysis_uid=uid)
|
||||
get_table(TABLE_DOM_RESULTS).delete(analysis_uid=uid)
|
||||
get_table(TABLE_REPORTS).delete(analysis_uid=uid)
|
||||
shutil.rmtree(media_dir_for(uid), ignore_errors=True)
|
||||
|
||||
|
||||
def purge_analysis(uid: str) -> None:
|
||||
reset_evidence(uid)
|
||||
get_table(TABLE_ANALYSES).delete(uid=uid)
|
||||
@@ -0,0 +1,52 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.jobs.isslop.config import settings_from_payload
|
||||
from devplacepy.services.jobs.isslop.events import KIND_ERROR, WorkerEvent
|
||||
from devplacepy.services.jobs.isslop.pipeline import run_pipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def emit(event: WorkerEvent) -> None:
|
||||
print(event.to_json(), flush=True)
|
||||
|
||||
|
||||
async def amain(payload_path: Path, workspace: Path) -> int:
|
||||
try:
|
||||
payload = json.loads(payload_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
emit(WorkerEvent(KIND_ERROR, f"Worker payload unreadable: {error}", {}))
|
||||
return 1
|
||||
settings = settings_from_payload(payload)
|
||||
source_url = str(payload.get("url", ""))
|
||||
failed = False
|
||||
try:
|
||||
async for event in run_pipeline(source_url, workspace, settings):
|
||||
emit(event)
|
||||
if event.kind == KIND_ERROR:
|
||||
failed = True
|
||||
except (OSError, ValueError, RuntimeError) as error:
|
||||
logger.exception("Pipeline crashed")
|
||||
emit(WorkerEvent(KIND_ERROR, f"Pipeline crashed: {error}", {}))
|
||||
return 1
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stderr)
|
||||
if len(sys.argv) != 3:
|
||||
emit(WorkerEvent(KIND_ERROR, "usage: worker <payload_json_path> <workspace_dir>", {}))
|
||||
sys.exit(2)
|
||||
exit_code = asyncio.run(amain(Path(sys.argv[1]), Path(sys.argv[2])))
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user