Compare commits
16 Commits
9d86d7b65c
...
499f91e16a
| Author | SHA1 | Date | |
|---|---|---|---|
| 499f91e16a | |||
| 9a8046ab2a | |||
| f1bdefd834 | |||
| 0f872336b1 | |||
| 7002b23eeb | |||
| c79dfd0291 | |||
| 8daee65011 | |||
| 7de7e29d3e | |||
| d556b02bfb | |||
| 574b076f5d | |||
| f521a1130e | |||
| b57e657cf8 | |||
| 0e8b015fe1 | |||
| d45bb37ce1 | |||
| 4618089cea | |||
| 9151876770 |
45
README.md
45
README.md
@ -19,7 +19,7 @@ Open `http://localhost:10500`.
|
||||
|
||||
| Layer | Technology |
|
||||
|-------|-----------|
|
||||
| Backend | Python 3.13+, FastAPI, Uvicorn (multi-worker in production) |
|
||||
| Backend | Python 3.12+, FastAPI, Uvicorn (multi-worker in production) |
|
||||
| Templates | Jinja2 (server-side rendered) |
|
||||
| Frontend | Pure ES6 JavaScript, one class per file |
|
||||
| Database | SQLite via `dataset` (auto-sync schema, `uid` PKs, WAL mode, 30s busy timeout) |
|
||||
@ -35,10 +35,10 @@ Open `http://localhost:10500`.
|
||||
devplacepy/
|
||||
main.py # FastAPI app, router registration
|
||||
config.py # Settings from env vars + .env
|
||||
database.py # dataset connection, index creation
|
||||
database/ # dataset connection, index creation (package)
|
||||
templating.py # Shared Jinja2 environment + globals
|
||||
avatar.py # Multiavatar generation, URL builder
|
||||
utils.py # Password hashing, session mgmt, time_ago, notification hook
|
||||
utils/ # Password hashing, session mgmt, time_ago, notification hook (package)
|
||||
models.py # Pydantic schemas
|
||||
push.py # Web push crypto, VAPID keys, encrypt/send/register
|
||||
routers/ # One file per domain (auth, feed, posts, push, ...)
|
||||
@ -63,8 +63,8 @@ devplacepy/
|
||||
| `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing, and line-range operations (`lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append`) for surgical edits to large text files (public read, owner write; all writes refused while the project is read-only) |
|
||||
| `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
|
||||
| `/forks` | Fork job status (`/forks/{uid}`); forks are queued via `/projects/{slug}/fork`. Any signed-in user can fork a project they can view into a new project they own; once the job finishes the response carries the new project URL |
|
||||
| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence and gap analysis, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}` |
|
||||
| `/projects/{slug}/containers` | Admin per-project container manager: Dockerfile CRUD with immutable versions, async image builds, and container instance creation. Reachable from the project page via the admin-only **Containers** button |
|
||||
| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence and gap analysis, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}`. `/tools/isslop` is the **AI Usage Analyzer**: classify a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Queue with `POST /tools/isslop/run`, poll `GET /tools/isslop/{uid}` or the event trail at `GET /tools/isslop/{uid}/events`, read the report at `GET /tools/isslop/{uid}/report` (`.md` to download) and embed the SVG authenticity badge from `GET /tools/isslop/{uid}/badge.svg` |
|
||||
| `/projects/{slug}/containers` | Admin per-project container manager: create and control container instances, all running the shared prebuilt `ppy` image (there is no in-app image building). Reachable from the project page via the admin-only **Containers** button |
|
||||
| `/admin/containers` | Admin **Containers** manager: list, create, edit, and control every container instance across all projects (scoped by project visibility - instances attached to another administrator's hidden project are excluded). The list (`/admin/containers`) has inline start/stop/restart/terminal/edit/delete on each row and a create form (pick a project, optionally a run-as user, a boot language with a source editor, restart policy, start-on-boot, plus env/ports/limits/ingress). Each instance has a detail page (`/admin/containers/{uid}`) with lifecycle controls, live logs and metrics, an interactive terminal, schedules, ingress, workspace sync, and a status history, and an edit page (`/admin/containers/{uid}/edit`) |
|
||||
| `/p/{slug}` | Public ingress proxy (HTTP + WebSocket) to a running container instance's published port, opt-in per instance via `ingress_slug` |
|
||||
| `/profile` | Profile view, editing, a public **Media** tab (`?tab=media`) showing every attachment a user uploaded newest first, and a live **online / last-seen** presence indicator |
|
||||
@ -72,7 +72,7 @@ devplacepy/
|
||||
| `/uploads` | File upload endpoints: `POST /uploads/upload` (multipart), `POST /uploads/upload-url` (from URL); served at `/static/uploads/` |
|
||||
| `/admin/trash` | Admin **Trash**: review, restore, and permanently purge soft-deleted content (posts, comments, gists, projects, news, project files, attachments) across the platform |
|
||||
| `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
|
||||
| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image and YouTube embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages, so typing an inline `@ai <instruction>` in a message executes it and the resolved result appears live in the chat for both participants. The `POST /messages/send` form remains as a no-JavaScript fallback |
|
||||
| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image and YouTube embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages, so typing an inline `@ai <instruction>` in a message executes it and the resolved result appears live in the chat for both participants. An opened conversation loads its 500 most recent messages; older history is retained in the database. The `POST /messages/send` form remains as a no-JavaScript fallback |
|
||||
| `/votes` | Upvote/downvote on posts, comments, projects |
|
||||
| `/reactions` | Emoji reactions on posts, comments, gists, projects |
|
||||
| `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
|
||||
@ -99,7 +99,7 @@ Member progression is driven by activity and peer recognition.
|
||||
|
||||
- **Stars** are the net vote score (`upvotes - downvotes`) on a post, project, or gist. A member's total stars is the sum across all their content and is the basis for ranking.
|
||||
- **XP and levels.** Members earn XP for contributing: posting (10), commenting (2), publishing a project (15) or gist (5), receiving an upvote (5), and gaining a follower (5). Each level requires 100 XP (`level = 1 + xp // 100`). The profile shows the current level and progress to the next.
|
||||
- **Badges** are awarded once and never revoked, across several themed groups (First steps, Explorer, Engagement, Content, Community, Reputation, Dedication, Levels). They cover three kinds of achievement: **content and reputation milestones** (10/50/100 posts, 25/100/500 stars, 10/50/100 followers, comment and project and gist counts, following 10 people, 7/30/100-day activity streaks, reaching levels 5/10/25/50/100); **first-time feature use** (your first comment, project, gist, fork, archive download, SEO audit, DeepSearch, container, direct message, bookmark, reaction, star given, follow, upload, project file, issue, poll vote, profile customization, and first conversation with Devii); and **usage tiers** for several of those features (for example reading 1/5/15 documentation pages, or giving 50/250 stars). Each profile has a collapsible **Achievements** showcase that lists every badge grouped by theme, with earned ones highlighted and locked ones shown with their unlock condition, so there is always a next prize to chase.
|
||||
- **Badges** are awarded once and never revoked, across several themed groups (First steps, Explorer, Engagement, Content, Community, Reputation, Dedication, Levels). They cover three kinds of achievement: **content and reputation milestones** (10/50/100 posts, 25/100/500 stars, 10/50/100 followers, comment and project and gist counts, following 10 people, 7/30/100-day activity streaks, reaching levels 5/10/25/50/100); **first-time feature use** (your first comment, project, gist, fork, archive download, SEO audit, DeepSearch, AI usage analysis, container, direct message, bookmark, reaction, star given, follow, upload, project file, issue, poll vote, profile customization, and first conversation with Devii); and **usage tiers** for several of those features (for example reading 1/5/15 documentation pages, or giving 50/250 stars). Each profile has a collapsible **Achievements** showcase that lists every badge grouped by theme, with earned ones highlighted and locked ones shown with their unlock condition, so there is always a next prize to chase.
|
||||
- **Leaderboard** (`/leaderboard`) ranks the top 50 members by total stars (single page, no pagination); a member's own rank is shown on their profile.
|
||||
- **Contribution heatmap and streaks.** Each profile shows a 12-month activity heatmap and the current/longest daily streak, derived from post/comment/gist/project timestamps (no extra storage).
|
||||
- **Social graph listings.** Each profile has Followers and Following tabs that paginate the follow graph (25 per page) and show a follow/unfollow control for each person. The same data is available as JSON at `GET /profile/{username}/followers` and `GET /profile/{username}/following`.
|
||||
@ -141,11 +141,11 @@ The farm refreshes live over the pub/sub bus (a watered build appears on the own
|
||||
- **Private projects** - an owner can mark a project private so it is visible only to them (and administrators) and excluded from listings, profiles, search, the sitemap, and zip access. Set at creation or toggled later from the project page.
|
||||
- **Read-only projects** - an owner can mark a project read-only, making its entire virtual filesystem immutable: every write, edit, line-edit, move, delete, and upload is refused from all paths (the web UI, the HTTP API, the Devii agent, and container workspace sync) until read-only is turned off. Devii may toggle read-only only after the user explicitly confirms.
|
||||
|
||||
XP awards are wired at the existing content-creation, vote, and follow hook points in the routers and centralized in `award_xp()` / `check_milestone_badges()` (`devplacepy/utils.py`). Existing accounts have their XP and levels backfilled once from prior activity at startup (`init_db()`).
|
||||
XP awards are wired at the existing content-creation, vote, and follow hook points in the routers and centralized in `award_xp()` / `check_milestone_badges()` (`devplacepy/utils/`). Existing accounts have their XP and levels backfilled once from prior activity at startup (`init_db()`).
|
||||
|
||||
## Vibe coding (Alpha, admin only)
|
||||
|
||||
Build software by talking to an AI agent instead of typing every line. Create a project for storage, attach a container to it (the shared `ppy` image, your files mounted at `/app`), start it, and open a terminal. The whole flow is drivable conversationally through Devii. Inside every container three agents ship preinstalled and run on **your own API key**, so all AI usage is metered to your account: **DevPlace Code (`dpc`)**, a coding agent in the same class as Claude Code; **`botje.py`**, a plug-and-play DevPlace bot you can copy and customise; and **`pagent`**, a minimal zero-dependency agent. Each container is launched with `PRAVDA_BASE_URL`, `PRAVDA_OPENAI_URL`, `PRAVDA_API_KEY`, `PRAVDA_USER_UID`, `PRAVDA_CONTAINER_NAME`, `PRAVDA_CONTAINER_UID`, and `PRAVDA_INGRESS_URL` already set. Publish a container port to a public URL at `/p/<slug>` by setting an `ingress_slug` and `ingress_port` (ask Devii to do it at create time). The feature is in **Alpha** and currently limited to administrators; the full walkthrough, including a tutorial that vibes a web app and puts it online, is at `/docs/getting-started-vibing.html`.
|
||||
Build software by talking to an AI agent instead of typing every line. Create a project for storage, attach a container to it (the shared `ppy` image, your files mounted at `/app`), start it, and open a terminal. The whole flow is drivable conversationally through Devii. Inside every container three agents ship preinstalled and run on **your own API key**, so all AI usage is metered to your account: **DevPlace Code (`dpc`)**, a coding agent in the same class as Claude Code; **`botje.py`**, a plug-and-play DevPlace bot you can copy and customise; and **`pagent`**, a minimal zero-dependency agent. Each container is launched with `DEVPLACE_BASE_URL`, `DEVPLACE_OPENAI_URL`, `DEVPLACE_API_KEY`, `DEVPLACE_USER_UID`, `DEVPLACE_CONTAINER_NAME`, `DEVPLACE_CONTAINER_UID`, and `DEVPLACE_INGRESS_URL` already set. Publish a container port to a public URL at `/p/<slug>` by setting an `ingress_slug` and `ingress_port` (ask Devii to do it at create time). The feature is in **Alpha** and currently limited to administrators; the full walkthrough, including a tutorial that vibes a web app and puts it online, is at `/docs/getting-started-vibing.html`.
|
||||
|
||||
## Admin: Audit Log
|
||||
|
||||
@ -195,7 +195,7 @@ Numeric values are floored to safe minimums so an invalid entry cannot lock out
|
||||
|
||||
The website uses a `session` cookie. For automation, every page and action also
|
||||
accepts three header-based methods, resolved centrally in `get_current_user`
|
||||
(`utils.py`) so they work everywhere with no per-route changes:
|
||||
(`utils/`) so they work everywhere with no per-route changes:
|
||||
|
||||
- **API key** - `X-API-KEY: <key>`
|
||||
- **Bearer** - `Authorization: Bearer <key>`
|
||||
@ -221,7 +221,7 @@ Every endpoint that renders a page or returns a redirect also speaks JSON, so an
|
||||
website does is automatable from the same URLs. A request gets JSON when it sends
|
||||
`Accept: application/json` or `Content-Type: application/json`; a normal browser navigation
|
||||
(`Accept: text/html`) always gets HTML, so existing behaviour is unchanged (the legacy
|
||||
`X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas.py` and built
|
||||
`X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas/` and built
|
||||
from the same context the templates use (sensitive user fields like `email`/`api_key`/
|
||||
`password_hash` are never exposed). Page GETs return the page payload; form actions return a
|
||||
uniform envelope `{ "ok": true, "redirect": "…", "data": {…} }`; errors return
|
||||
@ -379,6 +379,8 @@ and its full configuration are documented automatically - including future servi
|
||||
|
||||
`DeepsearchService` powers the public **Tools -> DeepSearch** researcher. Given a single research question it plans a set of diverse web search queries, crawls and reads the most relevant sources in a subprocess (plain HTTP first, headless-browser fallback for JavaScript-heavy pages, PDF documents streamed and text-extracted, every URL SSRF-guarded), de-duplicates content, and indexes everything into a per-session ChromaDB vector collection (embeddings via the AI gateway with a local fallback). A chain of agents (summarizer, critic, linker) then synthesises a cited report with a confidence score, source diversity and explicit gap analysis. Progress streams live over `WS /tools/deepsearch/{uid}/ws`; the report is at `/tools/deepsearch/{uid}/session` (HTML or JSON) and can be exported as Markdown, JSON or PDF. A grounded chat over the session's evidence runs at `WS /tools/deepsearch/{uid}/chat` using hybrid retrieval (vector + keyword/BM25). Runs can be paused, resumed or cancelled. A cross-session URL cache avoids re-fetching pages seen by earlier runs. CLI: `devplace deepsearch prune` / `devplace deepsearch clear`. ChromaDB, weasyprint, pypdf and Playwright are core dependencies.
|
||||
|
||||
`IsslopService` powers the public **Tools -> AI Usage Analyzer**, which classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Sources are acquired in an isolated subprocess (git URLs are probed with `git ls-remote` and shallow-cloned with a 3 GB guard; websites render in a stealth headless browser with an HTTP fallback, bounded by depth, file and byte caps), inventoried with exclusion rules, and scored by a multi-signal static engine (twenty-one detector families, 126 signal types across an origin axis and a quality-deficit axis). For a live website, the analyzer also opens the home page in a headless browser and inspects what actually renders: it fingerprints AI website-builder platforms directly (Lovable, Bolt.new, Framer and others) and checks the page's real computed styles, layout and build artifacts, not just its file contents. Representative files receive an AI review pass and images an AI-generation review through the internal gateway (model `molodetz`, internal key); the static engine stays authoritative when the gateway is unreachable. Every pipeline step is persisted as an ordered event trail and streamed live over the pub/sub topic `public.isslop.{uid}`. The verdict is an A-F authenticity grade with a human/AI split and one of five categories (`ai-slop`, `sophisticated-ai`, `human-clean`, `human-messy`, `uncertain`), published as a persistent report with an embeddable SVG badge. Members keep their analysis history on their account; guest history is session-bound and claimed by the account on the first signed-in visit. Admin settings (private-host allowance, AI/image review toggles, image cap, retention, concurrency) live on `/admin/services`. CLI: `devplace isslop analyze <url>` / `devplace isslop prune` / `devplace isslop clear`. Playwright plus playwright-stealth back the website crawler.
|
||||
|
||||
`BackupService` powers the admin **Admin -> Backups** dashboard, an enterprise-grade backup system that runs entirely as asynchronous jobs so it never impacts the running server. An administrator can back up one of four targets: the **database** (a consistent SQLite snapshot of the main database and the Devii task/lesson databases, taken with SQLite's online backup API so it is consistent under WAL), **uploads** (every attachment and project file), **keys and config** (VAPID keys), or the **full data directory** (database snapshot, uploads, and keys in one archive, excluding regenerable staging, locks, caches, and container workspaces). Each backup is compressed to a `tar.gz` in a stdlib subprocess off the request path and recorded with its size, file count, and a SHA-256 checksum. Archives live under `data/backups/` (sharded on the random uuid tail) and are served only through `/admin/backups/{uid}/download`, which is restricted to the **primary administrator** - the first user created with the Admin role. Every other administrator receives a 403 from the endpoint and sees the Download button disabled with the tooltip `Not available`; creating, running, deleting, and scheduling backups remain available to all administrators. The dashboard reports detailed storage usage - the size and file count of every major data area, the total data-directory footprint, the total size and count of stored backups, and disk usage (total, used, free, percent), computed in a worker thread and cached briefly so the page never blocks. Backups can be **scheduled** (CRUD) on an interval or 5-field cron expression with a `keep_last` rotation count that prunes older backups of the same schedule; the service evaluates schedules only on the lock-owning worker so each fires exactly once. Backup archives are permanent operational artifacts: job retention only removes the tracking row, never the archive, which is deleted only by an administrator, by schedule rotation, or via the CLI. CLI: `devplace backups list` / `devplace backups run <target>` / `devplace backups prune` / `devplace backups clear`. Devii tools: `backups_overview`, `backup_run`, `backup_status`, `backup_delete`, `backup_schedule_create`, `backup_schedule_delete` (all admin-only). The service creates and stores backups but does not restore them into a live server; restore is a documented manual procedure (stop the server, unpack the archive over the data directory, verify the checksum, restart).
|
||||
|
||||
### Adding a service
|
||||
@ -448,7 +450,7 @@ Configuration on the Services tab:
|
||||
| `bot_max_per_article` | `2` | How many bots may post about one article, each from a different angle |
|
||||
| `bot_article_ttl_days` | `7` | How long an article stays covered before it can be posted again |
|
||||
| `bot_gist_min_lines` | `6` | Reject generated snippets shorter than this many non-empty lines |
|
||||
| `bot_action_pause_min_seconds` / `bot_action_pause_max_seconds` | `5` / `30` | Idle pause window a bot takes after each action |
|
||||
| `bot_action_pause_min_seconds` / `bot_action_pause_max_seconds` | `5` / `45` | Idle pause window a bot takes after each action |
|
||||
| `bot_break_scale` | `1.0` | Multiplier on between-session breaks (below 1 = more active and costlier) |
|
||||
| `bot_ai_decisions` | disabled | Let each bot's AI-generated identity decide every action via the LLM instead of fixed probabilities (one decision call per page); see `aibots.md` |
|
||||
| `bot_decision_temperature` | `0.4` | Sampling temperature for the per-page decision call |
|
||||
@ -584,9 +586,9 @@ Configuration on the Services tab:
|
||||
|
||||
| Parameter | Default | Purpose |
|
||||
|-----------|---------|---------|
|
||||
| `devii_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | OpenAI-compatible reasoning endpoint |
|
||||
| `devii_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | OpenAI-compatible reasoning endpoint (defaults to the internal gateway) |
|
||||
| `devii_ai_model` | `molodetz` | Model name |
|
||||
| `devii_ai_key` | env fallback (`DEVII_AI_KEY`) | AI API key |
|
||||
| `devii_ai_key` | env fallback (`DEVII_AI_KEY`), then the gateway internal key | AI API key |
|
||||
| `devii_base_url` | this instance's origin | Platform Devii drives via each user's API key |
|
||||
| `devii_plan_required` / `devii_verify_required` | on / on | Enforce plan-first and verify-after-mutation |
|
||||
| `devii_max_iterations` | `40` | Tool-loop iterations per turn |
|
||||
@ -719,7 +721,7 @@ installable Progressive Web App. Push uses only standard libraries (`cryptograph
|
||||
|
||||
### Events
|
||||
|
||||
Every event flows through a single funnel - `create_notification()` in `utils.py` -
|
||||
Every event flows through a single funnel - `create_notification()` in `utils/` -
|
||||
which delivers on three independent channels, in-app, web push and Telegram, each gated by the
|
||||
recipient's preferences (see "Configurable notifications" below). Whenever the in-app
|
||||
channel delivers, the recipient's open browser also raises a live, click-through toast
|
||||
@ -790,8 +792,8 @@ every page load. `PushManager.js` owns registration, subscription, and the opt-i
|
||||
|
||||
### PWA
|
||||
|
||||
`manifest.json` (192/512 and maskable icons), `service-worker.js`, and an install
|
||||
button (`PwaInstaller.js`) make the app installable. The service worker uses a
|
||||
`manifest.json` (192/512 and maskable icons) and `service-worker.js` make the app
|
||||
installable via the browser's native install affordance. The service worker uses a
|
||||
network-first strategy for navigations and falls back to `static/offline.html` when
|
||||
offline. Installation requires a secure origin (HTTPS, or `localhost` for development).
|
||||
|
||||
@ -800,7 +802,6 @@ offline. Installation requires a secure origin (HTTPS, or `localhost` for develo
|
||||
| `devplacepy/push.py` | VAPID keys, payload encryption, send, register |
|
||||
| `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
|
||||
| `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
|
||||
| `static/js/PwaInstaller.js` | `beforeinstallprompt` capture + install button |
|
||||
| `static/service-worker.js` | Receives push, shows notification, offline fallback |
|
||||
| `static/manifest.json` | PWA manifest (icons, display, theme) |
|
||||
| `static/offline.html` | Offline fallback page |
|
||||
@ -846,7 +847,7 @@ Two background services bridge persisted state onto the bus so the interface upd
|
||||
|
||||
## Testing
|
||||
|
||||
- **932 tests** split into three tiers under `tests/`: `unit/` (pure in-process), `api/` (HTTP integration against the live server), and `e2e/` (Playwright browser)
|
||||
- **1959 tests** split into three tiers under `tests/`: `unit/` (pure in-process), `api/` (HTTP integration against the live server), and `e2e/` (Playwright browser)
|
||||
- **A directory tree that mirrors the path.** api/e2e follow the endpoint path - each route segment is a directory and the last segment is the file, `{param}` segments dropped (`GET /admin/ai-usage` -> `tests/e2e/admin/aiusage.py`, `GET /projects/{slug}/files/lines` -> `tests/api/projects/files/lines.py`). unit mirrors the source module path (`devplacepy/services/audit/store.py` -> `tests/unit/services/audit/store.py`). Run one tier with `make test-unit` / `make test-api` / `make test-e2e`
|
||||
- Playwright (NOT pytest-playwright plugin - conflicts, uninstall it)
|
||||
- Runs serially, one test at a time, in a single process (`make test`); the suite drives one uvicorn subprocess on port 10501 with its own temp database and `DEVPLACE_DATA_DIR`
|
||||
@ -912,12 +913,12 @@ The Container Manager drives the host Docker daemon, so `make docker-build`/`mak
|
||||
What the overlay (`docker-compose.containers.yml`) changes:
|
||||
|
||||
- **Docker CLI in the image** via the `INSTALL_DOCKER_CLI=true` build arg (the base image stays lean).
|
||||
- **Docker socket** mounted into the app container. This grants the app **root on the host** - every build/run/exec is admin-only, `--privileged` is never used, and all docker calls are argument-list subprocesses, but treat the whole feature as trusted-admins-only.
|
||||
- **Docker socket** mounted into the app container. This grants the app **root on the host** - every run/exec/lifecycle operation is admin-only, `--privileged` is never used, and all docker calls are argument-list subprocesses, but treat the whole feature as trusted-admins-only.
|
||||
- **Socket permissions:** the app runs as UID 1000, so the overlay adds the host `docker` group via `group_add`. `make` reads the gid straight from `/var/run/docker.sock` (`stat -c '%g'`), the exact group that owns the socket.
|
||||
- **Data dir at a consistent path (critical).** When the app (in its container) runs `docker run -v <path>:/app`, the daemon resolves `<path>` against the **host**, not the app container. So the workspace/data dir must be mounted at the **same absolute path** on host and in the container - the make targets set `DEVPLACE_DATA_DIR` to the project's `./data` (an absolute host path) and mount it at that identical path on both sides. (Build contexts go through the docker API as a tarball, so they can stay in the container's temp dir - only the `/app` bind mount needs path consistency.)
|
||||
- **Ingress reach:** published container ports live on the **host**, so the overlay sets `DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal` (with `extra_hosts: host-gateway`) so the `/p/<slug>` proxy can reach them. On a bare-metal `make prod` deploy the app is already on the host, so the default `127.0.0.1` works and no overlay is needed (just install the docker CLI and run the services).
|
||||
|
||||
Then enable **Container builds** and **Containers** on `/admin/services`. Builds default to `--network=host` (configurable on the service) so pip can reach PyPI; set the build network to empty to use the docker default.
|
||||
Then build the shared `ppy` image once with `make ppy` and enable **Containers** on `/admin/services`. There is no in-app image building; every instance runs that one prebuilt image.
|
||||
|
||||
### nginx specifics
|
||||
|
||||
@ -944,7 +945,7 @@ The version sits in the **path**, not a query string, because the frontend is un
|
||||
|
||||
### Bare-metal alternative
|
||||
|
||||
`make prod` runs the same app without containers (`uvicorn ... --workers 2 --proxy-headers`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
|
||||
`make prod` runs the same app without containers (`uvicorn ... --workers $(WEB_WORKERS) --proxy-headers`, where `WEB_WORKERS` defaults to `nproc`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
|
||||
|
||||
### Multi-worker safety
|
||||
|
||||
|
||||
@ -21,6 +21,9 @@ from devplacepy.cli.jobs import (
|
||||
cmd_forks_clear,
|
||||
cmd_seo_prune,
|
||||
cmd_seo_clear,
|
||||
cmd_isslop_prune,
|
||||
cmd_isslop_clear,
|
||||
cmd_isslop_analyze,
|
||||
cmd_seo_meta_prune,
|
||||
cmd_seo_meta_clear,
|
||||
cmd_deepsearch_prune,
|
||||
@ -65,6 +68,9 @@ __all__ = [
|
||||
"cmd_forks_clear",
|
||||
"cmd_seo_prune",
|
||||
"cmd_seo_clear",
|
||||
"cmd_isslop_prune",
|
||||
"cmd_isslop_clear",
|
||||
"cmd_isslop_analyze",
|
||||
"cmd_seo_meta_prune",
|
||||
"cmd_seo_meta_clear",
|
||||
"cmd_deepsearch_prune",
|
||||
|
||||
@ -210,6 +210,104 @@ def cmd_deepsearch_clear(args):
|
||||
print(f"Cleared {len(jobs)} DeepSearch job(s) and their collections")
|
||||
|
||||
|
||||
def cmd_isslop_prune(args):
|
||||
from datetime import datetime, timezone
|
||||
from devplacepy.services.jobs import queue
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
removed = 0
|
||||
for job in queue.list_jobs(kind="isslop", status=queue.DONE):
|
||||
expires_at = job.get("expires_at")
|
||||
if not expires_at:
|
||||
continue
|
||||
try:
|
||||
expiry = datetime.fromisoformat(expires_at)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if expiry < now:
|
||||
get_table("jobs").delete(uid=job["uid"])
|
||||
removed += 1
|
||||
_audit_cli("cli.isslop.prune", f"CLI pruned {removed} expired AI usage analysis jobs", metadata={"count": removed})
|
||||
print(f"Pruned {removed} expired AI usage analysis job(s) (reports persist)")
|
||||
|
||||
|
||||
def cmd_isslop_clear(args):
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.jobs.isslop import store
|
||||
|
||||
jobs = queue.list_jobs(kind="isslop")
|
||||
for job in jobs:
|
||||
get_table("jobs").delete(uid=job["uid"])
|
||||
analyses = list(get_table(store.TABLE_ANALYSES).find())
|
||||
for analysis in analyses:
|
||||
store.purge_analysis(analysis["uid"])
|
||||
_audit_cli(
|
||||
"cli.isslop.clear",
|
||||
f"CLI cleared {len(analyses)} AI usage analyses and {len(jobs)} job rows",
|
||||
metadata={"analyses": len(analyses), "jobs": len(jobs)},
|
||||
)
|
||||
print(f"Cleared {len(analyses)} AI usage analysis(es), their reports and {len(jobs)} job row(s)")
|
||||
|
||||
|
||||
def cmd_isslop_analyze(args):
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from devplacepy.config import ISSLOP_WORKSPACES_DIR, ensure_data_dirs
|
||||
from devplacepy.database import INTERNAL_GATEWAY_URL, internal_gateway_key
|
||||
from devplacepy.models import IsslopRunForm
|
||||
from devplacepy.services.jobs.isslop import store
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import remove_workspace, workspace_for
|
||||
from devplacepy.services.jobs.isslop.config import settings_from_payload
|
||||
from devplacepy.services.jobs.isslop.events import KIND_DONE, KIND_ERROR
|
||||
from devplacepy.services.jobs.isslop.persistence import EventPersister
|
||||
from devplacepy.services.jobs.isslop.pipeline import run_pipeline
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
url = IsslopRunForm(url=args.url).url
|
||||
ensure_data_dirs()
|
||||
uid = generate_uid()
|
||||
settings = settings_from_payload(
|
||||
{
|
||||
"url": url,
|
||||
"llm_endpoint": INTERNAL_GATEWAY_URL,
|
||||
"api_key": internal_gateway_key(),
|
||||
"allow_private": bool(args.allow_private),
|
||||
"media_dir": str(store.media_dir_for(uid)),
|
||||
}
|
||||
)
|
||||
store.create_analysis(uid, url, "system", "cli")
|
||||
persister = EventPersister(uid)
|
||||
store.update_analysis(uid, status="running")
|
||||
workspace = workspace_for(ISSLOP_WORKSPACES_DIR, url, uid)
|
||||
|
||||
async def run() -> int:
|
||||
failed = False
|
||||
try:
|
||||
async for event in run_pipeline(url, workspace, settings):
|
||||
persister.apply(event)
|
||||
if args.json:
|
||||
print(event.to_json(), flush=True)
|
||||
else:
|
||||
print(f"[{event.kind}] {event.message}", flush=True)
|
||||
if event.kind == KIND_ERROR:
|
||||
failed = True
|
||||
if event.kind == KIND_DONE and not args.json:
|
||||
print(f"Report: /tools/isslop/{uid}/report")
|
||||
print(f"Badge: /tools/isslop/{uid}/badge.svg")
|
||||
finally:
|
||||
remove_workspace(workspace)
|
||||
return 1 if failed else 0
|
||||
|
||||
exit_code = asyncio.run(run())
|
||||
_audit_cli(
|
||||
"cli.isslop.analyze",
|
||||
f"CLI AI usage analysis of {url}",
|
||||
metadata={"uid": uid, "failed": bool(exit_code)},
|
||||
)
|
||||
raise SystemExit(exit_code)
|
||||
|
||||
|
||||
def register_jobs(subparsers):
|
||||
zips = subparsers.add_parser("zips", help="Zip archive job management")
|
||||
zips_sub = zips.add_subparsers(title="action", dest="action")
|
||||
@ -265,3 +363,21 @@ def register_jobs(subparsers):
|
||||
"clear", help="Delete every DeepSearch session and job row"
|
||||
)
|
||||
deepsearch_clear.set_defaults(func=cmd_deepsearch_clear)
|
||||
|
||||
isslop = subparsers.add_parser("isslop", help="AI Usage Analyzer job management")
|
||||
isslop_sub = isslop.add_subparsers(title="action", dest="action")
|
||||
isslop_prune = isslop_sub.add_parser(
|
||||
"prune", help="Delete expired AI usage analysis job rows (analyses and reports persist)"
|
||||
)
|
||||
isslop_prune.set_defaults(func=cmd_isslop_prune)
|
||||
isslop_clear = isslop_sub.add_parser(
|
||||
"clear", help="Delete every AI usage analysis, its report and job rows"
|
||||
)
|
||||
isslop_clear.set_defaults(func=cmd_isslop_clear)
|
||||
isslop_analyze = isslop_sub.add_parser(
|
||||
"analyze", help="Run a AI usage analysis from the terminal and persist its report"
|
||||
)
|
||||
isslop_analyze.add_argument("url", help="Repository or website URL to classify")
|
||||
isslop_analyze.add_argument("--json", action="store_true", help="Emit raw JSON events")
|
||||
isslop_analyze.add_argument("--allow-private", action="store_true", dest="allow_private", help="Permit private and loopback hosts")
|
||||
isslop_analyze.set_defaults(func=cmd_isslop_analyze)
|
||||
|
||||
@ -26,6 +26,10 @@ PLANNING_REPORTS_DIR = DATA_DIR / "planning_reports"
|
||||
DBAPI_DIR = DATA_DIR / "dbapi"
|
||||
DEEPSEARCH_DIR = DATA_DIR / "deepsearch"
|
||||
DEEPSEARCH_CHROMA_DIR = DEEPSEARCH_DIR / "chroma"
|
||||
ISSLOP_DIR = DATA_DIR / "isslop"
|
||||
ISSLOP_WORKSPACES_DIR = ISSLOP_DIR / "workspaces"
|
||||
ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs"
|
||||
ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
|
||||
KEYS_DIR = DATA_DIR / "keys"
|
||||
BOT_DIR = DATA_DIR / "bot"
|
||||
LOCKS_DIR = DATA_DIR / "locks"
|
||||
@ -96,6 +100,10 @@ DATA_PATHS: dict[str, Path] = {
|
||||
"dbapi": DBAPI_DIR,
|
||||
"deepsearch": DEEPSEARCH_DIR,
|
||||
"deepsearch_chroma": DEEPSEARCH_CHROMA_DIR,
|
||||
"isslop": ISSLOP_DIR,
|
||||
"isslop_workspaces": ISSLOP_WORKSPACES_DIR,
|
||||
"isslop_runs": ISSLOP_RUNS_DIR,
|
||||
"isslop_media": ISSLOP_MEDIA_DIR,
|
||||
"keys": KEYS_DIR,
|
||||
"bot": BOT_DIR,
|
||||
"locks": LOCKS_DIR,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "signals"]
|
||||
TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "politics"]
|
||||
|
||||
REACTION_EMOJI = [
|
||||
"\U0001f44d",
|
||||
|
||||
@ -19,6 +19,7 @@ from devplacepy.database import (
|
||||
get_blocked_uids,
|
||||
get_poll_for_post,
|
||||
update_target_stars,
|
||||
clear_user_stars,
|
||||
get_target_owner_uid,
|
||||
resolve_object_url,
|
||||
soft_delete,
|
||||
@ -121,6 +122,10 @@ def create_content_item(
|
||||
**fields,
|
||||
}
|
||||
)
|
||||
if table_name == "projects":
|
||||
from devplacepy.templating import clear_user_projects_cache
|
||||
|
||||
clear_user_projects_cache(user["uid"])
|
||||
award_rewards(user["uid"], xp, badge)
|
||||
if attachment_uids:
|
||||
link_attachments(attachment_uids, target_type, uid)
|
||||
@ -209,6 +214,8 @@ def apply_vote(request, user: dict, target_type: str, target_uid: str, value: in
|
||||
update_target_stars(target_type, target_uid, net)
|
||||
|
||||
owner_uid = get_target_owner_uid(target_type, target_uid)
|
||||
if owner_uid:
|
||||
clear_user_stars(owner_uid)
|
||||
direction = "clear" if new_value == 0 else ("up" if new_value == 1 else "down")
|
||||
vote_links = [audit.target(target_type, target_uid)]
|
||||
if owner_uid and owner_uid != user["uid"]:
|
||||
@ -570,9 +577,11 @@ def delete_content_item(
|
||||
soft_delete_engagement("comment", comment_uids, actor)
|
||||
if target_type == "project":
|
||||
from devplacepy.project_files import soft_delete_all_project_files
|
||||
from devplacepy.templating import clear_user_projects_cache
|
||||
|
||||
soft_delete_all_project_files(item["uid"], actor)
|
||||
soft_delete_fork_relations(item["uid"], actor)
|
||||
clear_user_projects_cache(item["user_uid"])
|
||||
soft_delete(table_name, actor, stamp=stamp, uid=item["uid"])
|
||||
logger.info(f"{table_name} {item['uid']} soft-deleted by {user['username']}")
|
||||
audit.record(
|
||||
|
||||
@ -17,7 +17,7 @@ from .notifications import NOTIFICATION_TYPES, NOTIFICATION_CHANNELS, _NOTIFICAT
|
||||
from .forks import record_fork, get_fork_parent, count_forks, soft_delete_fork_relations, delete_fork_relations
|
||||
from .follows import get_follow_counts, get_follow_list, get_following_among
|
||||
from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
|
||||
from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
|
||||
from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
|
||||
from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
|
||||
from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news
|
||||
from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_deleted_media
|
||||
@ -191,6 +191,7 @@ __all__ = [
|
||||
"get_leaderboard",
|
||||
"get_user_rank",
|
||||
"get_user_stars",
|
||||
"clear_user_stars",
|
||||
"update_target_stars",
|
||||
"soft_delete_engagement",
|
||||
"delete_engagement",
|
||||
|
||||
@ -16,7 +16,10 @@ VOTABLE_TARGETS: dict[str, str] = {
|
||||
STAR_TARGETS: set[str] = {"post", "project", "gist"}
|
||||
|
||||
|
||||
_authors_cache = TTLCache(ttl=300, max_size=200)
|
||||
_authors_cache = TTLCache(ttl=15, max_size=200)
|
||||
|
||||
|
||||
_stars_cache = TTLCache(ttl=15, max_size=2000)
|
||||
|
||||
|
||||
def _ranked_authors() -> list:
|
||||
@ -84,7 +87,14 @@ def get_user_rank(user_uid: str):
|
||||
return _rank_map().get(user_uid)
|
||||
|
||||
|
||||
def clear_user_stars(user_uid: str) -> None:
|
||||
_stars_cache.pop(user_uid)
|
||||
|
||||
|
||||
def get_user_stars(user_uid: str) -> int:
|
||||
cached = _stars_cache.get(user_uid)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if "votes" not in db.tables:
|
||||
return 0
|
||||
target_union = " UNION ALL ".join(
|
||||
@ -94,14 +104,17 @@ def get_user_stars(user_uid: str) -> int:
|
||||
)
|
||||
if not target_union:
|
||||
return 0
|
||||
total = 0
|
||||
for row in db.query(
|
||||
f"SELECT COALESCE(SUM(v.value), 0) AS s "
|
||||
f"FROM votes v JOIN ({target_union}) t ON v.target_uid = t.uid AND v.target_type = t.target_type "
|
||||
f"WHERE v.deleted_at IS NULL",
|
||||
u=user_uid,
|
||||
):
|
||||
return row["s"] or 0
|
||||
return 0
|
||||
total = row["s"] or 0
|
||||
break
|
||||
_stars_cache.set(user_uid, total)
|
||||
return total
|
||||
|
||||
|
||||
def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> None:
|
||||
@ -110,7 +123,6 @@ def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> No
|
||||
return
|
||||
if target_type in STAR_TARGETS:
|
||||
get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
|
||||
_authors_cache.clear()
|
||||
|
||||
|
||||
def soft_delete_engagement(target_type: str, target_uids: list, deleted_by: str) -> None:
|
||||
|
||||
@ -37,9 +37,11 @@ def init_db():
|
||||
_index(db, "users", "idx_users_api_key", ["api_key"])
|
||||
_index(db, "users", "idx_users_role", ["role", "created_at"])
|
||||
_index(db, "users", "idx_users_last_seen", ["last_seen"])
|
||||
_index(db, "users", "idx_users_created_at", ["created_at"])
|
||||
_index(db, "posts", "idx_posts_user_uid", ["user_uid"])
|
||||
_index(db, "posts", "idx_posts_created_at", ["created_at"])
|
||||
_index(db, "posts", "idx_posts_topic", ["topic"])
|
||||
_index(db, "posts", "idx_posts_slug", ["slug"])
|
||||
if "posts" in tables:
|
||||
posts_table = get_table("posts")
|
||||
if not posts_table.has_column("tags"):
|
||||
@ -120,6 +122,15 @@ def init_db():
|
||||
_index(db, "votes", "idx_votes_target", ["target_uid", "target_type"])
|
||||
_index(db, "messages", "idx_messages_sender", ["sender_uid"])
|
||||
_index(db, "messages", "idx_messages_receiver", ["receiver_uid"])
|
||||
_index(
|
||||
db, "messages", "idx_messages_conversation", ["sender_uid", "receiver_uid"]
|
||||
)
|
||||
_index(
|
||||
db,
|
||||
"messages",
|
||||
"idx_messages_conversation_rev",
|
||||
["receiver_uid", "sender_uid"],
|
||||
)
|
||||
_index(db, "notifications", "idx_notifications_user", ["user_uid"])
|
||||
_index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
|
||||
_index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
|
||||
@ -140,6 +151,7 @@ def init_db():
|
||||
projects.create_column_by_example(column, example)
|
||||
|
||||
_index(db, "projects", "idx_projects_user", ["user_uid"])
|
||||
_index(db, "projects", "idx_projects_slug", ["slug"])
|
||||
_index(db, "projects", "idx_projects_private", ["is_private"])
|
||||
_index(db, "projects", "idx_projects_stars", ["stars"])
|
||||
_index(db, "projects", "idx_projects_type", ["project_type"])
|
||||
@ -170,6 +182,7 @@ def init_db():
|
||||
db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"]
|
||||
)
|
||||
_index(db, "badges", "idx_badges_user", ["user_uid"])
|
||||
_index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
|
||||
_index(db, "follows", "idx_follows_follower", ["follower_uid"])
|
||||
_index(db, "follows", "idx_follows_following", ["following_uid"])
|
||||
user_relations = get_table("user_relations")
|
||||
@ -189,6 +202,7 @@ def init_db():
|
||||
_index(db, "password_resets", "idx_password_resets_token", ["token"])
|
||||
_index(db, "gists", "idx_gists_user_uid", ["user_uid"])
|
||||
_index(db, "gists", "idx_gists_language", ["language"])
|
||||
_index(db, "gists", "idx_gists_slug", ["slug"])
|
||||
attachments = get_table("attachments")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
@ -244,6 +258,7 @@ def init_db():
|
||||
db["site_settings"].insert(
|
||||
{"uid": f"default_{key}", "key": key, "value": value}
|
||||
)
|
||||
_index(db, "site_settings", "idx_site_settings_key", ["key"])
|
||||
|
||||
news = get_table("news")
|
||||
for column, example in (
|
||||
@ -272,6 +287,7 @@ def init_db():
|
||||
news.create_column_by_example(column, example)
|
||||
|
||||
_index(db, "news", "idx_news_external_id", ["external_id"])
|
||||
_index(db, "news", "idx_news_slug", ["slug"])
|
||||
_index(db, "news", "idx_news_synced_at", ["synced_at"])
|
||||
_index(db, "news", "idx_news_status", ["status"])
|
||||
_index(db, "news", "idx_news_featured", ["featured"])
|
||||
@ -457,6 +473,8 @@ def init_db():
|
||||
instances.create_column_by_example(column, example)
|
||||
|
||||
_index(db, "instances", "idx_instances_project", ["project_uid"])
|
||||
_index(db, "instances", "idx_instances_slug", ["slug"])
|
||||
_index(db, "instances", "idx_instances_name", ["name"])
|
||||
_index(db, "instances", "idx_instances_state", ["desired_state", "status"])
|
||||
_index(db, "instances", "idx_instances_container", ["container_id"])
|
||||
_index(db, "instances", "idx_instances_ingress", ["ingress_slug"])
|
||||
@ -791,6 +809,108 @@ def init_db():
|
||||
deepsearch_url_cache.create_column_by_example(column, example)
|
||||
_index(db, "deepsearch_url_cache", "idx_deepsearch_url_cache_hash", ["url_hash"])
|
||||
|
||||
isslop_analyses = get_table("isslop_analyses")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("owner_kind", ""),
|
||||
("owner_id", ""),
|
||||
("source_url", ""),
|
||||
("source_kind", ""),
|
||||
("status", ""),
|
||||
("created_at", ""),
|
||||
("finished_at", ""),
|
||||
("content_hash", ""),
|
||||
("grade", ""),
|
||||
("slop_score", 0.0),
|
||||
("origin_score", 0.0),
|
||||
("quality_deficit_score", 0.0),
|
||||
("human_percent", 0.0),
|
||||
("ai_percent", 0.0),
|
||||
("category", ""),
|
||||
("confidence", ""),
|
||||
("files_total", 0),
|
||||
("files_analyzed", 0),
|
||||
("error_message_text", ""),
|
||||
("detected_builder", ""),
|
||||
("dom_slop_score", 0.0),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
):
|
||||
if not isslop_analyses.has_column(column):
|
||||
isslop_analyses.create_column_by_example(column, example)
|
||||
_index(db, "isslop_analyses", "idx_isslop_analyses_owner", ["owner_kind", "owner_id", "created_at"])
|
||||
_index(db, "isslop_analyses", "idx_isslop_analyses_status", ["status"])
|
||||
_index(db, "isslop_analyses", "idx_isslop_analyses_hash", ["content_hash"])
|
||||
|
||||
isslop_events = get_table("isslop_events")
|
||||
for column, example in (
|
||||
("analysis_uid", ""),
|
||||
("seq", 0),
|
||||
("kind", ""),
|
||||
("message", ""),
|
||||
("payload", ""),
|
||||
("created_at", ""),
|
||||
):
|
||||
if not isslop_events.has_column(column):
|
||||
isslop_events.create_column_by_example(column, example)
|
||||
_index(db, "isslop_events", "idx_isslop_events_analysis", ["analysis_uid", "seq"])
|
||||
|
||||
isslop_file_results = get_table("isslop_file_results")
|
||||
for column, example in (
|
||||
("analysis_uid", ""),
|
||||
("path", ""),
|
||||
("language", ""),
|
||||
("lines", 0),
|
||||
("origin_score", 0.0),
|
||||
("quality_deficit_score", 0.0),
|
||||
("category", ""),
|
||||
("signals", ""),
|
||||
("source", ""),
|
||||
):
|
||||
if not isslop_file_results.has_column(column):
|
||||
isslop_file_results.create_column_by_example(column, example)
|
||||
_index(db, "isslop_file_results", "idx_isslop_file_results_analysis", ["analysis_uid"])
|
||||
|
||||
isslop_image_results = get_table("isslop_image_results")
|
||||
for column, example in (
|
||||
("analysis_uid", ""),
|
||||
("path", ""),
|
||||
("ai_probability", 0.0),
|
||||
("grade", ""),
|
||||
("verdict", ""),
|
||||
("image_kind", ""),
|
||||
("tells", ""),
|
||||
("description", ""),
|
||||
("thumb", ""),
|
||||
):
|
||||
if not isslop_image_results.has_column(column):
|
||||
isslop_image_results.create_column_by_example(column, example)
|
||||
_index(db, "isslop_image_results", "idx_isslop_image_results_analysis", ["analysis_uid"])
|
||||
|
||||
isslop_dom_results = get_table("isslop_dom_results")
|
||||
for column, example in (
|
||||
("analysis_uid", ""),
|
||||
("url", ""),
|
||||
("detected_builder", ""),
|
||||
("signal_count", 0),
|
||||
("screenshot", ""),
|
||||
("signals", ""),
|
||||
):
|
||||
if not isslop_dom_results.has_column(column):
|
||||
isslop_dom_results.create_column_by_example(column, example)
|
||||
_index(db, "isslop_dom_results", "idx_isslop_dom_results_analysis", ["analysis_uid"])
|
||||
|
||||
isslop_reports = get_table("isslop_reports")
|
||||
for column, example in (
|
||||
("analysis_uid", ""),
|
||||
("markdown", ""),
|
||||
("model_used", ""),
|
||||
("generated_at", ""),
|
||||
):
|
||||
if not isslop_reports.has_column(column):
|
||||
isslop_reports.create_column_by_example(column, example)
|
||||
_index(db, "isslop_reports", "idx_isslop_reports_analysis", ["analysis_uid"], unique=True)
|
||||
|
||||
game_farms = get_table("game_farms")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
|
||||
@ -35,6 +35,7 @@ SOFT_DELETE_TABLES = [
|
||||
"notification_preferences",
|
||||
"deepsearch_sessions",
|
||||
"deepsearch_messages",
|
||||
"isslop_analyses",
|
||||
"devrant_tokens",
|
||||
"access_tokens",
|
||||
"email_accounts",
|
||||
|
||||
@ -458,7 +458,7 @@ four ways to sign requests.
|
||||
"string",
|
||||
True,
|
||||
"vote",
|
||||
"One of: comment, reply, mention, vote, follow, message, badge, level, issue.",
|
||||
"One of: comment, reply, mention, vote, follow, message, badge, level, issue, reminder, harvest_stolen.",
|
||||
),
|
||||
field(
|
||||
"channel",
|
||||
|
||||
@ -22,8 +22,9 @@ distinguish the form type).
|
||||
The POST endpoints are **actions**: they accept form fields, set or clear the `session` cookie,
|
||||
and return a `302` redirect (or the JSON envelope for JSON callers).
|
||||
|
||||
**Sign-up requires a valid `g-recaptcha-response`** when reCAPTCHA is enabled. Use the JSON
|
||||
envelope to see validation errors as `{ "error": "validation", "fields": {...} }`.
|
||||
**Sign-up requires a unique `username` and `email`** plus a `confirm_password` that matches the
|
||||
password; **you log in with your `email` and password**. JSON callers receive validation errors
|
||||
as a `422` with the shape `{ "fields": {...}, "messages": [...] }`.
|
||||
""",
|
||||
"endpoints": [
|
||||
endpoint(
|
||||
@ -45,10 +46,10 @@ envelope to see validation errors as `{ "error": "validation", "fields": {...} }
|
||||
encoding="form",
|
||||
destructive=False,
|
||||
params=[
|
||||
field("username", "form", "string", True, "alice", "Username, 3-20 characters."),
|
||||
field("username", "form", "string", True, "alice", "Username, 3-32 characters (letters, numbers, hyphens, underscores)."),
|
||||
field("email", "form", "string", True, "alice@example.com", "Email address; must be unique and contain an @."),
|
||||
field("password", "form", "string", True, "mysecret", "Password, 6+ characters."),
|
||||
field("confirm_password", "form", "string", True, "mysecret", "Must match password."),
|
||||
field("g-recaptcha-response", "form", "string", False, "", "reCAPTCHA token when enabled."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
@ -68,12 +69,13 @@ envelope to see validation errors as `{ "error": "validation", "fields": {...} }
|
||||
method="POST",
|
||||
path="/auth/login",
|
||||
title="Log in",
|
||||
summary="Authenticate with username and password. Sets the session cookie.",
|
||||
summary="Authenticate with email and password. Sets the session cookie.",
|
||||
auth="public",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("username", "form", "string", True, "alice", "Your username."),
|
||||
field("email", "form", "string", True, "alice@example.com", "Your registered email."),
|
||||
field("password", "form", "string", True, "mysecret", "Your password."),
|
||||
field("remember_me", "form", "string", False, "on", "Send 'on' to extend the session to the remember-me lifetime."),
|
||||
field("next", "form", "string", False, "", "Redirect target after login."),
|
||||
],
|
||||
),
|
||||
|
||||
@ -9,9 +9,10 @@ GROUP = {
|
||||
"intro": """
|
||||
# Container Manager
|
||||
|
||||
Build versioned Docker images for a project and run supervised container instances. Every endpoint is
|
||||
**administrator only** (running arbitrary Dockerfiles with docker socket access is root-equivalent).
|
||||
Mutations flip desired state; a single reconciler converges containers to it.
|
||||
Run supervised container instances for a project. There is no in-app image building: every instance
|
||||
runs one shared prebuilt image (`ppy:latest`) with the project's workspace mounted at `/app`. Every
|
||||
endpoint is **administrator only** (docker socket access is root-equivalent). Mutations flip desired
|
||||
state; a single reconciler converges containers to it.
|
||||
""",
|
||||
"endpoints": [
|
||||
endpoint(
|
||||
@ -118,7 +119,7 @@ Mutations flip desired state; a single reconciler converges containers to it.
|
||||
"python app.py",
|
||||
"Optional boot command.",
|
||||
),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (DEVPLACE_API_KEY, DEVPLACE_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("boot_language", "form", "enum", False, "none", "Boot source language.", ["none", "python", "bash"]),
|
||||
field("boot_script", "form", "textarea", False, "print('hi')", "Boot source code run on launch (takes precedence over boot_command)."),
|
||||
field(
|
||||
@ -478,7 +479,7 @@ Mutations flip desired state; a single reconciler converges containers to it.
|
||||
params=[
|
||||
field("project_slug", "form", "string", True, "PROJECT_SLUG", "Project that becomes the /app root."),
|
||||
field("name", "form", "string", True, "staging", "Instance name."),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (DEVPLACE_API_KEY, DEVPLACE_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("boot_language", "form", "enum", False, "none", "Boot source language.", ["none", "python", "bash"]),
|
||||
field("boot_script", "form", "textarea", False, "print('hi')", "Boot source code run on launch (takes precedence over boot_command)."),
|
||||
field("boot_command", "form", "string", False, "python app.py", "Fallback boot command when no boot_script is set."),
|
||||
|
||||
580
devplacepy/docs_api/groups/containers.py.bak
Normal file
580
devplacepy/docs_api/groups/containers.py.bak
Normal file
@ -0,0 +1,580 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .._shared import endpoint, field
|
||||
|
||||
GROUP = {
|
||||
"slug": "containers",
|
||||
"title": "Container Manager",
|
||||
"admin": True,
|
||||
"intro": """
|
||||
# Container Manager
|
||||
|
||||
Run supervised container instances for a project. There is no in-app image building: every instance
|
||||
runs one shared prebuilt image (`ppy:latest`) with the project's workspace mounted at `/app`. Every
|
||||
endpoint is **administrator only** (docker socket access is root-equivalent). Mutations flip desired
|
||||
state; a single reconciler converges containers to it.
|
||||
""",
|
||||
"endpoints": [
|
||||
endpoint(
|
||||
id="containers-page",
|
||||
method="GET",
|
||||
path="/projects/{project_slug}/containers",
|
||||
title="Container manager page",
|
||||
summary="The admin per-project container manager UI (instance creation and lifecycle). Returns 404 for an administrator who is not the owner of an administrator-hidden project.",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
)
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-index",
|
||||
method="GET",
|
||||
path="/admin/containers",
|
||||
title="Admin containers list",
|
||||
summary="The admin Containers section: every instance across all projects, each linking to its detail page. Instances attached to another administrator's hidden project are excluded, and per-instance actions return 404 for a non-owner administrator.",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-data",
|
||||
method="GET",
|
||||
path="/admin/containers/data",
|
||||
title="Admin containers list data",
|
||||
summary="JSON of every instance across all projects (decorated with project title/slug) for polling.",
|
||||
auth="admin",
|
||||
sample_response={
|
||||
"instances": [
|
||||
{
|
||||
"uid": "INSTANCE_UID",
|
||||
"name": "staging",
|
||||
"status": "running",
|
||||
"project_slug": "PROJECT_SLUG",
|
||||
"project_title": "My Project",
|
||||
"ingress_slug": "my-service",
|
||||
"restart_policy": "always",
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-instance",
|
||||
method="GET",
|
||||
path="/admin/containers/{uid}",
|
||||
title="Instance detail page",
|
||||
summary="The dedicated detail page for one instance (lifecycle, logs, metrics, terminal, schedules, ingress, sync).",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
params=[
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
)
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-edit-page",
|
||||
method="GET",
|
||||
path="/admin/containers/{uid}/edit",
|
||||
title="Edit instance page",
|
||||
summary="The edit page for one instance (run-as user, boot language/script/command, restart policy, start-on-boot, limits).",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
params=[
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
)
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-create-instance",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances",
|
||||
title="Create an instance",
|
||||
summary="Create and (by default) start an instance; it runs the shared ppy image with the project workspace mounted at /app.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field("name", "form", "string", True, "staging", "Instance name."),
|
||||
field(
|
||||
"boot_command",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"python app.py",
|
||||
"Optional boot command.",
|
||||
),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("boot_language", "form", "enum", False, "none", "Boot source language.", ["none", "python", "bash"]),
|
||||
field("boot_script", "form", "textarea", False, "print('hi')", "Boot source code run on launch (takes precedence over boot_command)."),
|
||||
field(
|
||||
"env",
|
||||
"form",
|
||||
"textarea",
|
||||
False,
|
||||
"KEY=VALUE",
|
||||
"Env vars, one KEY=VALUE per line.",
|
||||
),
|
||||
field(
|
||||
"ports",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"80",
|
||||
"Port maps. Bare container port auto-assigns a unique host port above 20000; host:container pins one.",
|
||||
),
|
||||
field("cpu_limit", "form", "string", False, "1.5", "CPU limit."),
|
||||
field(
|
||||
"mem_limit", "form", "string", False, "512m", "Memory limit."
|
||||
),
|
||||
field(
|
||||
"restart_policy",
|
||||
"form",
|
||||
"enum",
|
||||
False,
|
||||
"never",
|
||||
"Restart policy.",
|
||||
["never", "always", "on-failure", "unless-stopped"],
|
||||
),
|
||||
field("start_on_boot", "form", "boolean", False, "false", "Force running whenever the container service starts."),
|
||||
field(
|
||||
"ingress_slug",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"my-service",
|
||||
"Publish at /p/<slug> (optional).",
|
||||
),
|
||||
field(
|
||||
"ingress_port",
|
||||
"form",
|
||||
"integer",
|
||||
False,
|
||||
"8899",
|
||||
"Container port to publish (must be a mapped port).",
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-ingress",
|
||||
method="GET",
|
||||
path="/p/{slug}",
|
||||
title="Container ingress proxy",
|
||||
summary="Public reverse proxy (HTTP and WebSocket) to a running instance published via ingress_slug. The /p/<slug> prefix is stripped before forwarding.",
|
||||
auth="public",
|
||||
interactive=True,
|
||||
params=[
|
||||
field(
|
||||
"slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"my-service",
|
||||
"The instance's ingress_slug.",
|
||||
)
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-action",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/{action}",
|
||||
title="Instance lifecycle",
|
||||
summary="start, stop, restart, pause, or resume an instance (flips desired state).",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
field(
|
||||
"action",
|
||||
"path",
|
||||
"enum",
|
||||
True,
|
||||
"start",
|
||||
"Lifecycle action.",
|
||||
["start", "stop", "restart", "pause", "resume"],
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-logs",
|
||||
method="GET",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/logs",
|
||||
title="Instance logs",
|
||||
summary="Recent docker logs of a running instance.",
|
||||
auth="admin",
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
field("tail", "query", "integer", False, "200", "Number of lines."),
|
||||
],
|
||||
sample_response={"logs": "..."},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-sync",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/sync",
|
||||
title="Sync workspace",
|
||||
summary="Run a one-shot bidirectional newer-wins sync between the project files and the container workspace.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
],
|
||||
sample_response={"exported": 3, "imported": 1},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-delete",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/delete",
|
||||
title="Delete instance",
|
||||
summary="Remove a container instance and mark its container for removal.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-exec",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/exec",
|
||||
title="Exec a command",
|
||||
summary="Run a one-shot command inside a running instance and return its output.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
field(
|
||||
"command",
|
||||
"form",
|
||||
"string",
|
||||
True,
|
||||
"ls -la /app",
|
||||
"Shell command to run (via /bin/sh -c).",
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-data",
|
||||
method="GET",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}",
|
||||
title="Instance detail data",
|
||||
summary="Return the full instance row plus runtime info as JSON.",
|
||||
auth="admin",
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
],
|
||||
sample_response={"uid": "INSTANCE_UID", "name": "staging", "status": "running"},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-metrics",
|
||||
method="GET",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/metrics",
|
||||
title="Instance metrics",
|
||||
summary="Return recent metrics ring-buffer and aggregated stats for a running instance.",
|
||||
auth="admin",
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
],
|
||||
sample_response={"metrics": [], "stats": {}},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-schedules",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/schedules",
|
||||
title="Create a schedule",
|
||||
summary="Attach a cron, one-time, interval, or delay schedule to an instance.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
field(
|
||||
"action",
|
||||
"form",
|
||||
"string",
|
||||
True,
|
||||
"start",
|
||||
"Lifecycle action to run on schedule (start, stop, restart).",
|
||||
),
|
||||
field(
|
||||
"kind",
|
||||
"form",
|
||||
"string",
|
||||
True,
|
||||
"cron",
|
||||
"Schedule kind: cron, once, interval, or delay.",
|
||||
),
|
||||
field(
|
||||
"cron",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"0 * * * *",
|
||||
"Cron expression (when kind is cron).",
|
||||
),
|
||||
field(
|
||||
"run_at",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"2026-01-01T00:00:00",
|
||||
"ISO timestamp for a one-time run (when kind is once).",
|
||||
),
|
||||
field(
|
||||
"delay_seconds",
|
||||
"form",
|
||||
"integer",
|
||||
False,
|
||||
"60",
|
||||
"Seconds to wait before a single run (when kind is delay).",
|
||||
),
|
||||
field(
|
||||
"every_seconds",
|
||||
"form",
|
||||
"integer",
|
||||
False,
|
||||
"300",
|
||||
"Interval in seconds between runs (when kind is interval).",
|
||||
),
|
||||
field(
|
||||
"max_runs",
|
||||
"form",
|
||||
"integer",
|
||||
False,
|
||||
"10",
|
||||
"Optional cap on the number of runs.",
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-instance-schedule-delete",
|
||||
method="POST",
|
||||
path="/projects/{project_slug}/containers/instances/{uid}/schedules/{sid}/delete",
|
||||
title="Delete a schedule",
|
||||
summary="Remove a schedule from an instance.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field(
|
||||
"project_slug",
|
||||
"path",
|
||||
"string",
|
||||
True,
|
||||
"PROJECT_SLUG",
|
||||
"Project slug or uid.",
|
||||
),
|
||||
field(
|
||||
"uid", "path", "string", True, "INSTANCE_UID", "Instance uid."
|
||||
),
|
||||
field(
|
||||
"sid", "path", "string", True, "SCHEDULE_UID", "Schedule uid."
|
||||
),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-create",
|
||||
method="POST",
|
||||
path="/admin/containers/create",
|
||||
title="Admin create instance",
|
||||
summary="Create an instance from the admin Containers page: project search-select, run-as user, boot language/script, restart policy, start-on-boot, plus the usual options.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("project_slug", "form", "string", True, "PROJECT_SLUG", "Project that becomes the /app root."),
|
||||
field("name", "form", "string", True, "staging", "Instance name."),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user (always pravda, uid 1000)."),
|
||||
field("boot_language", "form", "enum", False, "none", "Boot source language.", ["none", "python", "bash"]),
|
||||
field("boot_script", "form", "textarea", False, "print('hi')", "Boot source code run on launch (takes precedence over boot_command)."),
|
||||
field("boot_command", "form", "string", False, "python app.py", "Fallback boot command when no boot_script is set."),
|
||||
field("restart_policy", "form", "enum", False, "never", "Restart policy.", ["never", "always", "on-failure", "unless-stopped"]),
|
||||
field("start_on_boot", "form", "boolean", False, "false", "Force running whenever the container service starts."),
|
||||
field("env", "form", "textarea", False, "KEY=VALUE", "Env vars, one KEY=VALUE per line."),
|
||||
field("ports", "form", "string", False, "80", "Port maps; bare container port auto-assigns a host port above 20000."),
|
||||
field("cpu_limit", "form", "string", False, "1.5", "CPU limit."),
|
||||
field("mem_limit", "form", "string", False, "512m", "Memory limit."),
|
||||
field("ingress_slug", "form", "string", False, "my-service", "Publish at /p/<slug> (optional)."),
|
||||
field("ingress_port", "form", "integer", False, "8899", "Container port to publish."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-edit",
|
||||
method="POST",
|
||||
path="/admin/containers/{uid}/edit",
|
||||
title="Admin edit instance",
|
||||
summary="Update an instance's run-as user, boot language/script/command, restart policy, start-on-boot flag, and resource limits.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("uid", "path", "string", True, "INSTANCE_UID", "Instance uid."),
|
||||
field("run_as_uid", "form", "string", False, "USER_UID", "Run-as user uid (identity + API key only)."),
|
||||
field("boot_language", "form", "enum", False, "none", "Boot source language.", ["none", "python", "bash"]),
|
||||
field("boot_script", "form", "textarea", False, "print('hi')", "Boot source code."),
|
||||
field("boot_command", "form", "string", False, "python app.py", "Fallback boot command."),
|
||||
field("restart_policy", "form", "enum", False, "never", "Restart policy.", ["never", "always", "on-failure", "unless-stopped"]),
|
||||
field("start_on_boot", "form", "boolean", False, "false", "Force running on container-service boot."),
|
||||
field("cpu_limit", "form", "string", False, "1.5", "CPU limit."),
|
||||
field("mem_limit", "form", "string", False, "512m", "Memory limit."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-action",
|
||||
method="POST",
|
||||
path="/admin/containers/{uid}/{action}",
|
||||
title="Admin instance lifecycle",
|
||||
summary="start, stop, restart, pause, or resume an instance from the admin Containers page (flips desired state).",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("uid", "path", "string", True, "INSTANCE_UID", "Instance uid."),
|
||||
field("action", "path", "enum", True, "start", "Lifecycle action.", ["start", "stop", "restart", "pause", "resume"]),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-sync",
|
||||
method="POST",
|
||||
path="/admin/containers/{uid}/sync",
|
||||
title="Admin bidirectional sync",
|
||||
summary="Run a one-shot bidirectional newer-wins sync between the project files and the container workspace.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("uid", "path", "string", True, "INSTANCE_UID", "Instance uid."),
|
||||
],
|
||||
sample_response={"exported": 3, "imported": 1},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-delete",
|
||||
method="POST",
|
||||
path="/admin/containers/{uid}/delete",
|
||||
title="Admin delete instance",
|
||||
summary="Soft-delete an instance and mark its container for removal.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("uid", "path", "string", True, "INSTANCE_UID", "Instance uid."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-project-search",
|
||||
method="GET",
|
||||
path="/admin/containers/projects/search",
|
||||
title="Admin project search",
|
||||
summary="Search projects by title for the admin create form (returns uid, slug, title).",
|
||||
auth="admin",
|
||||
params=[
|
||||
field("q", "query", "string", False, "api", "Title fragment."),
|
||||
],
|
||||
sample_response={"results": [{"uid": "PROJECT_UID", "slug": "PROJECT_SLUG", "title": "My Project"}]},
|
||||
),
|
||||
endpoint(
|
||||
id="containers-admin-user-search",
|
||||
method="GET",
|
||||
path="/admin/containers/users/search",
|
||||
title="Admin run-as user search",
|
||||
summary="Search users by username for the run-as-user select (returns uid, username).",
|
||||
auth="admin",
|
||||
params=[
|
||||
field("q", "query", "string", False, "alice", "Username fragment."),
|
||||
],
|
||||
sample_response={"results": [{"uid": "USER_UID", "username": "alice"}]},
|
||||
),
|
||||
],
|
||||
}
|
||||
@ -344,7 +344,7 @@ four ways to sign requests.
|
||||
method="POST",
|
||||
path="/profile/{username}/notifications",
|
||||
title="Toggle a notification preference",
|
||||
summary="Enable or disable one notification type on one channel (in-app or push). Admins may target any user. Types: comment, reply, mention, vote, follow, message, badge, level, issue.",
|
||||
summary="Enable or disable one notification type on one channel (in-app or push). Admins may target any user. Types: comment, reply, mention, vote, follow, message, badge, level, issue, reminder, harvest_stolen.",
|
||||
auth="user",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
@ -363,7 +363,7 @@ four ways to sign requests.
|
||||
"string",
|
||||
True,
|
||||
"vote",
|
||||
"One of: comment, reply, mention, vote, follow, message, badge, level, issue.",
|
||||
"One of: comment, reply, mention, vote, follow, message, badge, level, issue, reminder, harvest_stolen.",
|
||||
),
|
||||
field(
|
||||
"channel",
|
||||
|
||||
@ -4,7 +4,7 @@ from .._shared import endpoint, field
|
||||
|
||||
GROUP = {
|
||||
"slug": "tools",
|
||||
"title": "Tools (SEO & DeepSearch)",
|
||||
"title": "Tools (SEO, DeepSearch & AI Usage Analyzer)",
|
||||
"intro": """
|
||||
# Tools: SEO Diagnostics & DeepSearch
|
||||
|
||||
@ -17,6 +17,10 @@ technical, on-page, structured-data, Core Web Vitals, accessibility and AI-readi
|
||||
synthesises a cited report with confidence scoring and gap analysis, plus a grounded chat over
|
||||
the results.
|
||||
|
||||
**AI Usage Analyzer** classifies a git repository or website as AI slop, sophisticated AI-assisted
|
||||
work or genuine human work, and publishes a persistent report with an embeddable authenticity
|
||||
badge.
|
||||
|
||||
Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html). These are
|
||||
**capability URLs**: the job `uid` is an unguessable identifier, so anyone holding it can read the
|
||||
status and report.
|
||||
@ -229,5 +233,190 @@ status and report.
|
||||
"export_pdf_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.pdf",
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-run",
|
||||
method="POST",
|
||||
path="/tools/isslop/run",
|
||||
title="Queue a AI usage analysis",
|
||||
summary="Start a background authenticity analysis of a git repository or website. Returns the job uid plus status, events and report URLs.",
|
||||
auth="public",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("url", "form", "string", True, "https://github.com/owner/repository", "Repository (http/git/ssh) or website URL to classify."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "ISSLOP_UID",
|
||||
"status_url": "/tools/isslop/ISSLOP_UID",
|
||||
"events_url": "/tools/isslop/ISSLOP_UID/events",
|
||||
"report_url": "/tools/isslop/ISSLOP_UID/report",
|
||||
"topic": "public.isslop.ISSLOP_UID",
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-list",
|
||||
method="GET",
|
||||
path="/tools/isslop/list",
|
||||
title="My AI usage analyses",
|
||||
summary="List the caller's analyses, newest first. Member history is account-bound; guest history is session-bound and claimed by the account on first signed-in call.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("limit", "query", "integer", False, "50", "Maximum analyses to return (1-200)."),
|
||||
],
|
||||
sample_response={
|
||||
"analyses": [
|
||||
{
|
||||
"uid": "ISSLOP_UID",
|
||||
"status": "completed",
|
||||
"source_url": "https://github.com/owner/repository",
|
||||
"source_kind": "git",
|
||||
"grade": "B",
|
||||
"human_percent": 71.4,
|
||||
"ai_percent": 28.6,
|
||||
"category": "human-clean",
|
||||
"report_url": "/tools/isslop/ISSLOP_UID/report",
|
||||
"badge_url": "/tools/isslop/ISSLOP_UID/badge.svg",
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-status",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}",
|
||||
title="AI usage analysis status",
|
||||
summary="Poll an analysis. Once completed, grade, category and the human/AI split are populated.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid returned when the run was queued."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "ISSLOP_UID",
|
||||
"status": "completed",
|
||||
"source_url": "https://github.com/owner/repository",
|
||||
"source_kind": "git",
|
||||
"grade": "B",
|
||||
"slop_score": 31.2,
|
||||
"origin_score": 28.0,
|
||||
"quality_deficit_score": 22.5,
|
||||
"human_percent": 71.4,
|
||||
"ai_percent": 28.6,
|
||||
"category": "human-clean",
|
||||
"confidence": "medium",
|
||||
"files_total": 120,
|
||||
"files_analyzed": 96,
|
||||
"report_url": "/tools/isslop/ISSLOP_UID/report",
|
||||
"badge_url": "/tools/isslop/ISSLOP_UID/badge.svg",
|
||||
"events_url": "/tools/isslop/ISSLOP_UID/events",
|
||||
"topic": "public.isslop.ISSLOP_UID",
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-events",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/events",
|
||||
title="AI usage analysis event trail",
|
||||
summary="The persisted, ordered event trail of an analysis. Use ?after=SEQ to poll incrementally; live frames also stream on the pub/sub topic.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid."),
|
||||
field("after", "query", "integer", False, "0", "Return only events with a sequence number greater than this."),
|
||||
field("limit", "query", "integer", False, "2000", "Maximum events to return (1-5000)."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "ISSLOP_UID",
|
||||
"status": "running",
|
||||
"events": [
|
||||
{"seq": 1, "kind": "stage", "message": "Resolving source type", "data": {"stage": "resolve"}, "created_at": "2026-06-14T10:00:00+00:00"}
|
||||
],
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-report",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/report",
|
||||
title="AI usage analysis report",
|
||||
summary="Full report: verdict, markdown body, per-file results, image review and badge embeds. Negotiates HTML or JSON.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid of a finished run."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "ISSLOP_UID",
|
||||
"status": "completed",
|
||||
"source_url": "https://github.com/owner/repository",
|
||||
"grade": "B",
|
||||
"human_percent": 71.4,
|
||||
"ai_percent": 28.6,
|
||||
"category": "human-clean",
|
||||
"markdown": "# Verdict...",
|
||||
"generator_model": "molodetz",
|
||||
"badge": {
|
||||
"badge_url": "https://devplace.example/tools/isslop/ISSLOP_UID/badge.svg",
|
||||
"report_url": "https://devplace.example/tools/isslop/ISSLOP_UID/report",
|
||||
"markdown": "[](...)",
|
||||
"html": "<a href=...><img src=.../></a>",
|
||||
},
|
||||
"files": [{"path": "src/main.py", "language": "python", "lines": 120, "origin_score": 35.0, "quality_deficit_score": 18.0, "category": "human-clean", "signals": []}],
|
||||
"images": [{"path": "assets/hero.png", "ai_probability": 84.0, "grade": "F", "verdict": "ai-generated", "image_kind": "illustration", "tells": ["waxy skin"], "description": "...", "thumb_url": "/tools/isslop/ISSLOP_UID/media/0f3a9c2d1b4e5a67.webp"}],
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-report-md",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/report.md",
|
||||
title="Download report markdown",
|
||||
summary="Download the full report as a markdown file.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid of a finished run."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-source",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/source",
|
||||
title="Annotated source of a flagged file",
|
||||
summary="The persisted source of a signal-bearing file with its signals, rendered with line numbers and highlighted findings (HTML) or as JSON. Linked from the report's file table, signal chips and prose.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid."),
|
||||
field("path", "query", "string", True, "src/libs/Env.ts", "Workspace-relative file path from the report."),
|
||||
field("line", "query", "integer", False, "12", "Line to focus and highlight."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "ISSLOP_UID",
|
||||
"path": "src/libs/Env.ts",
|
||||
"language": "typescript",
|
||||
"category": "human-clean",
|
||||
"origin_score": 24.0,
|
||||
"quality_deficit_score": 34.9,
|
||||
"source": "import { createEnv } from '@t3-oss/env-nextjs';...",
|
||||
"truncated": False,
|
||||
"signals": [{"code": "PUBLIC_ENV_SECRET", "title": "Secret exposed via public env variable", "severity": "strong", "line": 12}],
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-media",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/media/{name}",
|
||||
title="Reviewed image thumbnail",
|
||||
summary="Aspect-preserving WebP thumbnail of a reviewed image, persisted as evidence. The name comes from the report's images[].thumb_url.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid."),
|
||||
field("name", "path", "string", True, "0f3a9c2d1b4e5a67.webp", "Thumbnail file name from the report."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="tools-isslop-badge",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/badge.svg",
|
||||
title="Authenticity badge",
|
||||
summary="Embeddable SVG badge showing the human score and authenticity grade, linking to the report.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "ISSLOP_UID", "Analysis uid."),
|
||||
],
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
@ -16,10 +16,37 @@ _RENDER_BLOCK = re.compile(
|
||||
r'<div class="docs-content" data-render>(.*?)</div>', re.DOTALL
|
||||
)
|
||||
|
||||
_HEADING = re.compile(r"<h([23])>(.*?)</h\1>", re.DOTALL)
|
||||
|
||||
|
||||
def heading_slug(text: str) -> str:
|
||||
plain = html.unescape(re.sub(r"<[^>]+>", "", text)).strip().lower()
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", plain).strip("-")
|
||||
return slug or "section"
|
||||
|
||||
|
||||
def _anchor_headings(rendered: str) -> str:
|
||||
seen: dict[str, int] = {}
|
||||
|
||||
def _inject(match: re.Match) -> str:
|
||||
level, inner = match.group(1), match.group(2)
|
||||
slug = heading_slug(inner)
|
||||
count = seen.get(slug, 0)
|
||||
seen[slug] = count + 1
|
||||
if count:
|
||||
slug = f"{slug}-{count}"
|
||||
return (
|
||||
f'<h{level} id="{slug}">{inner}'
|
||||
f'<a class="docs-heading-anchor" href="#{slug}" aria-label="Link to this section">#</a>'
|
||||
f"</h{level}>"
|
||||
)
|
||||
|
||||
return _HEADING.sub(_inject, rendered)
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def _render_markdown(source: str) -> str:
|
||||
return _markdown(source)
|
||||
return _anchor_headings(_markdown(source))
|
||||
|
||||
|
||||
def _convert(match: re.Match) -> str:
|
||||
|
||||
@ -104,6 +104,7 @@ from devplacepy.services.presence_relay import PresenceRelayService
|
||||
from devplacepy.services import presence
|
||||
from devplacepy.services.correction import PENDING_SCOPE_KEY
|
||||
from devplacepy.services.jobs.deepsearch.service import DeepsearchService
|
||||
from devplacepy.services.jobs.isslop.service import IsslopService
|
||||
from devplacepy.services.gitea.service import IssueTrackerService
|
||||
from devplacepy.services.containers.service import ContainerService
|
||||
from devplacepy.services.xmlrpc import XmlrpcService
|
||||
@ -214,6 +215,7 @@ class UploadStaticFiles(StaticFiles):
|
||||
else "attachment"
|
||||
)
|
||||
response.headers["Content-Disposition"] = disposition
|
||||
response.headers["Cache-Control"] = "public, max-age=604800"
|
||||
return response
|
||||
|
||||
|
||||
@ -227,6 +229,16 @@ class CachedStaticFiles(StaticFiles):
|
||||
return response
|
||||
|
||||
|
||||
class FallbackStaticFiles(StaticFiles):
|
||||
async def get_response(self, path, scope):
|
||||
response = await super().get_response(path, scope)
|
||||
if Path(path).name == "service-worker.js":
|
||||
response.headers["Cache-Control"] = "no-cache"
|
||||
else:
|
||||
response.headers["Cache-Control"] = "public, max-age=3600"
|
||||
return response
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
ensure_data_dirs()
|
||||
@ -250,6 +262,7 @@ async def lifespan(app: FastAPI):
|
||||
service_manager.register(LiveViewRelayService())
|
||||
service_manager.register(PresenceRelayService())
|
||||
service_manager.register(DeepsearchService())
|
||||
service_manager.register(IsslopService())
|
||||
service_manager.register(IssueCreateService())
|
||||
service_manager.register(PlanningReportService())
|
||||
service_manager.register(IssueTrackerService())
|
||||
@ -292,7 +305,7 @@ app.mount(
|
||||
CachedStaticFiles(directory=str(STATIC_DIR)),
|
||||
name="static_versioned",
|
||||
)
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
app.mount("/static", FallbackStaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
|
||||
|
||||
@app.exception_handler(404)
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import re
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
@ -460,6 +462,26 @@ class SeoRunForm(BaseModel):
|
||||
return text
|
||||
|
||||
|
||||
ISSLOP_URL_PATTERN = re.compile(r"^(https?://|git://|ssh://|git@)[\w./:@~^-]+$", re.IGNORECASE)
|
||||
ISSLOP_SINGLE_SLASH_PATTERN = re.compile(r"^(https?|git|ssh):/(?!/)", re.IGNORECASE)
|
||||
ISSLOP_SCHEME_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://", re.IGNORECASE)
|
||||
|
||||
|
||||
class IsslopRunForm(BaseModel):
|
||||
url: str = Field(min_length=4, max_length=2048)
|
||||
|
||||
@field_validator("url")
|
||||
@classmethod
|
||||
def url_scheme(cls, value):
|
||||
text = value.strip()
|
||||
text = ISSLOP_SINGLE_SLASH_PATTERN.sub(lambda match: f"{match.group(1)}://", text)
|
||||
if not ISSLOP_SCHEME_PATTERN.match(text) and not text.startswith("git@"):
|
||||
text = f"https://{text}"
|
||||
if not ISSLOP_URL_PATTERN.match(text):
|
||||
raise ValueError("URL must be an http(s), git or ssh source location")
|
||||
return text
|
||||
|
||||
|
||||
DEEPSEARCH_MIN_DEPTH = 1
|
||||
DEEPSEARCH_MAX_DEPTH = 4
|
||||
DEEPSEARCH_DEFAULT_DEPTH = 2
|
||||
|
||||
@ -148,6 +148,18 @@ DOCS_PAGES = [
|
||||
"kind": "prose",
|
||||
"section": SECTION_TOOLS,
|
||||
},
|
||||
{
|
||||
"slug": "tools-isslop",
|
||||
"title": "AI Usage Analyzer",
|
||||
"kind": "prose",
|
||||
"section": SECTION_TOOLS,
|
||||
},
|
||||
{
|
||||
"slug": "isslop-checks",
|
||||
"title": "AI Usage Analyzer checks",
|
||||
"kind": "prose",
|
||||
"section": SECTION_TOOLS,
|
||||
},
|
||||
# Claude Code - the native subagent, command, and workflow setup under .claude/
|
||||
{
|
||||
"slug": "claude",
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
@ -93,18 +94,28 @@ async def issue_detail(request: Request, number: int):
|
||||
if not gitea_config().is_configured:
|
||||
raise not_found("Issue not found")
|
||||
client = runtime.get_client()
|
||||
try:
|
||||
issue = await client.get_issue(number)
|
||||
except GiteaError as exc:
|
||||
if exc.status == 404:
|
||||
issue_result, comments_result = await asyncio.gather(
|
||||
client.get_issue(number),
|
||||
client.list_comments(number),
|
||||
return_exceptions=True,
|
||||
)
|
||||
if isinstance(issue_result, GiteaError):
|
||||
if issue_result.status == 404:
|
||||
raise not_found("Issue not found")
|
||||
logger.warning("Could not load issue #%s: %s", number, exc)
|
||||
logger.warning("Could not load issue #%s: %s", number, issue_result)
|
||||
return tracker_unavailable(request)
|
||||
try:
|
||||
comments = await client.list_comments(number)
|
||||
except GiteaError as exc:
|
||||
logger.warning("Could not load comments for issue #%s: %s", number, exc)
|
||||
if isinstance(issue_result, BaseException):
|
||||
raise issue_result
|
||||
issue = issue_result
|
||||
if isinstance(comments_result, BaseException):
|
||||
if not isinstance(comments_result, GiteaError):
|
||||
raise comments_result
|
||||
logger.warning(
|
||||
"Could not load comments for issue #%s: %s", number, comments_result
|
||||
)
|
||||
comments = []
|
||||
else:
|
||||
comments = comments_result
|
||||
|
||||
if user:
|
||||
mark_notifications_read_by_target(user["uid"], f"/issues?highlight={number}")
|
||||
|
||||
@ -42,6 +42,8 @@ router = APIRouter()
|
||||
|
||||
MAX_WS_ATTACHMENTS = 5
|
||||
|
||||
CONVERSATION_MESSAGE_LIMIT = 500
|
||||
|
||||
def mark_conversation_read(user_uid: str, other_uid: str) -> None:
|
||||
if "messages" not in db.tables:
|
||||
return
|
||||
@ -54,64 +56,65 @@ def mark_conversation_read(user_uid: str, other_uid: str) -> None:
|
||||
clear_messages_cache(user_uid)
|
||||
|
||||
def get_conversations(user_uid: str):
|
||||
messages_table = get_table("messages")
|
||||
raw = list(messages_table.find(sender_uid=user_uid)) + list(
|
||||
messages_table.find(receiver_uid=user_uid)
|
||||
)
|
||||
seen = set()
|
||||
all_messages = []
|
||||
for m in raw:
|
||||
if m["uid"] not in seen:
|
||||
seen.add(m["uid"])
|
||||
all_messages.append(m)
|
||||
|
||||
blocked = get_blocked_uids(user_uid)
|
||||
conversation_map = {}
|
||||
other_uids = set()
|
||||
for msg in all_messages:
|
||||
other_uid = (
|
||||
msg["receiver_uid"] if msg["sender_uid"] == user_uid else msg["sender_uid"]
|
||||
if "messages" not in db.tables:
|
||||
return []
|
||||
latest = list(
|
||||
db.query(
|
||||
"SELECT * FROM ("
|
||||
" SELECT *,"
|
||||
" CASE WHEN sender_uid = :me THEN receiver_uid ELSE sender_uid END AS other_uid,"
|
||||
" ROW_NUMBER() OVER ("
|
||||
" PARTITION BY CASE WHEN sender_uid = :me THEN receiver_uid ELSE sender_uid END"
|
||||
" ORDER BY created_at DESC, id DESC"
|
||||
" ) AS rn"
|
||||
" FROM messages"
|
||||
" WHERE sender_uid = :me OR receiver_uid = :me"
|
||||
") WHERE rn = 1 ORDER BY created_at DESC",
|
||||
me=user_uid,
|
||||
)
|
||||
)
|
||||
blocked = get_blocked_uids(user_uid)
|
||||
conversations = []
|
||||
other_uids = []
|
||||
for msg in latest:
|
||||
other_uid = msg["other_uid"]
|
||||
if other_uid in blocked:
|
||||
continue
|
||||
other_uids.add(other_uid)
|
||||
if (
|
||||
other_uid not in conversation_map
|
||||
or msg["created_at"] > conversation_map[other_uid]["last_message_at"]
|
||||
):
|
||||
conversation_map[other_uid] = {
|
||||
other_uids.append(other_uid)
|
||||
conversations.append(
|
||||
{
|
||||
"other_uid": other_uid,
|
||||
"other_user": None,
|
||||
"last_message": msg["content"],
|
||||
"last_message_at": msg["created_at"],
|
||||
"unread": msg["receiver_uid"] == user_uid and not msg["read"],
|
||||
}
|
||||
|
||||
)
|
||||
if other_uids:
|
||||
users_map = get_users_by_uids(list(other_uids))
|
||||
for uid, conv in conversation_map.items():
|
||||
conv["other_user"] = users_map.get(uid)
|
||||
|
||||
conversations = sorted(
|
||||
conversation_map.values(),
|
||||
key=lambda c: c["last_message_at"],
|
||||
reverse=True,
|
||||
)
|
||||
users_map = get_users_by_uids(other_uids)
|
||||
for conv in conversations:
|
||||
conv["other_user"] = users_map.get(conv["other_uid"])
|
||||
for conv in conversations:
|
||||
conv.pop("other_uid", None)
|
||||
return conversations
|
||||
|
||||
def get_conversation_messages(user_uid: str, other_uid: str):
|
||||
if other_uid in get_blocked_uids(user_uid):
|
||||
return [], None
|
||||
messages_table = get_table("messages")
|
||||
raw = list(messages_table.find(sender_uid=user_uid, receiver_uid=other_uid)) + list(
|
||||
messages_table.find(sender_uid=other_uid, receiver_uid=user_uid)
|
||||
if "messages" not in db.tables:
|
||||
return [], get_users_by_uids([other_uid]).get(other_uid)
|
||||
msgs = list(
|
||||
db.query(
|
||||
"SELECT * FROM messages"
|
||||
" WHERE (sender_uid = :me AND receiver_uid = :other)"
|
||||
" OR (sender_uid = :other AND receiver_uid = :me)"
|
||||
" ORDER BY created_at DESC, id DESC LIMIT :lim",
|
||||
me=user_uid,
|
||||
other=other_uid,
|
||||
lim=CONVERSATION_MESSAGE_LIMIT,
|
||||
)
|
||||
)
|
||||
seen = set()
|
||||
msgs = []
|
||||
for m in raw:
|
||||
if m["uid"] not in seen:
|
||||
seen.add(m["uid"])
|
||||
msgs.append(m)
|
||||
msgs.sort(key=lambda m: m["created_at"])
|
||||
msgs.reverse()
|
||||
|
||||
user_ids = list({m["sender_uid"] for m in msgs} | {other_uid})
|
||||
users_map = get_users_by_uids(user_ids)
|
||||
|
||||
@ -40,7 +40,7 @@ from devplacepy.utils import (
|
||||
track_action,
|
||||
build_achievements,
|
||||
)
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.responses import respond, action_result, wants_json
|
||||
from devplacepy.schemas import ProfileOut
|
||||
from devplacepy.avatar import avatar_url, avatar_seed
|
||||
from devplacepy.seo import (
|
||||
@ -181,16 +181,20 @@ async def profile_page(
|
||||
achievements = build_achievements({b["badge_name"] for b in badges})
|
||||
badge_total = sum(group["total"] for group in achievements)
|
||||
badge_earned = sum(group["earned"] for group in achievements)
|
||||
projects = list(
|
||||
get_table("projects").find(user_uid=profile_user["uid"], deleted_at=None)
|
||||
)
|
||||
projects = [p for p in projects if can_view_project(p, current_user)]
|
||||
gists_raw = list(
|
||||
get_table("gists").find(user_uid=profile_user["uid"], deleted_at=None)
|
||||
)
|
||||
include_collections = wants_json(request)
|
||||
projects = []
|
||||
if tab == "projects" or include_collections:
|
||||
projects = list(
|
||||
get_table("projects").find(user_uid=profile_user["uid"], deleted_at=None)
|
||||
)
|
||||
projects = [p for p in projects if can_view_project(p, current_user)]
|
||||
gists = []
|
||||
for g in gists_raw:
|
||||
gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
|
||||
if tab == "gists" or include_collections:
|
||||
gists_raw = list(
|
||||
get_table("gists").find(user_uid=profile_user["uid"], deleted_at=None)
|
||||
)
|
||||
for g in gists_raw:
|
||||
gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
|
||||
posts_count = get_table("posts").count(
|
||||
user_uid=profile_user["uid"], deleted_at=None
|
||||
)
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from . import deepsearch, index, seo
|
||||
from . import deepsearch, index, isslop, seo
|
||||
|
||||
router = index.router
|
||||
router.include_router(seo.router, prefix="/seo")
|
||||
router.include_router(deepsearch.router, prefix="/deepsearch")
|
||||
router.include_router(isslop.router, prefix="/isslop")
|
||||
|
||||
@ -26,6 +26,17 @@ TOOLS = [
|
||||
"structured-data, performance, accessibility and AI-readiness checks."
|
||||
),
|
||||
},
|
||||
{
|
||||
"slug": "isslop",
|
||||
"name": "AI Usage Analyzer",
|
||||
"icon": "🧪",
|
||||
"url": "/tools/isslop",
|
||||
"description": (
|
||||
"Measure how a codebase or website was made: untouched AI defaults, AI steered by a "
|
||||
"knowing hand, or work no model would ever produce. Multi-signal analysis, image "
|
||||
"forensics and a shareable authenticity badge."
|
||||
),
|
||||
},
|
||||
{
|
||||
"slug": "deepsearch",
|
||||
"name": "DeepSearch",
|
||||
|
||||
506
devplacepy/routers/tools/isslop.py
Normal file
506
devplacepy/routers/tools/isslop.py
Normal file
@ -0,0 +1,506 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import quote
|
||||
from typing import Annotated
|
||||
|
||||
import uuid_utils
|
||||
from fastapi import Depends, APIRouter, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Response
|
||||
|
||||
from devplacepy.config import ISSLOP_MEDIA_DIR
|
||||
from devplacepy.constants import DEVII_GUEST_COOKIE
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.models import IsslopRunForm
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import IsslopAnalysisOut, IsslopListOut, IsslopReportOut, IsslopSourceOut
|
||||
from devplacepy.seo import base_seo_context, site_url, web_application_schema, website_schema
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.jobs.isslop import store
|
||||
from devplacepy.services.jobs.isslop.badge import badge_html, badge_markdown, render_badge
|
||||
from devplacepy.services.jobs.isslop.service import topic_for
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, not_found, track_action
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
ACTIVE_STATES = (queue.PENDING, queue.RUNNING)
|
||||
GUEST_COOKIE_MAX_AGE = 31536000
|
||||
EVENT_LIMIT_MAX = 5000
|
||||
MEDIA_NAME_PATTERN = re.compile(r"^[a-f0-9]{16}\.webp$")
|
||||
SOURCE_NAME_PATTERN = re.compile(r"^s[a-f0-9]{16}\.txt$")
|
||||
SOURCE_LINE_CONTEXT = 400000
|
||||
|
||||
|
||||
def _owner(request: Request) -> tuple[str, str] | None:
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return "user", user["uid"]
|
||||
guest = request.cookies.get(DEVII_GUEST_COOKIE)
|
||||
if guest:
|
||||
return "guest", guest
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_owner(request: Request) -> tuple[str, str, str]:
|
||||
owner = _owner(request)
|
||||
if owner:
|
||||
return owner[0], owner[1], ""
|
||||
minted = uuid_utils.uuid7().hex
|
||||
return "guest", minted, minted
|
||||
|
||||
|
||||
def _set_guest_cookie(response, minted: str) -> None:
|
||||
if minted:
|
||||
response.set_cookie(
|
||||
DEVII_GUEST_COOKIE,
|
||||
minted,
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
max_age=GUEST_COOKIE_MAX_AGE,
|
||||
)
|
||||
|
||||
|
||||
def _sync_guest_history(request: Request) -> None:
|
||||
user = get_current_user(request)
|
||||
guest = request.cookies.get(DEVII_GUEST_COOKIE)
|
||||
if user and guest:
|
||||
store.claim_guest_analyses(guest, user["uid"])
|
||||
|
||||
|
||||
def _analysis_payload(row: dict) -> dict:
|
||||
uid = row.get("uid", "")
|
||||
return {
|
||||
"uid": uid,
|
||||
"status": row.get("status", ""),
|
||||
"source_url": row.get("source_url", ""),
|
||||
"source_kind": row.get("source_kind", "") or "unknown",
|
||||
"grade": row.get("grade"),
|
||||
"slop_score": row.get("slop_score"),
|
||||
"origin_score": row.get("origin_score"),
|
||||
"quality_deficit_score": row.get("quality_deficit_score"),
|
||||
"human_percent": row.get("human_percent"),
|
||||
"ai_percent": row.get("ai_percent"),
|
||||
"category": row.get("category"),
|
||||
"confidence": row.get("confidence"),
|
||||
"files_total": int(row.get("files_total") or 0),
|
||||
"files_analyzed": int(row.get("files_analyzed") or 0),
|
||||
"detected_builder": row.get("detected_builder") or None,
|
||||
"dom_slop_score": row.get("dom_slop_score"),
|
||||
"error": row.get("error_message_text") or None,
|
||||
"report_url": f"/tools/isslop/{uid}/report",
|
||||
"badge_url": f"/tools/isslop/{uid}/badge.svg",
|
||||
"events_url": f"/tools/isslop/{uid}/events",
|
||||
"topic": topic_for(uid),
|
||||
"created_at": row.get("created_at"),
|
||||
"finished_at": row.get("finished_at"),
|
||||
}
|
||||
|
||||
|
||||
SEVERITY_RANK = {"strong": 0, "medium": 1, "weak": 2}
|
||||
SIGNAL_LINES_CAP = 8
|
||||
|
||||
|
||||
def _signal_groups(signals: list) -> list:
|
||||
groups: dict[str, dict] = {}
|
||||
for signal in signals:
|
||||
if not isinstance(signal, dict):
|
||||
continue
|
||||
code = str(signal.get("code", ""))
|
||||
entry = groups.setdefault(
|
||||
code,
|
||||
{
|
||||
"code": code,
|
||||
"title": str(signal.get("title", "")),
|
||||
"severity": str(signal.get("severity", "weak")),
|
||||
"count": 0,
|
||||
"lines": [],
|
||||
},
|
||||
)
|
||||
entry["count"] += 1
|
||||
line = signal.get("line")
|
||||
if isinstance(line, int) and len(entry["lines"]) < SIGNAL_LINES_CAP:
|
||||
entry["lines"].append(line)
|
||||
return sorted(
|
||||
groups.values(),
|
||||
key=lambda group: (SEVERITY_RANK.get(group["severity"], 3), -group["count"], group["code"]),
|
||||
)
|
||||
|
||||
|
||||
def _source_url(uid: str, path: str, line: int = 0) -> str:
|
||||
url = f"/tools/isslop/{uid}/source?path={quote(path, safe='')}"
|
||||
if line > 0:
|
||||
url += f"&line={line}#L{line}"
|
||||
return url
|
||||
|
||||
|
||||
CRITERIA_URL = "/docs/isslop-checks.html"
|
||||
|
||||
|
||||
def _linkify_sources(markdown: str, uid: str, paths: set[str], signal_codes: set[str]) -> str:
|
||||
for path in sorted(paths, key=len, reverse=True):
|
||||
escaped = re.escape(path)
|
||||
markdown = re.sub(
|
||||
rf"`{escaped}:(\d+)`",
|
||||
lambda match, p=path: f"[`{p}:{match.group(1)}`]({_source_url(uid, p, int(match.group(1)))})",
|
||||
markdown,
|
||||
)
|
||||
markdown = markdown.replace(f"`{path}`", f"[`{path}`]({_source_url(uid, path)})")
|
||||
markdown = re.sub(
|
||||
rf"(?<![\w/`\(\[]){escaped}(?::(\d+))?(?![\w/`])",
|
||||
lambda match, p=path: (
|
||||
f"[{p}:{match.group(1)}]({_source_url(uid, p, int(match.group(1)))})"
|
||||
if match.group(1)
|
||||
else f"[{p}]({_source_url(uid, p)})"
|
||||
),
|
||||
markdown,
|
||||
)
|
||||
for code in sorted(signal_codes, key=len, reverse=True):
|
||||
escaped = re.escape(code)
|
||||
markdown = markdown.replace(f"`{code}`", f"[`{code}`]({CRITERIA_URL})")
|
||||
markdown = re.sub(
|
||||
rf"(?<![\w`\[]){escaped}(?![\w`])",
|
||||
f"[{code}]({CRITERIA_URL})",
|
||||
markdown,
|
||||
)
|
||||
return markdown
|
||||
|
||||
|
||||
def _badge_info(request: Request, uid: str) -> dict:
|
||||
base = site_url(request).rstrip("/")
|
||||
badge_url = f"{base}/tools/isslop/{uid}/badge.svg"
|
||||
report_url = f"{base}/tools/isslop/{uid}/report"
|
||||
return {
|
||||
"badge_url": badge_url,
|
||||
"report_url": report_url,
|
||||
"markdown": badge_markdown(badge_url, report_url),
|
||||
"html": badge_html(badge_url, report_url),
|
||||
}
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def isslop_page(request: Request):
|
||||
user = get_current_user(request)
|
||||
_sync_guest_history(request)
|
||||
base = site_url(request)
|
||||
description = (
|
||||
"Measure how a codebase or website was made: untouched AI defaults, AI steered by a "
|
||||
"knowing hand, or work no model would ever produce. Transparent, reproducible analysis "
|
||||
"with a shareable authenticity badge."
|
||||
)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="AI Usage Analyzer",
|
||||
description=description,
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Tools", "url": "/tools"},
|
||||
{"name": "AI Usage Analyzer", "url": "/tools/isslop"},
|
||||
],
|
||||
schemas=[
|
||||
website_schema(base),
|
||||
web_application_schema("AI Usage Analyzer", description, "/tools/isslop", base),
|
||||
],
|
||||
)
|
||||
minted = "" if _owner(request) else uuid_utils.uuid7().hex
|
||||
response = templates.TemplateResponse(
|
||||
request,
|
||||
"tools/isslop.html",
|
||||
{**seo_ctx, "request": request, "user": user},
|
||||
)
|
||||
_set_guest_cookie(response, minted)
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
async def isslop_run(request: Request, data: Annotated[IsslopRunForm, Depends(json_or_form(IsslopRunForm))]):
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
owner_kind, owner_id, minted = _ensure_owner(request)
|
||||
active = [
|
||||
job
|
||||
for job in queue.list_jobs(kind="isslop", owner=(owner_kind, owner_id))
|
||||
if job.get("status") in ACTIVE_STATES
|
||||
]
|
||||
if active:
|
||||
audit.record(
|
||||
request,
|
||||
"isslop.run.request",
|
||||
result="denied",
|
||||
summary=f"AI usage analysis denied for {data.url}: analysis already running",
|
||||
metadata={"target": data.url, "reason": "active_job", "uid": active[0]["uid"]},
|
||||
links=[audit.job(active[0]["uid"])],
|
||||
)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"status": 429,
|
||||
"message": "You already have an analysis running. Wait for it to finish.",
|
||||
"uid": active[0]["uid"],
|
||||
}
|
||||
},
|
||||
status_code=429,
|
||||
)
|
||||
uid = queue.enqueue(
|
||||
"isslop",
|
||||
{"url": data.url},
|
||||
owner_kind,
|
||||
owner_id,
|
||||
f"AI usage: {data.url}"[:64],
|
||||
)
|
||||
store.create_analysis(uid, data.url, owner_kind, owner_id)
|
||||
audit.record(
|
||||
request,
|
||||
"isslop.run.request",
|
||||
summary=f"requested AI usage analysis of {data.url}",
|
||||
metadata={"target": data.url},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
if owner_kind == "user":
|
||||
track_action(owner_id, "isslop")
|
||||
response = JSONResponse(
|
||||
{
|
||||
"uid": uid,
|
||||
"status_url": f"/tools/isslop/{uid}",
|
||||
"events_url": f"/tools/isslop/{uid}/events",
|
||||
"report_url": f"/tools/isslop/{uid}/report",
|
||||
"topic": topic_for(uid),
|
||||
}
|
||||
)
|
||||
_set_guest_cookie(response, minted)
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
async def isslop_list(request: Request, limit: int = store.LIST_LIMIT_DEFAULT):
|
||||
_sync_guest_history(request)
|
||||
owner = _owner(request)
|
||||
rows = []
|
||||
if owner:
|
||||
rows = store.list_analyses(owner[0], owner[1], min(max(1, limit), 200))
|
||||
return JSONResponse(
|
||||
IsslopListOut.model_validate(
|
||||
{"analyses": [_analysis_payload(row) for row in rows]}
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{uid}")
|
||||
async def isslop_status(request: Request, uid: str):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
return JSONResponse(
|
||||
IsslopAnalysisOut.model_validate(_analysis_payload(row)).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{uid}/events")
|
||||
async def isslop_events(request: Request, uid: str, after: int = 0, limit: int = store.EVENT_LIMIT_DEFAULT):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
rows = store.events_for(uid, max(0, after), min(max(1, limit), EVENT_LIMIT_MAX))
|
||||
events = [
|
||||
{
|
||||
"seq": event["seq"],
|
||||
"kind": event["kind"],
|
||||
"message": event["message"],
|
||||
"data": store.decode_json(event.get("payload"), {}),
|
||||
"created_at": event["created_at"],
|
||||
}
|
||||
for event in rows
|
||||
]
|
||||
return JSONResponse({"uid": uid, "status": row.get("status", ""), "events": events})
|
||||
|
||||
|
||||
@router.get("/{uid}/report")
|
||||
async def isslop_report(request: Request, uid: str):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
report = store.get_report(uid)
|
||||
files = []
|
||||
linkable_paths: set[str] = set()
|
||||
signal_codes: set[str] = set()
|
||||
for item in store.file_results_for(uid):
|
||||
signals = store.decode_json(item.get("signals"), [])
|
||||
for signal in signals:
|
||||
if isinstance(signal, dict) and signal.get("code"):
|
||||
signal_codes.add(str(signal["code"]))
|
||||
source_name = str(item.get("source") or "")
|
||||
has_source = bool(source_name and SOURCE_NAME_PATTERN.match(source_name))
|
||||
path = item.get("path", "")
|
||||
if has_source:
|
||||
linkable_paths.add(path)
|
||||
files.append(
|
||||
{
|
||||
"path": path,
|
||||
"language": item.get("language", "unknown"),
|
||||
"lines": int(item.get("lines") or 0),
|
||||
"origin_score": float(item.get("origin_score") or 0.0),
|
||||
"quality_deficit_score": float(item.get("quality_deficit_score") or 0.0),
|
||||
"category": item.get("category", "uncertain"),
|
||||
"signals": signals,
|
||||
"signal_groups": _signal_groups(signals),
|
||||
"source_url": _source_url(uid, path) if has_source else None,
|
||||
}
|
||||
)
|
||||
images = [
|
||||
{
|
||||
"path": item.get("path", ""),
|
||||
"ai_probability": float(item.get("ai_probability") or 0.0),
|
||||
"grade": item.get("grade", "n/a"),
|
||||
"verdict": item.get("verdict", "uncertain"),
|
||||
"image_kind": item.get("image_kind", "image"),
|
||||
"tells": store.decode_json(item.get("tells"), []),
|
||||
"description": item.get("description", ""),
|
||||
"thumb_url": (
|
||||
f"/tools/isslop/{uid}/media/{item['thumb']}"
|
||||
if item.get("thumb") and MEDIA_NAME_PATTERN.match(str(item["thumb"]))
|
||||
else None
|
||||
),
|
||||
}
|
||||
for item in store.image_results_for(uid)
|
||||
]
|
||||
dom_pages = [
|
||||
{
|
||||
"url": item.get("url", ""),
|
||||
"detected_builder": item.get("detected_builder") or None,
|
||||
"signal_count": int(item.get("signal_count") or 0),
|
||||
"signals": store.decode_json(item.get("signals"), []),
|
||||
"screenshot_url": (
|
||||
f"/tools/isslop/{uid}/media/{item['screenshot']}"
|
||||
if item.get("screenshot") and MEDIA_NAME_PATTERN.match(str(item["screenshot"]))
|
||||
else None
|
||||
),
|
||||
}
|
||||
for item in store.dom_results_for(uid)
|
||||
]
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="AI usage analysis report",
|
||||
description=f"Authenticity analysis of {row.get('source_url', '')}",
|
||||
robots="noindex,nofollow",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Tools", "url": "/tools"},
|
||||
{"name": "AI Usage Analyzer", "url": "/tools/isslop"},
|
||||
{"name": "Report", "url": f"/tools/isslop/{uid}/report"},
|
||||
],
|
||||
)
|
||||
context = {
|
||||
**seo_ctx,
|
||||
**_analysis_payload(row),
|
||||
"content_hash": row.get("content_hash"),
|
||||
"markdown": _linkify_sources(report.get("markdown", ""), uid, linkable_paths, signal_codes) if report else "",
|
||||
"generator_model": report.get("model_used", "") if report else "",
|
||||
"generated_at": report.get("generated_at") if report else None,
|
||||
"badge": _badge_info(request, uid),
|
||||
"files": files,
|
||||
"images": images,
|
||||
"dom_pages": dom_pages,
|
||||
"request": request,
|
||||
"user": get_current_user(request),
|
||||
}
|
||||
return respond(request, "tools/isslop_report.html", context, model=IsslopReportOut)
|
||||
|
||||
|
||||
@router.get("/{uid}/report.md")
|
||||
async def isslop_report_markdown(request: Request, uid: str):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
report = store.get_report(uid)
|
||||
if not report:
|
||||
raise not_found("Report not yet generated for this analysis")
|
||||
return Response(
|
||||
content=report["markdown"],
|
||||
media_type="text/markdown; charset=utf-8",
|
||||
headers={"content-disposition": f'attachment; filename="ai-usage-report-{uid}.md"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{uid}/source")
|
||||
async def isslop_source(request: Request, uid: str, path: str, line: int = 0):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
result = store.file_result_for(uid, path)
|
||||
source_name = str(result.get("source") or "") if result else ""
|
||||
if not result or not SOURCE_NAME_PATTERN.match(source_name):
|
||||
raise not_found("Source not available for this file")
|
||||
source_path = (store.media_dir_for(uid) / source_name).resolve()
|
||||
if not source_path.is_relative_to(ISSLOP_MEDIA_DIR.resolve()) or not source_path.is_file():
|
||||
raise not_found("Source not available for this file")
|
||||
text = source_path.read_text(encoding="utf-8", errors="replace")
|
||||
signals = store.decode_json(result.get("signals"), [])
|
||||
marked: dict[int, list] = {}
|
||||
for signal in signals:
|
||||
if isinstance(signal, dict) and isinstance(signal.get("line"), int) and signal["line"] > 0:
|
||||
marked.setdefault(signal["line"], []).append(signal)
|
||||
source_lines = text.split("\n")
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title=f"Source: {path}",
|
||||
description=f"Annotated source of {path} from the AI usage analysis",
|
||||
robots="noindex,nofollow",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Tools", "url": "/tools"},
|
||||
{"name": "AI Usage Analyzer", "url": "/tools/isslop"},
|
||||
{"name": "Report", "url": f"/tools/isslop/{uid}/report"},
|
||||
{"name": "Source", "url": _source_url(uid, path)},
|
||||
],
|
||||
)
|
||||
context = {
|
||||
**seo_ctx,
|
||||
"uid": uid,
|
||||
"path": path,
|
||||
"language": result.get("language", "unknown"),
|
||||
"category": result.get("category", "uncertain"),
|
||||
"origin_score": float(result.get("origin_score") or 0.0),
|
||||
"quality_deficit_score": float(result.get("quality_deficit_score") or 0.0),
|
||||
"source": text,
|
||||
"truncated": len(text) >= SOURCE_LINE_CONTEXT,
|
||||
"signals": signals,
|
||||
"source_lines": source_lines,
|
||||
"marked_lines": marked,
|
||||
"focus_line": max(0, line),
|
||||
"report_url": f"/tools/isslop/{uid}/report",
|
||||
"request": request,
|
||||
"user": get_current_user(request),
|
||||
}
|
||||
return respond(request, "tools/isslop_source.html", context, model=IsslopSourceOut)
|
||||
|
||||
|
||||
@router.get("/{uid}/media/{name}")
|
||||
async def isslop_media(request: Request, uid: str, name: str):
|
||||
row = store.get_analysis(uid)
|
||||
if not row or not MEDIA_NAME_PATTERN.match(name):
|
||||
raise not_found("Image not available")
|
||||
root = ISSLOP_MEDIA_DIR.resolve()
|
||||
path = (store.media_dir_for(uid) / name).resolve()
|
||||
if not path.is_relative_to(root) or not path.is_file():
|
||||
raise not_found("Image not available")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type="image/webp",
|
||||
headers={"cache-control": "public, max-age=86400"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{uid}/badge.svg")
|
||||
async def isslop_badge(request: Request, uid: str):
|
||||
row = store.get_analysis(uid)
|
||||
if not row:
|
||||
raise not_found("Analysis not found")
|
||||
report_url = _badge_info(request, uid)["report_url"]
|
||||
svg = render_badge(row.get("human_percent"), row.get("grade"), report_url)
|
||||
return Response(
|
||||
content=svg,
|
||||
media_type="image/svg+xml",
|
||||
headers={"cache-control": "no-cache, max-age=300"},
|
||||
)
|
||||
@ -86,6 +86,10 @@ from devplacepy.schemas.jobs import (
|
||||
SeoMetaOut,
|
||||
SeoReportOut,
|
||||
ZipJobOut,
|
||||
IsslopAnalysisOut,
|
||||
IsslopListOut,
|
||||
IsslopReportOut,
|
||||
IsslopSourceOut,
|
||||
)
|
||||
from devplacepy.schemas.backups import (
|
||||
BackupDashboardOut,
|
||||
|
||||
@ -153,3 +153,75 @@ class DbQueryJobOut(_Out):
|
||||
error: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
completed_at: Optional[str] = None
|
||||
|
||||
|
||||
class IsslopAnalysisOut(_Out):
|
||||
uid: str = ""
|
||||
status: str = ""
|
||||
source_url: str = ""
|
||||
source_kind: str = ""
|
||||
grade: Optional[str] = None
|
||||
slop_score: Optional[float] = None
|
||||
origin_score: Optional[float] = None
|
||||
quality_deficit_score: Optional[float] = None
|
||||
human_percent: Optional[float] = None
|
||||
ai_percent: Optional[float] = None
|
||||
category: Optional[str] = None
|
||||
confidence: Optional[str] = None
|
||||
files_total: int = 0
|
||||
files_analyzed: int = 0
|
||||
detected_builder: Optional[str] = None
|
||||
dom_slop_score: Optional[float] = None
|
||||
error: Optional[str] = None
|
||||
report_url: Optional[str] = None
|
||||
badge_url: Optional[str] = None
|
||||
events_url: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
|
||||
|
||||
class IsslopListOut(_Out):
|
||||
analyses: list = []
|
||||
|
||||
|
||||
class IsslopReportOut(_Out):
|
||||
uid: str = ""
|
||||
status: str = ""
|
||||
source_url: str = ""
|
||||
source_kind: str = ""
|
||||
grade: Optional[str] = None
|
||||
slop_score: Optional[float] = None
|
||||
origin_score: Optional[float] = None
|
||||
quality_deficit_score: Optional[float] = None
|
||||
human_percent: Optional[float] = None
|
||||
ai_percent: Optional[float] = None
|
||||
category: Optional[str] = None
|
||||
confidence: Optional[str] = None
|
||||
files_total: int = 0
|
||||
files_analyzed: int = 0
|
||||
detected_builder: Optional[str] = None
|
||||
dom_slop_score: Optional[float] = None
|
||||
error: Optional[str] = None
|
||||
content_hash: Optional[str] = None
|
||||
markdown: str = ""
|
||||
generator_model: str = ""
|
||||
generated_at: Optional[str] = None
|
||||
badge: dict = {}
|
||||
files: list = []
|
||||
images: list = []
|
||||
dom_pages: list = []
|
||||
created_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
|
||||
|
||||
class IsslopSourceOut(_Out):
|
||||
uid: str = ""
|
||||
path: str = ""
|
||||
language: str = ""
|
||||
category: str = ""
|
||||
origin_score: float = 0.0
|
||||
quality_deficit_score: float = 0.0
|
||||
source: str = ""
|
||||
truncated: bool = False
|
||||
signals: list = []
|
||||
|
||||
@ -32,6 +32,7 @@ CATEGORY_BY_PREFIX: dict[str, str] = {
|
||||
"proxy": "ingress",
|
||||
"seo": "tools",
|
||||
"deepsearch": "tools",
|
||||
"isslop": "tools",
|
||||
"ai": "ai",
|
||||
"database": "database",
|
||||
"pubsub": "pubsub",
|
||||
|
||||
@ -102,7 +102,7 @@ REACT_RATES = {
|
||||
}
|
||||
REACT_RATE_DEFAULT = 0.20
|
||||
|
||||
CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random"]
|
||||
CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random", "politics"]
|
||||
|
||||
GIST_LANGUAGES = [
|
||||
"python",
|
||||
@ -122,7 +122,7 @@ GIST_LANGUAGES = [
|
||||
"lua",
|
||||
]
|
||||
|
||||
FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun"]
|
||||
FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun", "politics"]
|
||||
PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
|
||||
|
||||
SEARCH_TERMS = {
|
||||
@ -240,6 +240,7 @@ PERSONA_CATEGORY_WEIGHTS = {
|
||||
"question": 1,
|
||||
"showcase": 1,
|
||||
"fun": 1,
|
||||
"politics": 1,
|
||||
},
|
||||
"hobbyist_maker": {
|
||||
"showcase": 3,
|
||||
@ -256,6 +257,7 @@ PERSONA_CATEGORY_WEIGHTS = {
|
||||
"showcase": 1,
|
||||
"rant": 1,
|
||||
"fun": 1,
|
||||
"politics": 1,
|
||||
},
|
||||
"minimalist": {
|
||||
"random": 3,
|
||||
@ -280,6 +282,7 @@ PERSONA_CATEGORY_WEIGHTS = {
|
||||
"random": 1,
|
||||
"showcase": 1,
|
||||
"devlog": 1,
|
||||
"politics": 2,
|
||||
},
|
||||
"mentor": {
|
||||
"devlog": 3,
|
||||
@ -288,6 +291,7 @@ PERSONA_CATEGORY_WEIGHTS = {
|
||||
"random": 1,
|
||||
"rant": 1,
|
||||
"fun": 1,
|
||||
"politics": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -305,7 +305,7 @@ class BotEngageMixin:
|
||||
else:
|
||||
comment = f"@{mention_target} {comment}"
|
||||
|
||||
comment = comment[:2000]
|
||||
comment = self._sanitize_mentions(comment)[:2000]
|
||||
mention_log = f" @{mention_target}" if mention_target else ""
|
||||
|
||||
textarea_sels = [
|
||||
@ -437,7 +437,7 @@ class BotEngageMixin:
|
||||
return False
|
||||
if mentioner and f"@{mentioner.lower()}" not in reply.lower():
|
||||
reply = f"@{mentioner} {reply}"
|
||||
reply = reply[:2000]
|
||||
reply = self._sanitize_mentions(reply)[:2000]
|
||||
textarea_sels = [
|
||||
".reply-form textarea[name='content']",
|
||||
".comment-form textarea[name='content']",
|
||||
|
||||
@ -11,6 +11,8 @@ from devplacepy.services.bot.config import persona_article_score, pick_category
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MENTION_RE = re.compile(r"@([A-Za-z0-9_-]+)")
|
||||
|
||||
|
||||
class BotHelpersMixin:
|
||||
def _identity(self) -> str:
|
||||
@ -31,6 +33,21 @@ class BotHelpersMixin:
|
||||
m = re.search(r"/(?:posts|gists|news|projects)/([^/?#]+)", url)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
def _sanitize_mentions(self, text: str) -> str:
|
||||
own = (self.state.username or "").lower()
|
||||
seen: set[str] = set()
|
||||
|
||||
def keep(match: "re.Match[str]") -> str:
|
||||
handle = match.group(1)
|
||||
lowered = handle.lower()
|
||||
if lowered == own or lowered in seen:
|
||||
return ""
|
||||
seen.add(lowered)
|
||||
return match.group(0)
|
||||
|
||||
cleaned = _MENTION_RE.sub(keep, text)
|
||||
return re.sub(r"\s{2,}", " ", cleaned).strip()
|
||||
|
||||
def _sync_cost(self) -> None:
|
||||
self.state.total_cost = self.llm.total_cost
|
||||
self.state.total_calls = self.llm.total_calls
|
||||
|
||||
@ -115,7 +115,10 @@ class LLMClient:
|
||||
@staticmethod
|
||||
def clean(text: str, preserve_md: bool = False) -> str:
|
||||
if not preserve_md:
|
||||
text = re.sub(r"\*\*|__|\*|_", "", text)
|
||||
text = re.sub(r"\*+", "", text)
|
||||
text = re.sub(r"(?<![A-Za-z0-9])__(?=\S)(.*?)(?<=\S)__(?![A-Za-z0-9])", r"\1", text)
|
||||
text = re.sub(r"(?<![A-Za-z0-9])_(?=\S)(.*?)(?<=\S)_(?![A-Za-z0-9])", r"\1", text)
|
||||
text = re.sub(r"(?<![A-Za-z0-9])_+(?![A-Za-z0-9])", "", text)
|
||||
text = text.replace("—", "-").replace("–", "-")
|
||||
text = text.replace("‘", "'").replace("’", "'")
|
||||
text = text.replace("“", '"').replace("”", '"')
|
||||
@ -193,6 +196,7 @@ class LLMClient:
|
||||
"rant": "Write it as an opinionated rant - strong viewpoint, passionate criticism, but keep it substantive.",
|
||||
"fun": "Write it lighthearted and playful, but stay tied to the actual tech topic. Humor about the technology itself, not off-topic jokes or lyrics.",
|
||||
"random": "Be natural and conversational - share your thoughts like any casual discussion.",
|
||||
"politics": "Write it as a measured take on the politics of this technology - regulation, governance, open-source licensing, industry power, or ethics. Stay substantive and non-partisan; argue the policy angle, not party lines.",
|
||||
}
|
||||
persona_extra = (
|
||||
f" {persona_extras.get(persona, 'Be casual. No markdown.')}"
|
||||
|
||||
@ -28,7 +28,7 @@ class BotSocialMixin:
|
||||
try:
|
||||
await links.nth(i).click(timeout=5000)
|
||||
await b._idle(1.0, 2.0)
|
||||
uname = h.split("/profile/")[-1].split("?")[0]
|
||||
uname = h.split("/profile/")[-1].split("?")[0].split("/")[0]
|
||||
if uname not in self.state.known_users:
|
||||
self.state.known_users.append(uname)
|
||||
self.state.profiles_viewed += 1
|
||||
@ -446,6 +446,9 @@ class BotSocialMixin:
|
||||
)
|
||||
except Exception:
|
||||
mentioner = ""
|
||||
if mentioner and mentioner.lower() == self.state.username.lower():
|
||||
self._log("Mention target resolves to self, skipping")
|
||||
return False
|
||||
try:
|
||||
await comment_loc.scroll_into_view_if_needed(timeout=2000)
|
||||
await b._idle(0.3, 0.9)
|
||||
@ -485,7 +488,7 @@ class BotSocialMixin:
|
||||
return False
|
||||
if mentioner and f"@{mentioner.lower()}" not in reply.lower():
|
||||
reply = f"@{mentioner} {reply}"
|
||||
reply = reply[:2000]
|
||||
reply = self._sanitize_mentions(reply)[:2000]
|
||||
textarea_sels = [
|
||||
".comment .reply-form textarea[name='content']",
|
||||
".reply-form textarea[name='content']",
|
||||
|
||||
@ -415,13 +415,13 @@ def pravda_env(instance: dict) -> dict:
|
||||
slug = instance.get("ingress_slug") or ""
|
||||
ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else ""
|
||||
return {
|
||||
"PRAVDA_BASE_URL": base_url,
|
||||
"PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
|
||||
"PRAVDA_API_KEY": api_key,
|
||||
"PRAVDA_USER_UID": instance.get("run_as_uid") or user_uid,
|
||||
"PRAVDA_CONTAINER_NAME": instance.get("name") or "",
|
||||
"PRAVDA_CONTAINER_UID": instance.get("uid") or "",
|
||||
"PRAVDA_INGRESS_URL": ingress_url,
|
||||
"DEVPLACE_BASE_URL": base_url,
|
||||
"DEVPLACE_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
|
||||
"DEVPLACE_API_KEY": api_key,
|
||||
"DEVPLACE_USER_UID": instance.get("run_as_uid") or user_uid,
|
||||
"DEVPLACE_CONTAINER_NAME": instance.get("name") or "",
|
||||
"DEVPLACE_CONTAINER_UID": instance.get("uid") or "",
|
||||
"DEVPLACE_INGRESS_URL": ingress_url,
|
||||
}
|
||||
|
||||
|
||||
|
||||
637
devplacepy/services/containers/api.py.bak
Normal file
637
devplacepy/services/containers/api.py.bak
Normal file
@ -0,0 +1,637 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy import config, project_files, stealth
|
||||
from devplacepy.services.containers import store
|
||||
from devplacepy.services.containers.backend.base import (
|
||||
WORKSPACE_MOUNT,
|
||||
Mount,
|
||||
PortMapping,
|
||||
RunSpec,
|
||||
)
|
||||
from devplacepy.services.containers.runtime import get_backend
|
||||
from devplacepy.services.devii.tasks.schedule import (
|
||||
Schedule,
|
||||
now_utc,
|
||||
to_iso,
|
||||
)
|
||||
|
||||
IMAGE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}$")
|
||||
INGRESS_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
MEM_RE = re.compile(r"^\d+(\.\d+)?[bkmgBKMG]?$")
|
||||
CPU_RE = re.compile(r"^\d+(\.\d+)?$")
|
||||
ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
INSTANCE_LABEL = "devplace.instance"
|
||||
PROJECT_LABEL = "devplace.project"
|
||||
HOST_PORT_MIN = 20001
|
||||
HOST_PORT_MAX = 65535
|
||||
BOOT_LANGUAGES = ("none", "python", "bash")
|
||||
BOOT_SCRIPT_FILES = {"python": ".devplace_boot.py", "bash": ".devplace_boot.sh"}
|
||||
BOOT_SCRIPT_RUNNERS = {"python": "python", "bash": "bash"}
|
||||
MAX_BOOT_SCRIPT_CHARS = 100_000
|
||||
|
||||
|
||||
class ContainerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _validate_limits(cpu_limit: str, mem_limit: str) -> None:
|
||||
if cpu_limit and not CPU_RE.match(str(cpu_limit)):
|
||||
raise ContainerError("cpu limit must be a number, e.g. 1 or 1.5")
|
||||
if mem_limit and not MEM_RE.match(str(mem_limit)):
|
||||
raise ContainerError("memory limit must look like 512m, 1g, or a byte count")
|
||||
|
||||
|
||||
def validate_run_as(run_as_uid) -> str:
|
||||
uid = str(run_as_uid or "").strip()
|
||||
if not uid:
|
||||
return ""
|
||||
from devplacepy import database
|
||||
|
||||
user = database.get_users_by_uids([uid]).get(uid)
|
||||
if not user:
|
||||
raise ContainerError(f"run-as user not found: {uid}")
|
||||
return uid
|
||||
|
||||
|
||||
def validate_boot(boot_language, boot_script) -> tuple:
|
||||
language = str(boot_language or "none").strip().lower() or "none"
|
||||
if language not in BOOT_LANGUAGES:
|
||||
raise ContainerError(
|
||||
f"boot language must be one of {', '.join(BOOT_LANGUAGES)}"
|
||||
)
|
||||
script = str(boot_script or "")
|
||||
if language == "none":
|
||||
script = ""
|
||||
if len(script) > MAX_BOOT_SCRIPT_CHARS:
|
||||
raise ContainerError(
|
||||
f"boot script exceeds the {MAX_BOOT_SCRIPT_CHARS}-character limit"
|
||||
)
|
||||
if language != "none" and not script.strip():
|
||||
raise ContainerError("boot script is required when a boot language is set")
|
||||
return language, script
|
||||
|
||||
|
||||
def parse_ports(value) -> list:
|
||||
ports = []
|
||||
if not value:
|
||||
return ports
|
||||
items = (
|
||||
value if isinstance(value, list) else str(value).replace(",", "\n").splitlines()
|
||||
)
|
||||
for item in items:
|
||||
item = str(item).strip()
|
||||
if not item:
|
||||
continue
|
||||
proto = "tcp"
|
||||
if "/" in item:
|
||||
item, proto = item.split("/", 1)
|
||||
if ":" in item:
|
||||
host, container = item.split(":", 1)
|
||||
else:
|
||||
host, container = "0", item
|
||||
if not host.isdigit() or not container.isdigit():
|
||||
raise ContainerError(
|
||||
f"port '{item}' must be numeric host:container or a bare container port"
|
||||
)
|
||||
ports.append(PortMapping(int(host), int(container), proto.strip() or "tcp"))
|
||||
return ports
|
||||
|
||||
|
||||
def used_host_ports() -> set:
|
||||
ports = set()
|
||||
for instance in store.all_instances():
|
||||
for mapping in json.loads(instance.get("ports_json") or "[]"):
|
||||
host = int(mapping.get("host") or 0)
|
||||
if host:
|
||||
ports.add(host)
|
||||
return ports
|
||||
|
||||
|
||||
def _host_port_free(port: int) -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind(("0.0.0.0", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def allocate_host_port(reserved: set) -> int:
|
||||
for port in range(HOST_PORT_MIN, HOST_PORT_MAX + 1):
|
||||
if port in reserved:
|
||||
continue
|
||||
if _host_port_free(port):
|
||||
return port
|
||||
raise ContainerError(
|
||||
f"no free host port available in range {HOST_PORT_MIN}-{HOST_PORT_MAX}"
|
||||
)
|
||||
|
||||
|
||||
def assign_host_ports(port_list: list) -> list:
|
||||
reserved = used_host_ports()
|
||||
assigned = []
|
||||
for mapping in port_list:
|
||||
host = mapping.host
|
||||
if not host:
|
||||
host = allocate_host_port(reserved)
|
||||
elif host in reserved:
|
||||
raise ContainerError(
|
||||
f"host port {host} is already published by another instance"
|
||||
)
|
||||
reserved.add(host)
|
||||
assigned.append(PortMapping(host, mapping.container, mapping.proto))
|
||||
return assigned
|
||||
|
||||
|
||||
def parse_env(value) -> dict:
|
||||
env = {}
|
||||
if not value:
|
||||
return env
|
||||
if isinstance(value, dict):
|
||||
items = value.items()
|
||||
else:
|
||||
items = (line.split("=", 1) for line in str(value).splitlines() if "=" in line)
|
||||
for key, val in items:
|
||||
key = str(key).strip()
|
||||
if not ENV_KEY_RE.match(key):
|
||||
raise ContainerError(f"invalid environment variable name: {key}")
|
||||
env[key] = str(val)
|
||||
return env
|
||||
|
||||
|
||||
# ---------------- instances ----------------
|
||||
|
||||
|
||||
def validate_ingress(slug: str, port, port_list) -> tuple:
|
||||
slug = (slug or "").strip().lower()
|
||||
if not slug:
|
||||
return "", 0
|
||||
if not INGRESS_SLUG_RE.match(slug):
|
||||
raise ContainerError(
|
||||
"ingress slug must be lowercase letters, digits, or '-' (max 63 chars)"
|
||||
)
|
||||
for other in store.all_instances():
|
||||
if other.get("ingress_slug") == slug:
|
||||
raise ContainerError(f"ingress slug '{slug}' is already in use")
|
||||
ingress_port = int(port) if port else 0
|
||||
container_ports = {p.container for p in port_list}
|
||||
if ingress_port and ingress_port not in container_ports:
|
||||
raise ContainerError(
|
||||
f"ingress_port {ingress_port} must be one of the container ports you mapped"
|
||||
)
|
||||
if not ingress_port and len(container_ports) != 1:
|
||||
raise ContainerError(
|
||||
"set ingress_port to choose which mapped container port to publish"
|
||||
)
|
||||
return slug, ingress_port
|
||||
|
||||
|
||||
async def create_instance(
|
||||
project: dict,
|
||||
*,
|
||||
name: str,
|
||||
boot_command: str = "",
|
||||
boot_language: str = "none",
|
||||
boot_script: str = "",
|
||||
run_as_uid: str = "",
|
||||
start_on_boot: bool = False,
|
||||
env="",
|
||||
cpu_limit: str = "",
|
||||
mem_limit: str = "",
|
||||
ports="",
|
||||
volumes="",
|
||||
restart_policy: str = "never",
|
||||
autostart: bool = True,
|
||||
ingress_slug: str = "",
|
||||
ingress_port=None,
|
||||
actor=("system", "system"),
|
||||
) -> dict:
|
||||
if not await get_backend().image_exists(config.CONTAINER_IMAGE):
|
||||
raise ContainerError(
|
||||
f"the '{config.CONTAINER_IMAGE}' image is not built - run 'make ppy'"
|
||||
)
|
||||
if restart_policy not in store.RESTART_POLICIES:
|
||||
raise ContainerError(
|
||||
f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
|
||||
)
|
||||
_validate_limits(cpu_limit, mem_limit)
|
||||
run_as_uid = validate_run_as(run_as_uid)
|
||||
boot_language, boot_script = validate_boot(boot_language, boot_script)
|
||||
port_list = assign_host_ports(parse_ports(ports))
|
||||
env_map = parse_env(env)
|
||||
name = (name or "").strip()
|
||||
if not name:
|
||||
raise ContainerError("instance name is required")
|
||||
ingress_slug, ingress_port = validate_ingress(ingress_slug, ingress_port, port_list)
|
||||
|
||||
workspace = Path(config.CONTAINER_WORKSPACES_DIR) / project["uid"]
|
||||
await asyncio.to_thread(
|
||||
project_files.export_to_dir, project["uid"], "", str(workspace)
|
||||
)
|
||||
|
||||
row = {
|
||||
"project_uid": project["uid"],
|
||||
"created_by": actor[1] if actor and actor[0] == "user" else "",
|
||||
"owner_uid": project.get("user_uid", ""),
|
||||
"run_as_uid": run_as_uid,
|
||||
"name": name,
|
||||
"boot_command": boot_command or "",
|
||||
"boot_language": boot_language,
|
||||
"boot_script": boot_script,
|
||||
"start_on_boot": 1 if start_on_boot else 0,
|
||||
"env_json": json.dumps(env_map),
|
||||
"cpu_limit": str(cpu_limit or ""),
|
||||
"mem_limit": str(mem_limit or ""),
|
||||
"ports_json": json.dumps(
|
||||
[
|
||||
{"host": p.host, "container": p.container, "proto": p.proto}
|
||||
for p in port_list
|
||||
]
|
||||
),
|
||||
"volumes_json": volumes
|
||||
if isinstance(volumes, str)
|
||||
else json.dumps(volumes or []),
|
||||
"restart_policy": restart_policy,
|
||||
"ingress_slug": ingress_slug,
|
||||
"ingress_port": ingress_port,
|
||||
"desired_state": store.DESIRED_RUNNING if autostart else store.DESIRED_STOPPED,
|
||||
"status": store.ST_CREATED,
|
||||
"workspace_dir": str(workspace),
|
||||
}
|
||||
instance = store.create_instance(row)
|
||||
store.record_event(
|
||||
instance, "created", actor[0], actor[1], {"image": config.CONTAINER_IMAGE}
|
||||
)
|
||||
if actor and actor[0] == "user":
|
||||
from devplacepy.utils import track_action
|
||||
|
||||
track_action(actor[1], "container")
|
||||
return instance
|
||||
|
||||
|
||||
def set_desired_state(
|
||||
instance: dict, desired: str, *, actor=("system", "system")
|
||||
) -> dict:
|
||||
if desired not in (
|
||||
store.DESIRED_RUNNING,
|
||||
store.DESIRED_STOPPED,
|
||||
store.DESIRED_PAUSED,
|
||||
):
|
||||
raise ContainerError("desired state must be running, stopped, or paused")
|
||||
store.update_instance(instance["uid"], {"desired_state": desired})
|
||||
store.record_event(instance, f"desire_{desired}", actor[0], actor[1])
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def request_restart(instance: dict, *, actor=("system", "system")) -> dict:
|
||||
store.update_instance(
|
||||
instance["uid"],
|
||||
{"desired_state": store.DESIRED_RUNNING, "status": store.ST_RESTARTING},
|
||||
)
|
||||
store.record_event(instance, "restart", actor[0], actor[1])
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None:
|
||||
store.update_instance(
|
||||
instance["uid"],
|
||||
{"desired_state": store.DESIRED_STOPPED, "status": store.ST_REMOVING},
|
||||
)
|
||||
store.record_event(instance, "remove", actor[0], actor[1])
|
||||
|
||||
|
||||
def update_instance_config(
|
||||
instance: dict,
|
||||
*,
|
||||
run_as_uid=None,
|
||||
boot_language=None,
|
||||
boot_script=None,
|
||||
boot_command=None,
|
||||
restart_policy=None,
|
||||
start_on_boot=None,
|
||||
cpu_limit=None,
|
||||
mem_limit=None,
|
||||
actor=("system", "system"),
|
||||
) -> dict:
|
||||
changes: dict = {}
|
||||
if run_as_uid is not None:
|
||||
changes["run_as_uid"] = validate_run_as(run_as_uid)
|
||||
if boot_language is not None or boot_script is not None:
|
||||
language = (
|
||||
boot_language
|
||||
if boot_language is not None
|
||||
else instance.get("boot_language", "none")
|
||||
)
|
||||
script = (
|
||||
boot_script if boot_script is not None else instance.get("boot_script", "")
|
||||
)
|
||||
language, script = validate_boot(language, script)
|
||||
changes["boot_language"] = language
|
||||
changes["boot_script"] = script
|
||||
if boot_command is not None:
|
||||
changes["boot_command"] = str(boot_command or "")[:500]
|
||||
if restart_policy is not None:
|
||||
if restart_policy not in store.RESTART_POLICIES:
|
||||
raise ContainerError(
|
||||
f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
|
||||
)
|
||||
changes["restart_policy"] = restart_policy
|
||||
if start_on_boot is not None:
|
||||
changes["start_on_boot"] = 1 if start_on_boot else 0
|
||||
if cpu_limit is not None or mem_limit is not None:
|
||||
cpu = cpu_limit if cpu_limit is not None else instance.get("cpu_limit", "")
|
||||
mem = mem_limit if mem_limit is not None else instance.get("mem_limit", "")
|
||||
_validate_limits(cpu, mem)
|
||||
changes["cpu_limit"] = str(cpu or "")
|
||||
changes["mem_limit"] = str(mem or "")
|
||||
if not changes:
|
||||
return store.get_instance(instance["uid"])
|
||||
store.update_instance(instance["uid"], changes)
|
||||
store.record_event(
|
||||
instance, "configure", actor[0], actor[1], {"fields": sorted(changes)}
|
||||
)
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def set_start_on_boot(
|
||||
instance: dict, enabled: bool, *, actor=("system", "system")
|
||||
) -> dict:
|
||||
store.update_instance(instance["uid"], {"start_on_boot": 1 if enabled else 0})
|
||||
store.record_event(
|
||||
instance, "start_on_boot", actor[0], actor[1], {"enabled": bool(enabled)}
|
||||
)
|
||||
return store.get_instance(instance["uid"])
|
||||
|
||||
|
||||
def materialize_boot_script(instance: dict) -> None:
|
||||
language = (instance.get("boot_language") or "none").strip().lower()
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
return
|
||||
for filename in BOOT_SCRIPT_FILES.values():
|
||||
stale = Path(workspace) / filename
|
||||
if stale.is_file():
|
||||
try:
|
||||
stale.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
if language not in BOOT_SCRIPT_FILES:
|
||||
return
|
||||
script = instance.get("boot_script") or ""
|
||||
if not script.strip():
|
||||
return
|
||||
target = Path(workspace) / BOOT_SCRIPT_FILES[language]
|
||||
try:
|
||||
Path(workspace).mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(script, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def pravda_env(instance: dict) -> dict:
|
||||
from devplacepy import database, seo
|
||||
|
||||
base_url = seo.public_base_url()
|
||||
api_key = ""
|
||||
user_uid = instance.get("owner_uid") or ""
|
||||
for uid in (
|
||||
instance.get("run_as_uid"),
|
||||
instance.get("created_by"),
|
||||
instance.get("owner_uid"),
|
||||
):
|
||||
if not uid:
|
||||
continue
|
||||
user = database.get_users_by_uids([uid]).get(uid)
|
||||
if user and user.get("api_key"):
|
||||
api_key = user["api_key"]
|
||||
break
|
||||
slug = instance.get("ingress_slug") or ""
|
||||
ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else ""
|
||||
return {
|
||||
"PRAVDA_BASE_URL": base_url,
|
||||
"PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
|
||||
"PRAVDA_API_KEY": api_key,
|
||||
"PRAVDA_USER_UID": instance.get("run_as_uid") or user_uid,
|
||||
"PRAVDA_CONTAINER_NAME": instance.get("name") or "",
|
||||
"PRAVDA_CONTAINER_UID": instance.get("uid") or "",
|
||||
"PRAVDA_INGRESS_URL": ingress_url,
|
||||
}
|
||||
|
||||
|
||||
def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
|
||||
env = {**json.loads(instance.get("env_json") or "{}"), **pravda_env(instance)}
|
||||
ports = [
|
||||
PortMapping(p["host"], p["container"], p.get("proto", "tcp"))
|
||||
for p in json.loads(instance.get("ports_json") or "[]")
|
||||
]
|
||||
mounts = [Mount(instance["workspace_dir"], WORKSPACE_MOUNT, "rw")]
|
||||
for extra in json.loads(instance.get("volumes_json") or "[]"):
|
||||
if isinstance(extra, dict) and extra.get("host") and extra.get("container"):
|
||||
mounts.append(
|
||||
Mount(extra["host"], extra["container"], extra.get("mode", "rw"))
|
||||
)
|
||||
language = (instance.get("boot_language") or "none").strip().lower()
|
||||
boot = (instance.get("boot_command") or "").strip()
|
||||
if language in BOOT_SCRIPT_FILES and (instance.get("boot_script") or "").strip():
|
||||
script_path = f"{WORKSPACE_MOUNT}/{BOOT_SCRIPT_FILES[language]}"
|
||||
command = [BOOT_SCRIPT_RUNNERS[language], script_path]
|
||||
elif boot:
|
||||
command = ["/bin/sh", "-c", boot]
|
||||
else:
|
||||
command = ["sleep", "infinity"]
|
||||
return RunSpec(
|
||||
image=image_tag,
|
||||
name=instance["slug"],
|
||||
labels={
|
||||
INSTANCE_LABEL: instance["uid"],
|
||||
PROJECT_LABEL: instance["project_uid"],
|
||||
},
|
||||
env=env,
|
||||
cpu_limit=instance.get("cpu_limit", ""),
|
||||
mem_limit=instance.get("mem_limit", ""),
|
||||
ports=ports,
|
||||
mounts=mounts,
|
||||
restart_policy=instance.get("restart_policy", "never"),
|
||||
command=command,
|
||||
)
|
||||
|
||||
|
||||
async def sync_workspace(instance: dict, user: dict) -> dict:
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
raise ContainerError("instance has no workspace")
|
||||
counts = await asyncio.to_thread(
|
||||
project_files.sync_dir_bidirectional, instance["project_uid"], workspace, user
|
||||
)
|
||||
store.record_event(
|
||||
instance,
|
||||
"sync",
|
||||
"user",
|
||||
user["uid"],
|
||||
{"exported": counts["exported"], "imported": counts["imported"]},
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def sync_bidirectional_sync(instance: dict, user: dict) -> dict:
|
||||
workspace = instance.get("workspace_dir")
|
||||
if not workspace:
|
||||
return {"exported": 0, "imported": 0}
|
||||
counts = project_files.sync_dir_bidirectional(
|
||||
instance["project_uid"], workspace, user
|
||||
)
|
||||
if counts["exported"] or counts["imported"]:
|
||||
store.record_event(
|
||||
instance,
|
||||
"sync",
|
||||
"service",
|
||||
"system",
|
||||
{"exported": counts["exported"], "imported": counts["imported"]},
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def add_schedule(instance: dict, action: str, schedule: Schedule) -> dict:
|
||||
if action not in ("start", "stop"):
|
||||
raise ContainerError("schedule action must be start or stop")
|
||||
first = schedule.first_run(now_utc())
|
||||
return store.create_schedule(instance, action, schedule.columns(), to_iso(first))
|
||||
|
||||
|
||||
# ---------------- aggregation ----------------
|
||||
|
||||
|
||||
def _percentile(values: list, pct: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1))))
|
||||
return float(ordered[index])
|
||||
|
||||
|
||||
def instance_stats(instance_uid: str) -> dict:
|
||||
metrics = store.recent_metrics(instance_uid, limit=720)
|
||||
cpu = [m.get("cpu_pct", 0) for m in metrics]
|
||||
mem = [m.get("mem_bytes", 0) for m in metrics]
|
||||
return {
|
||||
"samples": len(metrics),
|
||||
"cpu_avg": round(sum(cpu) / len(cpu), 2) if cpu else 0.0,
|
||||
"cpu_p95": round(_percentile(cpu, 95), 2),
|
||||
"mem_max": max(mem) if mem else 0,
|
||||
"mem_avg": int(sum(mem) / len(mem)) if mem else 0,
|
||||
}
|
||||
|
||||
|
||||
def _port_reachable(host: str, port: int, timeout: float = 0.3) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _http_probe(host: str, port: int, timeout: float = 1.0) -> str:
|
||||
try:
|
||||
with stealth.stealth_sync_client(timeout=timeout) as client:
|
||||
response = client.get(f"http://{host}:{port}/")
|
||||
return f"HTTP {response.status_code}"
|
||||
except Exception as exc: # noqa: BLE001 - diagnostic, any failure is informative
|
||||
return f"unreachable: {type(exc).__name__}"
|
||||
|
||||
|
||||
def _net_entry(data: dict) -> dict:
|
||||
net = (data or {}).get("NetworkSettings") or {}
|
||||
if net.get("IPAddress") or net.get("Gateway"):
|
||||
return net
|
||||
for entry in (net.get("Networks") or {}).values():
|
||||
if entry and (entry.get("IPAddress") or entry.get("Gateway")):
|
||||
return entry
|
||||
return {}
|
||||
|
||||
|
||||
def container_ip_from_inspect(data: dict) -> str:
|
||||
return (_net_entry(data).get("IPAddress") or "").strip()
|
||||
|
||||
|
||||
def container_gateway_from_inspect(data: dict) -> str:
|
||||
return (_net_entry(data).get("Gateway") or "").strip()
|
||||
|
||||
|
||||
def _ingress_container_port(instance: dict, port_maps: list) -> int:
|
||||
ingress_port = int(instance.get("ingress_port") or 0)
|
||||
if ingress_port:
|
||||
for mapping in port_maps:
|
||||
if int(mapping.get("container") or 0) == ingress_port:
|
||||
return ingress_port
|
||||
return 0
|
||||
return int(port_maps[0].get("container") or 0) if port_maps else 0
|
||||
|
||||
|
||||
def _host_port_for(port_maps: list, container_port: int) -> int:
|
||||
for mapping in port_maps:
|
||||
if int(mapping.get("container") or 0) == container_port:
|
||||
return int(mapping.get("host") or 0)
|
||||
return 0
|
||||
|
||||
|
||||
def proxy_target(instance: dict) -> tuple:
|
||||
port_maps = json.loads(instance.get("ports_json") or "[]")
|
||||
container_port = _ingress_container_port(instance, port_maps)
|
||||
if not container_port:
|
||||
return None, None
|
||||
host_port = _host_port_for(port_maps, container_port)
|
||||
if config.CONTAINER_PROXY_HOST:
|
||||
return (config.CONTAINER_PROXY_HOST, host_port) if host_port else (None, None)
|
||||
if not host_port:
|
||||
return None, None
|
||||
gateway = (instance.get("container_gateway") or "").strip()
|
||||
return (gateway or "127.0.0.1", host_port)
|
||||
|
||||
|
||||
def instance_runtime(instance: dict) -> dict:
|
||||
boot = (instance.get("boot_command") or "").strip()
|
||||
port_maps = json.loads(instance.get("ports_json") or "[]")
|
||||
container_ip = (instance.get("container_ip") or "").strip()
|
||||
container_gateway = (instance.get("container_gateway") or "").strip()
|
||||
target_host, target_port = proxy_target(instance)
|
||||
probe_host = config.CONTAINER_PROXY_HOST or container_gateway or "127.0.0.1"
|
||||
ports = []
|
||||
for mapping in port_maps:
|
||||
host_port = int(mapping.get("host") or 0)
|
||||
ports.append(
|
||||
{
|
||||
"container": int(mapping.get("container") or 0),
|
||||
"host": host_port,
|
||||
"proto": mapping.get("proto", "tcp"),
|
||||
"reachable": _port_reachable(probe_host, host_port)
|
||||
if host_port
|
||||
else False,
|
||||
}
|
||||
)
|
||||
ingress_serving = (
|
||||
_http_probe(target_host, target_port)
|
||||
if target_host and target_port
|
||||
else "no ingress port mapped"
|
||||
)
|
||||
return {
|
||||
"command": boot or "image CMD (no boot_command set)",
|
||||
"ports": ports,
|
||||
"container_ip": container_ip,
|
||||
"container_gateway": container_gateway,
|
||||
"ingress_target": (
|
||||
f"{target_host}:{target_port}" if target_host and target_port else ""
|
||||
),
|
||||
"ingress_port": int(instance.get("ingress_port") or 0),
|
||||
"ingress_serving": ingress_serving,
|
||||
"status_ok_for_ingress": instance.get("status") == store.ST_RUNNING,
|
||||
"restart_count": int(instance.get("restart_count") or 0),
|
||||
"exit_code": instance.get("exit_code"),
|
||||
"container_id": (instance.get("container_id") or "")[:12],
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
" retoor <retoor@molodetz.nl>
|
||||
" Self-contained config for the ppy container. No external plugins or managers:
|
||||
" it works out of the box with the stock vim, and the AI edit feature uses only
|
||||
" curl and the PRAVDA_* gateway env that every instance is launched with.
|
||||
" curl and the DEVPLACE_* gateway env that every instance is launched with.
|
||||
|
||||
set nocompatible
|
||||
filetype plugin indent on
|
||||
@ -70,7 +70,7 @@ function! s:GetVisualSelection() abort
|
||||
endfunction
|
||||
|
||||
function! s:GatewayUrl() abort
|
||||
let l:base = substitute($PRAVDA_OPENAI_URL, '/\+$', '', '')
|
||||
let l:base = substitute($DEVPLACE_OPENAI_URL, '/\+$', '', '')
|
||||
if empty(l:base)
|
||||
return 'https://openai.app.molodetz.nl/v1/chat/completions'
|
||||
elseif l:base =~# '/chat/completions$'
|
||||
@ -92,7 +92,7 @@ function! AiEditSelection() abort
|
||||
endif
|
||||
let l:prompt = l:instruction . "\n\nHere is the text:\n" . l:orig
|
||||
\ . "\n\nOutput only the transformed text. No explanations, markdown, or code blocks."
|
||||
let l:api_key = !empty($PRAVDA_API_KEY) ? $PRAVDA_API_KEY : $DEEPSEEK_API_KEY
|
||||
let l:api_key = !empty($DEVPLACE_API_KEY) ? $DEVPLACE_API_KEY : $DEEPSEEK_API_KEY
|
||||
let l:json = '{"model":"deepseek-chat","messages":[{"role":"user","content":' . json_encode(l:prompt) . '}]}'
|
||||
let l:cmd = 'curl -sS -X POST ' . shellescape(s:GatewayUrl())
|
||||
\ . ' -H ' . shellescape('Authorization: Bearer ' . l:api_key)
|
||||
|
||||
119
devplacepy/services/containers/files/.vimrc.bak
Normal file
119
devplacepy/services/containers/files/.vimrc.bak
Normal file
@ -0,0 +1,119 @@
|
||||
" retoor <retoor@molodetz.nl>
|
||||
" Self-contained config for the ppy container. No external plugins or managers:
|
||||
" it works out of the box with the stock vim, and the AI edit feature uses only
|
||||
" curl and the PRAVDA_* gateway env that every instance is launched with.
|
||||
|
||||
set nocompatible
|
||||
filetype plugin indent on
|
||||
syntax on
|
||||
|
||||
set encoding=utf-8
|
||||
set fileencoding=utf-8
|
||||
set termencoding=utf-8
|
||||
set mouse=a
|
||||
set backspace=indent,eol,start
|
||||
set autoindent
|
||||
set smartindent
|
||||
set tabstop=4
|
||||
set shiftwidth=4
|
||||
set expandtab
|
||||
set number
|
||||
set showmatch
|
||||
set showtabline=2
|
||||
set laststatus=2
|
||||
set hidden
|
||||
set incsearch
|
||||
set hlsearch
|
||||
set wildmenu
|
||||
set ttimeoutlen=50
|
||||
|
||||
if has('clipboard')
|
||||
set clipboard=unnamedplus
|
||||
endif
|
||||
|
||||
if !isdirectory(expand('~/.vim/undo'))
|
||||
call mkdir(expand('~/.vim/undo'), 'p')
|
||||
endif
|
||||
set undofile
|
||||
set undodir=~/.vim/undo
|
||||
|
||||
set statusline=%f\ %h%m%r\ %=\ [%{&filetype}]\ [%l,%c]\ %p%%
|
||||
highlight StatusLine cterm=bold ctermfg=15 ctermbg=24
|
||||
highlight StatusLineNC cterm=none ctermfg=250 ctermbg=236
|
||||
highlight ErrorMsg cterm=bold ctermfg=red ctermbg=none
|
||||
|
||||
let mapleader = ","
|
||||
|
||||
inoremap <C-n> <ESC>:tabnext<CR>
|
||||
inoremap <C-p> <ESC>:tabprevious<CR>
|
||||
nnoremap <C-n> :tabnext<CR>
|
||||
nnoremap <C-p> :tabprevious<CR>
|
||||
nnoremap <Tab> :tabnext<CR>
|
||||
nnoremap <C-Tab> :tabprevious<CR>
|
||||
nnoremap <C-e> :tabnew<Space>
|
||||
inoremap <C-e> <ESC>:tabnew<Space>
|
||||
|
||||
if has('autocmd')
|
||||
autocmd BufReadPost * if line("'\"") > 0 && line("'\"") <= line("$") | exe "normal! g'\"" | endif
|
||||
endif
|
||||
|
||||
function! s:GetVisualSelection() abort
|
||||
let [l:line_start, l:col_start] = [line("'<"), col("'<")]
|
||||
let [l:line_end, l:col_end] = [line("'>"), col("'>")]
|
||||
let l:lines = getline(l:line_start, l:line_end)
|
||||
if empty(l:lines)
|
||||
return ''
|
||||
endif
|
||||
let l:lines[-1] = l:lines[-1][: l:col_end - (l:line_start == l:line_end ? 1 : 2)]
|
||||
let l:lines[0] = l:lines[0][l:col_start - 1 :]
|
||||
return join(l:lines, "\n")
|
||||
endfunction
|
||||
|
||||
function! s:GatewayUrl() abort
|
||||
let l:base = substitute($PRAVDA_OPENAI_URL, '/\+$', '', '')
|
||||
if empty(l:base)
|
||||
return 'https://openai.app.molodetz.nl/v1/chat/completions'
|
||||
elseif l:base =~# '/chat/completions$'
|
||||
return l:base
|
||||
endif
|
||||
return l:base . '/chat/completions'
|
||||
endfunction
|
||||
|
||||
function! AiEditSelection() abort
|
||||
let l:instruction = input('AI instruction: ')
|
||||
if empty(l:instruction)
|
||||
echo 'Cancelled.'
|
||||
return
|
||||
endif
|
||||
let l:orig = s:GetVisualSelection()
|
||||
if empty(l:orig)
|
||||
echo 'No selection.'
|
||||
return
|
||||
endif
|
||||
let l:prompt = l:instruction . "\n\nHere is the text:\n" . l:orig
|
||||
\ . "\n\nOutput only the transformed text. No explanations, markdown, or code blocks."
|
||||
let l:api_key = !empty($PRAVDA_API_KEY) ? $PRAVDA_API_KEY : $DEEPSEEK_API_KEY
|
||||
let l:json = '{"model":"deepseek-chat","messages":[{"role":"user","content":' . json_encode(l:prompt) . '}]}'
|
||||
let l:cmd = 'curl -sS -X POST ' . shellescape(s:GatewayUrl())
|
||||
\ . ' -H ' . shellescape('Authorization: Bearer ' . l:api_key)
|
||||
\ . ' -H ' . shellescape('Content-Type: application/json')
|
||||
\ . ' -d ' . shellescape(l:json)
|
||||
let l:reply = system(l:cmd)
|
||||
if v:shell_error
|
||||
echohl ErrorMsg | echom 'AI request failed' | echohl None
|
||||
return
|
||||
endif
|
||||
let l:text = matchstr(l:reply, '"content":\s*"\zs\(.\{-}\)\ze"\s*[,}]')
|
||||
if empty(l:text)
|
||||
echohl ErrorMsg | echom 'No content in AI response' | echohl None
|
||||
return
|
||||
endif
|
||||
let l:text = substitute(l:text, '\\n', "\n", 'g')
|
||||
let l:text = substitute(l:text, '\\"', '"', 'g')
|
||||
let l:text = substitute(l:text, '\\t', "\t", 'g')
|
||||
normal! gv
|
||||
normal! c
|
||||
call feedkeys(l:text, 'n')
|
||||
endfunction
|
||||
|
||||
xnoremap <silent> <Leader>a :<C-u>call AiEditSelection()<CR>
|
||||
@ -49,14 +49,14 @@ from urllib.parse import quote, unquote, urlencode, urlparse, urlsplit, urlunspl
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _resolve_devplace_url() -> str:
|
||||
base = os.environ.get("PRAVDA_BASE_URL", "").strip().rstrip("/")
|
||||
base = os.environ.get("DEVPLACE_BASE_URL", "").strip().rstrip("/")
|
||||
if base:
|
||||
return base
|
||||
return os.environ.get("DEVPLACE_URL", "https://devplace.net").strip().rstrip("/")
|
||||
|
||||
|
||||
def _resolve_llm_endpoint() -> str:
|
||||
base = os.environ.get("PRAVDA_OPENAI_URL", "").strip().rstrip("/")
|
||||
base = os.environ.get("DEVPLACE_OPENAI_URL", "").strip().rstrip("/")
|
||||
if not base:
|
||||
base = "https://openai.app.molodetz.nl/v1"
|
||||
return base if base.endswith("/chat/completions") else base + "/chat/completions"
|
||||
@ -64,7 +64,7 @@ def _resolve_llm_endpoint() -> str:
|
||||
|
||||
DEVPLACE_URL = _resolve_devplace_url()
|
||||
DEVPLACE_API_KEY = (
|
||||
os.environ.get("PRAVDA_API_KEY")
|
||||
os.environ.get("DEVPLACE_API_KEY")
|
||||
or os.environ.get("DEVPLACE_API_KEY")
|
||||
or "019ea58c-fae0-7112-8025-e629a54104a4"
|
||||
)
|
||||
@ -81,7 +81,7 @@ LLM_ENDPOINT = _resolve_llm_endpoint()
|
||||
LLM_BASE_URL = LLM_ENDPOINT.rsplit("/chat/completions", 1)[0]
|
||||
MODEL = "molodetz"
|
||||
LLM_API_KEY = str(
|
||||
os.environ.get("PRAVDA_API_KEY")
|
||||
os.environ.get("DEVPLACE_API_KEY")
|
||||
or os.environ.get("LLM_API_KEY")
|
||||
or ""
|
||||
)
|
||||
|
||||
2940
devplacepy/services/containers/files/bot.py.bak
Normal file
2940
devplacepy/services/containers/files/bot.py.bak
Normal file
File diff suppressed because it is too large
Load Diff
@ -659,7 +659,7 @@ from typing import Any, Callable, Optional
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
def _resolve_llm_endpoint() -> str:
|
||||
base = os.environ.get("PRAVDA_OPENAI_URL", "").strip().rstrip("/")
|
||||
base = os.environ.get("DEVPLACE_OPENAI_URL", "").strip().rstrip("/")
|
||||
if not base:
|
||||
base = "https://openai.app.molodetz.nl/v1"
|
||||
return base if base.endswith("/chat/completions") else base + "/chat/completions"
|
||||
@ -669,7 +669,7 @@ LLM_ENDPOINT = _resolve_llm_endpoint()
|
||||
LLM_BASE_URL = LLM_ENDPOINT.rsplit("/chat/completions", 1)[0]
|
||||
MODEL = "molodetz"
|
||||
API_KEY = (
|
||||
os.environ.get("PRAVDA_API_KEY")
|
||||
os.environ.get("DEVPLACE_API_KEY")
|
||||
or os.environ.get("LLM_API_KEY")
|
||||
or str(uuid.uuid4())
|
||||
)
|
||||
|
||||
2337
devplacepy/services/containers/files/d.py.bak
Executable file
2337
devplacepy/services/containers/files/d.py.bak
Executable file
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@ -41,14 +41,14 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode, urljoin, urlparse
|
||||
|
||||
def _resolve_api_url() -> str:
|
||||
base = os.environ.get("PRAVDA_OPENAI_URL", "").strip().rstrip("/")
|
||||
base = os.environ.get("DEVPLACE_OPENAI_URL", "").strip().rstrip("/")
|
||||
if not base:
|
||||
return "https://openai.app.molodetz.nl/v1/chat/completions"
|
||||
return base if base.endswith("/chat/completions") else base + "/chat/completions"
|
||||
|
||||
|
||||
API_URL = _resolve_api_url()
|
||||
API_KEY = os.environ.get("PRAVDA_API_KEY") or os.environ.get("DEEPSEEK_API_KEY") or ""
|
||||
API_KEY = os.environ.get("DEVPLACE_API_KEY") or os.environ.get("DEEPSEEK_API_KEY") or ""
|
||||
RSEARCH_BASE_URL = "https://rsearch.app.molodetz.nl"
|
||||
RSEARCH_MODES = ("search", "chat", "describe", "health")
|
||||
RSEARCH_MAX_BYTES = 8 * 1024 * 1024
|
||||
@ -1644,7 +1644,7 @@ async def describe_image(prompt: str, image_path: str):
|
||||
|
||||
api_key = API_KEY
|
||||
if not api_key:
|
||||
return json.dumps({"status": "error", "error": "PRAVDA_API_KEY or DEEPSEEK_API_KEY missing"})
|
||||
return json.dumps({"status": "error", "error": "DEVPLACE_API_KEY or DEEPSEEK_API_KEY missing"})
|
||||
|
||||
payload = {
|
||||
"model": DEFAULT_MODEL,
|
||||
@ -1811,7 +1811,7 @@ async def delegate(task: str, allowed_tools: Optional[list] = None):
|
||||
"""
|
||||
api_key = API_KEY
|
||||
if not api_key:
|
||||
return json.dumps({"status": "error", "error": "PRAVDA_API_KEY or DEEPSEEK_API_KEY missing"})
|
||||
return json.dumps({"status": "error", "error": "DEVPLACE_API_KEY or DEEPSEEK_API_KEY missing"})
|
||||
if allowed_tools:
|
||||
sub_payloads = [
|
||||
t for t in get_tool_payloads()
|
||||
@ -2567,7 +2567,7 @@ async def amain(headed: bool = False, prompt: str = "", timeout: float = 0) -> N
|
||||
|
||||
api_key = API_KEY
|
||||
if not api_key:
|
||||
md.print("**Error:** `PRAVDA_API_KEY` or `DEEPSEEK_API_KEY` missing.")
|
||||
md.print("**Error:** `DEVPLACE_API_KEY` or `DEEPSEEK_API_KEY` missing.")
|
||||
sys.exit(1)
|
||||
|
||||
clone_mode = os.environ.get("MAK_CLONE_MODE") == "1"
|
||||
|
||||
2669
devplacepy/services/containers/files/pagent.bak
Executable file
2669
devplacepy/services/containers/files/pagent.bak
Executable file
File diff suppressed because it is too large
Load Diff
@ -108,4 +108,58 @@ TOOLS_ACTIONS: tuple[Action, ...] = (
|
||||
params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="isslop",
|
||||
method="POST",
|
||||
path="/tools/isslop/run",
|
||||
summary="Classify a repository or website as AI slop or human work",
|
||||
description=(
|
||||
"Queues a background AI Usage Analyzer job and returns {uid, status_url, report_url}. "
|
||||
"Poll the status with isslop_status until status is 'completed', then share the "
|
||||
"authenticity grade, category, human/AI split and report_url. Accepts http(s), "
|
||||
"git and ssh source URLs."
|
||||
),
|
||||
params=(
|
||||
body("url", "Repository or website URL to classify.", required=True),
|
||||
),
|
||||
requires_auth=True,
|
||||
),
|
||||
Action(
|
||||
name="isslop_status",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}",
|
||||
summary="Check a AI usage analysis and obtain its verdict once finished",
|
||||
description=(
|
||||
"Returns the analysis status. When status is 'completed', grade, category, "
|
||||
"human_percent, ai_percent and report_url are populated; while 'pending' or "
|
||||
"'running', poll again shortly."
|
||||
),
|
||||
params=(path("uid", "Analysis uid returned by isslop."),),
|
||||
requires_auth=True,
|
||||
),
|
||||
Action(
|
||||
name="isslop_report",
|
||||
method="GET",
|
||||
path="/tools/isslop/{uid}/report",
|
||||
summary="Read a finished AI usage analysis report",
|
||||
description=(
|
||||
"Returns the full report for a finished analysis: authenticity grade, human/AI "
|
||||
"split, markdown findings, per-file scores with signals, the image review and the "
|
||||
"embeddable badge snippets. Use it after isslop_status reports status 'completed'."
|
||||
),
|
||||
params=(path("uid", "Analysis uid returned by isslop."),),
|
||||
requires_auth=True,
|
||||
),
|
||||
Action(
|
||||
name="isslop_list",
|
||||
method="GET",
|
||||
path="/tools/isslop/list",
|
||||
summary="List the user's AI usage analyses",
|
||||
description=(
|
||||
"Returns the signed-in user's analysis history, newest first, each with its grade, "
|
||||
"category, status and report_url."
|
||||
),
|
||||
params=(),
|
||||
requires_auth=True,
|
||||
),
|
||||
)
|
||||
|
||||
@ -55,7 +55,7 @@ CONTAINER_ACTIONS: tuple[Action, ...] = (
|
||||
),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"Optional DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user, which is always pravda (uid 1000).",
|
||||
"Optional DevPlace user uid whose identity and API key are injected (DEVPLACE_API_KEY, DEVPLACE_USER_UID). Does NOT change the container OS user, which is always pravda (uid 1000).",
|
||||
),
|
||||
arg(
|
||||
"start_on_boot",
|
||||
@ -118,7 +118,7 @@ CONTAINER_ACTIONS: tuple[Action, ...] = (
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID); pass empty to clear. Does NOT change the container OS user (always pravda, uid 1000).",
|
||||
"DevPlace user uid whose identity and API key are injected (DEVPLACE_API_KEY, DEVPLACE_USER_UID); pass empty to clear. Does NOT change the container OS user (always pravda, uid 1000).",
|
||||
),
|
||||
arg("boot_language", "Boot source language: 'none', 'python', or 'bash'."),
|
||||
arg("boot_script", "Boot source code body run on launch."),
|
||||
|
||||
200
devplacepy/services/devii/actions/container_actions.py.bak
Normal file
200
devplacepy/services/devii/actions/container_actions.py.bak
Normal file
@ -0,0 +1,200 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
|
||||
def arg(
|
||||
name: str, description: str, required: bool = False, kind: str = "string"
|
||||
) -> Param:
|
||||
return Param(
|
||||
name=name,
|
||||
location="body",
|
||||
description=description,
|
||||
required=required,
|
||||
type=kind,
|
||||
)
|
||||
|
||||
|
||||
SLUG = arg(
|
||||
"project_slug",
|
||||
"Project slug or uid that owns the container resources.",
|
||||
required=True,
|
||||
)
|
||||
|
||||
CONTAINER_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="container_list_instances",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="List a project's container instances and their status",
|
||||
params=(SLUG,),
|
||||
),
|
||||
Action(
|
||||
name="container_create_instance",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Create and start a container instance (runs the shared ppy image with the project files mounted at /app)",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("name", "Instance name.", required=True),
|
||||
arg(
|
||||
"boot_command", "Optional command to run on boot, e.g. 'python app.py'."
|
||||
),
|
||||
arg(
|
||||
"boot_language",
|
||||
"Optional boot source language: 'none', 'python', or 'bash'. When set with boot_script, the script is materialized into /app and run on launch (takes precedence over boot_command).",
|
||||
),
|
||||
arg(
|
||||
"boot_script",
|
||||
"Optional boot source code (the body of the python or bash script) run on launch when boot_language is python or bash.",
|
||||
),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"Optional DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user, which is always pravda (uid 1000).",
|
||||
),
|
||||
arg(
|
||||
"start_on_boot",
|
||||
"Force this instance to running whenever the container service starts ('true' or 'false', default false).",
|
||||
),
|
||||
arg("restart_policy", "never, always, on-failure, or unless-stopped."),
|
||||
arg("env", "Optional env vars as KEY=VALUE lines."),
|
||||
arg(
|
||||
"ports",
|
||||
"Port maps per line or comma separated. Use a bare container port (e.g. '8899') to auto-assign a unique host port above 20000, or 'host:container' to pin one.",
|
||||
),
|
||||
arg("cpu_limit", "Optional CPU limit, e.g. 1 or 1.5."),
|
||||
arg("mem_limit", "Optional memory limit, e.g. 512m or 1g."),
|
||||
arg("autostart", "Start immediately ('true' or 'false', default true)."),
|
||||
arg(
|
||||
"ingress_slug",
|
||||
"Optional public ingress slug; the service is then reachable at /p/<slug>.",
|
||||
),
|
||||
arg(
|
||||
"ingress_port",
|
||||
"Container port to publish at /p/<slug> (must be one of the mapped ports).",
|
||||
kind="integer",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_instance_action",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Control an instance: start, stop, restart, pause, resume, delete, or sync",
|
||||
description="sync imports the container /app workspace back into the project files.",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"action",
|
||||
"start, stop, restart, pause, resume, delete, or sync.",
|
||||
required=True,
|
||||
),
|
||||
arg(
|
||||
"confirm",
|
||||
"Required only for action=delete: set true ONLY after the user has explicitly "
|
||||
"confirmed destroying the instance. Leave unset otherwise; the delete is refused "
|
||||
"until you pass confirm=true.",
|
||||
kind="boolean",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_configure_instance",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Update an instance's run-as user, boot language/script/command, restart policy, start-on-boot flag, and resource limits",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"run_as_uid",
|
||||
"DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID); pass empty to clear. Does NOT change the container OS user (always pravda, uid 1000).",
|
||||
),
|
||||
arg("boot_language", "Boot source language: 'none', 'python', or 'bash'."),
|
||||
arg("boot_script", "Boot source code body run on launch."),
|
||||
arg("boot_command", "Fallback boot command used when no boot_script is set."),
|
||||
arg("restart_policy", "never, always, on-failure, or unless-stopped."),
|
||||
arg(
|
||||
"start_on_boot",
|
||||
"Force running on container-service start ('true' or 'false').",
|
||||
),
|
||||
arg("cpu_limit", "CPU limit, e.g. 1 or 1.5."),
|
||||
arg("mem_limit", "Memory limit, e.g. 512m or 1g."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_logs",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="Read the recent logs of a running instance",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg("tail", "Number of log lines (default 200).", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_exec",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Run a one-shot command inside a running instance and return its output. The command runs in /app (the project workspace) by default, so never prefix it with 'cd /app'",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg(
|
||||
"command",
|
||||
"Command to run, e.g. 'git clone ... && ls'. Runs in /app already; do not prepend 'cd /app'.",
|
||||
required=True,
|
||||
),
|
||||
arg(
|
||||
"confirm",
|
||||
"Required only when the command is destructive (rm, dd, truncate, drop, etc.): set "
|
||||
"true ONLY after the user has explicitly confirmed. Leave unset otherwise; a "
|
||||
"destructive command is refused until you pass confirm=true.",
|
||||
kind="boolean",
|
||||
),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="container_stats",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
summary="Get aggregated resource and runtime statistics for an instance",
|
||||
params=(SLUG, arg("instance", "Instance name, slug, or uid.", required=True)),
|
||||
),
|
||||
Action(
|
||||
name="container_schedule",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
handler="container",
|
||||
requires_admin=True,
|
||||
summary="Schedule a start or stop of an instance (cron, interval, or one-time)",
|
||||
params=(
|
||||
SLUG,
|
||||
arg("instance", "Instance name, slug, or uid.", required=True),
|
||||
arg("action", "start or stop.", required=True),
|
||||
arg("kind", "once, interval, or cron.", required=True),
|
||||
arg("cron", "Cron expression for kind=cron, e.g. '0 2 * * *'."),
|
||||
arg("run_at", "ISO time for kind=once, e.g. 2026-06-15T02:00:00."),
|
||||
arg("every_seconds", "Interval seconds for kind=interval.", kind="integer"),
|
||||
),
|
||||
),
|
||||
)
|
||||
@ -287,7 +287,7 @@ class Dispatcher:
|
||||
self._rsearch = RsearchController(settings, owner_kind, owner_id)
|
||||
from ..container import ContainerController
|
||||
|
||||
self._container = ContainerController(client)
|
||||
self._container = ContainerController(client, owner_id=owner_id)
|
||||
from ..customization import CustomizationController
|
||||
|
||||
self._customization = CustomizationController(owner_kind, owner_id)
|
||||
|
||||
@ -4,7 +4,7 @@ import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.database import get_table, resolve_by_slug
|
||||
from devplacepy.database import get_table, get_users_by_uids, resolve_by_slug
|
||||
from devplacepy.services.containers import api, store
|
||||
from devplacepy.services.containers.api import ContainerError
|
||||
from devplacepy.services.containers.runtime import get_backend
|
||||
@ -15,8 +15,9 @@ logger = logging.getLogger("devii.container")
|
||||
|
||||
|
||||
class ContainerController:
|
||||
def __init__(self, client: Any = None) -> None:
|
||||
def __init__(self, client: Any = None, owner_id: str = "") -> None:
|
||||
self._client = client
|
||||
self._owner_id = owner_id
|
||||
|
||||
def _ingress_url(self, instance: dict):
|
||||
slug = instance.get("ingress_slug")
|
||||
@ -29,11 +30,19 @@ class ContainerController:
|
||||
|
||||
def _actor_user(self) -> dict:
|
||||
username = getattr(self._client, "username", None)
|
||||
if username:
|
||||
if username and username != "api-key":
|
||||
user = get_table("users").find_one(username=username)
|
||||
if user:
|
||||
return user
|
||||
return {"uid": "admin", "username": username or "admin"}
|
||||
if self._owner_id:
|
||||
user = get_users_by_uids([self._owner_id]).get(self._owner_id)
|
||||
if user:
|
||||
return user
|
||||
return {
|
||||
"uid": "admin",
|
||||
"username": username or "admin",
|
||||
"role": "Admin",
|
||||
}
|
||||
|
||||
def _project(self, arguments: dict) -> dict:
|
||||
from devplacepy.content import can_view_project
|
||||
|
||||
@ -2,12 +2,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
from .. import economy
|
||||
from .common import GameError, _farms, _iso, _now, _plots, _update_farm
|
||||
|
||||
|
||||
_leaderboard_cache = TTLCache(ttl=15, max_size=8)
|
||||
|
||||
|
||||
def get_farm(user_uid: str) -> dict | None:
|
||||
return _farms().find_one(user_uid=user_uid)
|
||||
|
||||
@ -95,6 +99,9 @@ def upgrade_ci(user: dict) -> dict:
|
||||
|
||||
|
||||
def leaderboard(limit: int = 25) -> list[dict]:
|
||||
cached = _leaderboard_cache.get(f"top:{limit}")
|
||||
if cached is not None:
|
||||
return cached
|
||||
farms = sorted(
|
||||
_farms().find(),
|
||||
key=economy.farm_score,
|
||||
@ -122,4 +129,5 @@ def leaderboard(limit: int = 25) -> list[dict]:
|
||||
"score": economy.farm_score(farm),
|
||||
}
|
||||
)
|
||||
_leaderboard_cache.set(f"top:{limit}", entries)
|
||||
return entries
|
||||
|
||||
1
devplacepy/services/jobs/isslop/__init__.py
Normal file
1
devplacepy/services/jobs/isslop/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
1
devplacepy/services/jobs/isslop/acquisition/__init__.py
Normal file
1
devplacepy/services/jobs/isslop/acquisition/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
313
devplacepy/services/jobs/isslop/acquisition/browser.py
Normal file
313
devplacepy/services/jobs/isslop/acquisition/browser.py
Normal file
@ -0,0 +1,313 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from types import TracebackType
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.acquisition.domcapture import (
|
||||
DOM_EXTRACT_SCRIPT,
|
||||
DOM_INIT_SCRIPT,
|
||||
DomSnapshot,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
CONSOLE_SAMPLE_CAP,
|
||||
DOM_CAPTURE_TIMEOUT_MS,
|
||||
HEADER_VALUE_CAP,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSER_USER_AGENT: str = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
BROWSER_VIEWPORT: dict[str, int] = {"width": 1440, "height": 900}
|
||||
BROWSER_LOCALE: str = "en-US"
|
||||
BROWSER_TIMEZONE: str = "America/New_York"
|
||||
|
||||
LAUNCH_ARGS: list[str] = [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-gpu",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-extensions",
|
||||
]
|
||||
|
||||
NAVIGATION_TIMEOUT_MS: int = 30000
|
||||
RENDER_SETTLE_MS: int = 1200
|
||||
FETCH_TIMEOUT_MS: int = 25000
|
||||
BROWSER_CLOSE_TIMEOUT_S: float = 15.0
|
||||
LAUNCH_TIMEOUT_S: float = 60.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RenderResult:
|
||||
url: str
|
||||
final_url: str
|
||||
status: int
|
||||
html: str
|
||||
content_type: str
|
||||
ok: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
url: str
|
||||
status: int
|
||||
body: bytes
|
||||
content_type: str
|
||||
ok: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def browser_available() -> bool:
|
||||
try:
|
||||
import playwright.async_api # noqa: F401
|
||||
import playwright_stealth # noqa: F401
|
||||
except ImportError as error:
|
||||
logger.info("Stealth browser unavailable, will use HTTP client: %s", error)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class StealthBrowser:
|
||||
def __init__(self, tab_concurrency: int = 4) -> None:
|
||||
self._tab_concurrency = max(1, tab_concurrency)
|
||||
self._stealth_manager = None
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._tab_semaphore = asyncio.Semaphore(self._tab_concurrency)
|
||||
|
||||
async def __aenter__(self) -> "StealthBrowser":
|
||||
await self._start()
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: Optional[type[BaseException]],
|
||||
exc: Optional[BaseException],
|
||||
traceback: Optional[TracebackType],
|
||||
) -> None:
|
||||
await self.aclose()
|
||||
|
||||
async def _start(self) -> None:
|
||||
from playwright.async_api import async_playwright
|
||||
from playwright_stealth import Stealth
|
||||
|
||||
stealth = Stealth()
|
||||
self._stealth_manager = stealth.use_async(async_playwright())
|
||||
self._playwright = await asyncio.wait_for(self._stealth_manager.__aenter__(), timeout=LAUNCH_TIMEOUT_S)
|
||||
self._browser = await self._launch_browser()
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent=BROWSER_USER_AGENT,
|
||||
viewport=BROWSER_VIEWPORT,
|
||||
locale=BROWSER_LOCALE,
|
||||
timezone_id=BROWSER_TIMEZONE,
|
||||
ignore_https_errors=True,
|
||||
java_script_enabled=True,
|
||||
)
|
||||
self._context.set_default_navigation_timeout(NAVIGATION_TIMEOUT_MS)
|
||||
self._context.set_default_timeout(NAVIGATION_TIMEOUT_MS)
|
||||
logger.info("Stealth browser ready (tab concurrency %d)", self._tab_concurrency)
|
||||
|
||||
async def _launch_browser(self):
|
||||
errors: list[str] = []
|
||||
for channel in ("chrome", None):
|
||||
try:
|
||||
kwargs: dict[str, object] = {"headless": True, "args": LAUNCH_ARGS}
|
||||
if channel:
|
||||
kwargs["channel"] = channel
|
||||
browser = await asyncio.wait_for(
|
||||
self._playwright.chromium.launch(**kwargs), timeout=LAUNCH_TIMEOUT_S
|
||||
)
|
||||
logger.info("Chromium launched (channel=%s)", channel or "bundled")
|
||||
return browser
|
||||
except Exception as error: # noqa: BLE001 - launch failures are heterogeneous, we retry the next channel
|
||||
errors.append(f"{channel or 'bundled'}: {error}")
|
||||
logger.warning("Browser launch failed on channel %s: %s", channel or "bundled", error)
|
||||
raise RuntimeError(f"Could not launch any Chromium build: {'; '.join(errors)}")
|
||||
|
||||
async def _goto_and_settle(self, page: object, url: str) -> object:
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=NAVIGATION_TIMEOUT_MS)
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=RENDER_SETTLE_MS * 3)
|
||||
except PlaywrightTimeout:
|
||||
await page.wait_for_timeout(RENDER_SETTLE_MS)
|
||||
return response
|
||||
|
||||
async def _build_render_result(self, page: object, response: object, url: str) -> RenderResult:
|
||||
html = await page.content()
|
||||
status = response.status if response else 0
|
||||
content_type = ""
|
||||
if response is not None:
|
||||
headers = await response.all_headers()
|
||||
content_type = headers.get("content-type", "")
|
||||
final_url = page.url
|
||||
return RenderResult(
|
||||
url=url,
|
||||
final_url=final_url,
|
||||
status=status,
|
||||
html=html,
|
||||
content_type=content_type or "text/html",
|
||||
ok=bool(html) and status < 400,
|
||||
)
|
||||
|
||||
async def render(self, url: str) -> RenderResult:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
async with self._tab_semaphore:
|
||||
page = None
|
||||
try:
|
||||
page = await self._context.new_page()
|
||||
response = await self._goto_and_settle(page, url)
|
||||
return await self._build_render_result(page, response, url)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Render failed for %s: %s", url, error)
|
||||
return RenderResult(url, url, 0, "", "", False, str(error))
|
||||
finally:
|
||||
if page is not None:
|
||||
try:
|
||||
await page.close()
|
||||
except Exception as error: # noqa: BLE001 - a failed tab close must never abort the crawl
|
||||
logger.debug("Tab close error for %s: %s", url, error)
|
||||
|
||||
async def _evaluate_dom_and_screenshot(self, page: object) -> tuple[dict, Optional[bytes]]:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
dom: dict = {}
|
||||
try:
|
||||
dom = await page.evaluate(DOM_EXTRACT_SCRIPT)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("DOM evaluate failed: %s", error)
|
||||
dom = {}
|
||||
screenshot_bytes: Optional[bytes] = None
|
||||
try:
|
||||
screenshot_bytes = await page.screenshot(full_page=False)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Screenshot capture failed: %s", error)
|
||||
screenshot_bytes = None
|
||||
return dom if isinstance(dom, dict) else {}, screenshot_bytes
|
||||
|
||||
async def capture(self, url: str) -> DomSnapshot:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeout
|
||||
|
||||
async with self._tab_semaphore:
|
||||
page = None
|
||||
try:
|
||||
page = await self._context.new_page()
|
||||
console_warnings: list[str] = []
|
||||
console_errors: list[str] = []
|
||||
|
||||
def _on_console(message: object) -> None:
|
||||
try:
|
||||
kind = message.type
|
||||
text = message.text[:HEADER_VALUE_CAP]
|
||||
except Exception as error:
|
||||
logger.debug("Console handler error: %s", error)
|
||||
return
|
||||
if kind == "warning" and len(console_warnings) < CONSOLE_SAMPLE_CAP:
|
||||
console_warnings.append(text)
|
||||
elif kind == "error" and len(console_errors) < CONSOLE_SAMPLE_CAP:
|
||||
console_errors.append(text)
|
||||
|
||||
page.on("console", _on_console)
|
||||
|
||||
response_headers: dict[str, str] = {}
|
||||
resource_hosts: list[str] = []
|
||||
seen_hosts: set[str] = set()
|
||||
captured_main = False
|
||||
|
||||
def _on_response(response: object) -> None:
|
||||
nonlocal captured_main
|
||||
try:
|
||||
hostname = urlparse(response.url).hostname
|
||||
if hostname and hostname not in seen_hosts:
|
||||
seen_hosts.add(hostname)
|
||||
resource_hosts.append(hostname)
|
||||
if not captured_main and response.frame == page.main_frame:
|
||||
captured_main = True
|
||||
for key, value in response.headers.items():
|
||||
response_headers[key] = str(value)[:HEADER_VALUE_CAP]
|
||||
except Exception as error:
|
||||
logger.debug("Response handler error: %s", error)
|
||||
|
||||
page.on("response", _on_response)
|
||||
await page.add_init_script(DOM_INIT_SCRIPT)
|
||||
response = await self._goto_and_settle(page, url)
|
||||
render = await self._build_render_result(page, response, url)
|
||||
dom: dict = {}
|
||||
screenshot_bytes: Optional[bytes] = None
|
||||
try:
|
||||
dom, screenshot_bytes = await asyncio.wait_for(
|
||||
self._evaluate_dom_and_screenshot(page),
|
||||
timeout=DOM_CAPTURE_TIMEOUT_MS / 1000,
|
||||
)
|
||||
except asyncio.TimeoutError as error:
|
||||
logger.warning("DOM capture timed out for %s: %s", url, error)
|
||||
return DomSnapshot(
|
||||
render=render,
|
||||
dom=dom,
|
||||
console_warnings=console_warnings,
|
||||
console_errors=console_errors,
|
||||
response_headers=response_headers,
|
||||
resource_hosts=resource_hosts,
|
||||
screenshot_bytes=screenshot_bytes,
|
||||
)
|
||||
except (PlaywrightTimeout, PlaywrightError) as error:
|
||||
logger.warning("Capture failed for %s: %s", url, error)
|
||||
return DomSnapshot(render=RenderResult(url, url, 0, "", "", False, str(error)))
|
||||
finally:
|
||||
if page is not None:
|
||||
try:
|
||||
await page.close()
|
||||
except Exception as error:
|
||||
logger.debug("Tab close error for %s: %s", url, error)
|
||||
|
||||
async def fetch(self, url: str, max_bytes: int) -> FetchResult:
|
||||
from playwright.async_api import Error as PlaywrightError
|
||||
|
||||
try:
|
||||
response = await self._context.request.get(url, timeout=FETCH_TIMEOUT_MS)
|
||||
body = await response.body()
|
||||
if len(body) > max_bytes:
|
||||
body = body[:max_bytes]
|
||||
content_type = response.headers.get("content-type", "")
|
||||
return FetchResult(url, response.status, body, content_type, response.status < 400)
|
||||
except (PlaywrightError, asyncio.TimeoutError) as error:
|
||||
logger.warning("Fetch failed for %s: %s", url, error)
|
||||
return FetchResult(url, 0, b"", "", False, str(error))
|
||||
|
||||
async def aclose(self) -> None:
|
||||
for closer, label in (
|
||||
(self._context, "context"),
|
||||
(self._browser, "browser"),
|
||||
):
|
||||
if closer is not None:
|
||||
try:
|
||||
await asyncio.wait_for(closer.close(), timeout=BROWSER_CLOSE_TIMEOUT_S)
|
||||
except Exception as error: # noqa: BLE001 - cleanup must be exhaustive
|
||||
logger.warning("Error closing %s: %s", label, error)
|
||||
self._context = None
|
||||
self._browser = None
|
||||
if self._stealth_manager is not None:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stealth_manager.__aexit__(None, None, None), timeout=BROWSER_CLOSE_TIMEOUT_S
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - playwright teardown must not leak
|
||||
logger.warning("Error stopping playwright: %s", error)
|
||||
self._stealth_manager = None
|
||||
self._playwright = None
|
||||
logger.info("Stealth browser closed")
|
||||
212
devplacepy/services/jobs/isslop/acquisition/domcapture.py
Normal file
212
devplacepy/services/jobs/isslop/acquisition/domcapture.py
Normal file
@ -0,0 +1,212 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import RenderResult
|
||||
|
||||
DOM_INIT_SCRIPT = """
|
||||
window.__isslop = {};
|
||||
"""
|
||||
|
||||
DOM_EXTRACT_SCRIPT = r"""
|
||||
() => {
|
||||
const safe = (fn, fallback) => { try { return fn(); } catch (e) { return fallback; } };
|
||||
const metaByName = (name) => safe(() => {
|
||||
const el = document.querySelector(`meta[name="${name}"]`);
|
||||
return el ? (el.getAttribute('content') || '') : '';
|
||||
}, '');
|
||||
const metaByProperty = (name) => safe(() => {
|
||||
const el = document.querySelector(`meta[property="${name}"]`);
|
||||
return el ? (el.getAttribute('content') || '') : '';
|
||||
}, '');
|
||||
const linkHref = (rel) => safe(() => {
|
||||
const el = document.querySelector(`link[rel="${rel}"]`);
|
||||
return el ? (el.getAttribute('href') || '') : '';
|
||||
}, '');
|
||||
const meta = {
|
||||
title: safe(() => (document.title || '').trim(), ''),
|
||||
generator: metaByName('generator'),
|
||||
description: metaByName('description'),
|
||||
ogImage: metaByProperty('og:image'),
|
||||
canonical: linkHref('canonical'),
|
||||
htmlLang: safe(() => document.documentElement.getAttribute('lang') || '', ''),
|
||||
favicon: safe(() => linkHref('icon') || linkHref('shortcut icon'), ''),
|
||||
};
|
||||
const badgeSelectors = [
|
||||
'#lovable-badge',
|
||||
'.replit-badge',
|
||||
'#base44-badge',
|
||||
'#__framer-badge-container',
|
||||
'[data-radix-root]',
|
||||
'iframe[src*="claudeusercontent.com"]',
|
||||
];
|
||||
const badgeHits = {};
|
||||
badgeSelectors.forEach((sel) => {
|
||||
badgeHits[sel] = safe(() => !!document.querySelector(sel), false);
|
||||
});
|
||||
const headings = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('h1,h2,h3,h4,h5,h6').forEach((h) => {
|
||||
if (headings.length >= 60) return;
|
||||
headings.push({
|
||||
level: parseInt(h.tagName.substring(1), 10),
|
||||
text: (h.textContent || '').trim().slice(0, 160),
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const landmarks = {
|
||||
main: safe(() => document.querySelectorAll('main').length, 0),
|
||||
nav: safe(() => document.querySelectorAll('nav').length, 0),
|
||||
header: safe(() => document.querySelectorAll('header').length, 0),
|
||||
footer: safe(() => document.querySelectorAll('footer').length, 0),
|
||||
article: safe(() => document.querySelectorAll('article').length, 0),
|
||||
};
|
||||
const images = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('img').forEach((img) => {
|
||||
if (images.length >= 100) return;
|
||||
images.push({
|
||||
src: img.getAttribute('src') || '',
|
||||
alt: img.getAttribute('alt') || '',
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const classCounts = {};
|
||||
safe(() => {
|
||||
document.querySelectorAll('[class]').forEach((el) => {
|
||||
const raw = el.getAttribute('class') || '';
|
||||
raw.split(/\s+/).filter(Boolean).forEach((token) => {
|
||||
classCounts[token] = (classCounts[token] || 0) + 1;
|
||||
});
|
||||
});
|
||||
}, null);
|
||||
const classNames = {};
|
||||
Object.keys(classCounts)
|
||||
.sort((a, b) => classCounts[b] - classCounts[a])
|
||||
.slice(0, 40)
|
||||
.forEach((token) => { classNames[token] = classCounts[token]; });
|
||||
const colorSample = (tag, el) => safe(() => {
|
||||
const style = getComputedStyle(el);
|
||||
return {
|
||||
tag: tag,
|
||||
backgroundImage: style.backgroundImage || '',
|
||||
backgroundColor: style.backgroundColor || '',
|
||||
color: style.color || '',
|
||||
boxShadow: style.boxShadow || '',
|
||||
};
|
||||
}, null);
|
||||
const colors = [];
|
||||
safe(() => {
|
||||
if (document.body) {
|
||||
const sample = colorSample('body', document.body);
|
||||
if (sample) colors.push(sample);
|
||||
}
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('h1,h2')).slice(0, 6).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('button, a.btn, [class*="btn"]')).slice(0, 10).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('nav, header')).slice(0, 4).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('[class*="hero"]')).slice(0, 4).forEach((el) => {
|
||||
const sample = colorSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) colors.push(sample);
|
||||
});
|
||||
}, null);
|
||||
const fontSample = (tag, el) => safe(() => {
|
||||
const style = getComputedStyle(el);
|
||||
return {
|
||||
tag: tag,
|
||||
fontFamily: style.fontFamily || '',
|
||||
fontWeight: style.fontWeight || '',
|
||||
letterSpacing: style.letterSpacing || '',
|
||||
fontSize: style.fontSize || '',
|
||||
lineHeight: style.lineHeight || '',
|
||||
};
|
||||
}, null);
|
||||
const fonts = [];
|
||||
safe(() => {
|
||||
if (document.body) {
|
||||
const sample = fontSample('body', document.body);
|
||||
if (sample) fonts.push(sample);
|
||||
}
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('h1,h2,h3')).slice(0, 10).forEach((el) => {
|
||||
const sample = fontSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) fonts.push(sample);
|
||||
});
|
||||
}, null);
|
||||
safe(() => {
|
||||
Array.from(document.querySelectorAll('p, button')).slice(0, 8).forEach((el) => {
|
||||
if (fonts.length >= 20) return;
|
||||
const sample = fontSample(el.tagName.toLowerCase(), el);
|
||||
if (sample) fonts.push(sample);
|
||||
});
|
||||
}, null);
|
||||
const scripts = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('script[src]').forEach((s) => {
|
||||
if (scripts.length >= 60) return;
|
||||
scripts.push(s.getAttribute('src') || '');
|
||||
});
|
||||
}, null);
|
||||
const links = [];
|
||||
safe(() => {
|
||||
document.querySelectorAll('link[href]').forEach((l) => {
|
||||
if (links.length >= 60) return;
|
||||
links.push(l.getAttribute('href') || '');
|
||||
});
|
||||
}, null);
|
||||
const wordCount = safe(() => {
|
||||
const text = document.body ? (document.body.innerText || '') : '';
|
||||
return text.split(/\s+/).filter(Boolean).length;
|
||||
}, 0);
|
||||
const deadAnchorCount = safe(() => document.querySelectorAll('a[href="#"]').length, 0);
|
||||
const jsonldCount = safe(() => document.querySelectorAll('script[type="application/ld+json"]').length, 0);
|
||||
const robotsMeta = metaByName('robots');
|
||||
return {
|
||||
meta: meta,
|
||||
badgeHits: badgeHits,
|
||||
headings: headings.slice(0, 60),
|
||||
landmarks: landmarks,
|
||||
images: images.slice(0, 100),
|
||||
classNames: classNames,
|
||||
colors: colors.slice(0, 30),
|
||||
fonts: fonts.slice(0, 20),
|
||||
jsonld: jsonldCount,
|
||||
scripts: scripts.slice(0, 60),
|
||||
links: links.slice(0, 60),
|
||||
wordCount: wordCount,
|
||||
deadAnchorCount: deadAnchorCount,
|
||||
robotsMeta: robotsMeta,
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomSnapshot:
|
||||
render: RenderResult
|
||||
dom: dict = field(default_factory=dict)
|
||||
console_warnings: list[str] = field(default_factory=list)
|
||||
console_errors: list[str] = field(default_factory=list)
|
||||
response_headers: dict[str, str] = field(default_factory=dict)
|
||||
resource_hosts: list[str] = field(default_factory=list)
|
||||
screenshot_bytes: bytes | None = None
|
||||
163
devplacepy/services/jobs/isslop/acquisition/git.py
Normal file
163
devplacepy/services/jobs/isslop/acquisition/git.py
Normal file
@ -0,0 +1,163 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import directory_size_bytes
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
GIT_CLONE_TIMEOUT_SECONDS,
|
||||
GIT_SIZE_LIMIT_BYTES,
|
||||
GIT_SIZE_POLL_SECONDS,
|
||||
WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RepositoryTooLargeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CloneFailedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SizePreflight:
|
||||
known: bool
|
||||
size_bytes: int
|
||||
origin: str
|
||||
|
||||
|
||||
def _owner_repo(url: str) -> Optional[tuple[str, str, str]]:
|
||||
match = re.match(r"^git@([\w.-]+):(.+)$", url)
|
||||
if match:
|
||||
host, path = match.group(1), match.group(2)
|
||||
else:
|
||||
parsed = urlparse(url)
|
||||
host, path = parsed.hostname or "", parsed.path
|
||||
parts = [part for part in path.strip("/").removesuffix(".git").split("/") if part]
|
||||
if len(parts) < 2 or not host:
|
||||
return None
|
||||
return host, parts[0], parts[1]
|
||||
|
||||
|
||||
async def preflight_size(url: str) -> SizePreflight:
|
||||
located = _owner_repo(url)
|
||||
if located is None:
|
||||
return SizePreflight(known=False, size_bytes=0, origin="unparseable")
|
||||
host, owner, repo = located
|
||||
candidates: list[tuple[str, str]] = []
|
||||
if host == "github.com":
|
||||
candidates.append((f"https://api.github.com/repos/{owner}/{repo}", "github"))
|
||||
else:
|
||||
candidates.append((f"https://{host}/api/v1/repos/{owner}/{repo}", "gitea"))
|
||||
async with stealth_async_client(timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS, follow_redirects=True) as client:
|
||||
for api_url, origin in candidates:
|
||||
try:
|
||||
response = await client.get(api_url, headers={"accept": "application/json"})
|
||||
except httpx.HTTPError as error:
|
||||
logger.info("Size preflight unavailable via %s: %s", origin, error)
|
||||
continue
|
||||
if response.status_code != 200:
|
||||
logger.info("Size preflight %s returned HTTP %d", origin, response.status_code)
|
||||
continue
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
continue
|
||||
size_kb = body.get("size")
|
||||
if isinstance(size_kb, (int, float)) and size_kb > 0:
|
||||
size_bytes = int(size_kb) * 1024
|
||||
logger.info("Preflight size via %s: %d bytes", origin, size_bytes)
|
||||
return SizePreflight(known=True, size_bytes=size_bytes, origin=origin)
|
||||
return SizePreflight(known=False, size_bytes=0, origin="unknown")
|
||||
|
||||
|
||||
async def clone_repository(url: str, workspace: Path, size_limit: int = GIT_SIZE_LIMIT_BYTES) -> AsyncIterator[str]:
|
||||
preflight = await preflight_size(url)
|
||||
if preflight.known:
|
||||
yield f"Repository size reported by {preflight.origin}: {preflight.size_bytes / (1024 * 1024):.1f} MB"
|
||||
if preflight.size_bytes > size_limit:
|
||||
raise RepositoryTooLargeError(
|
||||
f"Repository is {preflight.size_bytes / (1024 ** 3):.2f} GB which exceeds the 3 GB limit"
|
||||
)
|
||||
else:
|
||||
yield "Repository size not determinable up front, monitoring during clone"
|
||||
|
||||
argv = [
|
||||
"git",
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--single-branch",
|
||||
"--no-tags",
|
||||
url,
|
||||
str(workspace),
|
||||
]
|
||||
yield f"Cloning with depth 1: {url}"
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
env={"GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "true", "PATH": "/usr/bin:/bin:/usr/local/bin"},
|
||||
)
|
||||
|
||||
aborted_reason: list[str] = []
|
||||
|
||||
async def monitor() -> None:
|
||||
while proc.returncode is None:
|
||||
await asyncio.sleep(GIT_SIZE_POLL_SECONDS)
|
||||
if not workspace.exists():
|
||||
continue
|
||||
size = await asyncio.to_thread(directory_size_bytes, workspace)
|
||||
logger.debug("Clone size check: %d bytes", size)
|
||||
if size > size_limit:
|
||||
aborted_reason.append(f"Clone exceeded 3 GB limit at {size / (1024 ** 3):.2f} GB, cancelled")
|
||||
proc.kill()
|
||||
return
|
||||
|
||||
monitor_task = asyncio.create_task(monitor())
|
||||
output_lines: list[str] = []
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
while True:
|
||||
try:
|
||||
line = await asyncio.wait_for(proc.stdout.readline(), timeout=GIT_CLONE_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
raise CloneFailedError("Clone timed out")
|
||||
if not line:
|
||||
break
|
||||
text = line.decode("utf-8", errors="replace").strip()
|
||||
if text:
|
||||
output_lines.append(text)
|
||||
yield f"git: {text}"
|
||||
await proc.wait()
|
||||
finally:
|
||||
monitor_task.cancel()
|
||||
try:
|
||||
await monitor_task
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("Clone size monitor stopped")
|
||||
|
||||
if aborted_reason:
|
||||
raise RepositoryTooLargeError(aborted_reason[0])
|
||||
if proc.returncode != 0:
|
||||
tail = " | ".join(output_lines[-4:])
|
||||
raise CloneFailedError(f"git clone failed with code {proc.returncode}: {tail}")
|
||||
final_size = await asyncio.to_thread(directory_size_bytes, workspace)
|
||||
if final_size > size_limit:
|
||||
raise RepositoryTooLargeError(f"Repository is {final_size / (1024 ** 3):.2f} GB which exceeds the 3 GB limit")
|
||||
yield f"Clone complete: {final_size / (1024 * 1024):.1f} MB on disk"
|
||||
98
devplacepy/services/jobs/isslop/acquisition/source.py
Normal file
98
devplacepy/services/jobs/isslop/acquisition/source.py
Normal file
@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.config import GIT_PROBE_TIMEOUT_SECONDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND_GIT: str = "git"
|
||||
KIND_WEBSITE: str = "website"
|
||||
|
||||
GIT_URL_PATTERN = re.compile(r"(\.git$|^git://|^ssh://git@|^git@)", re.IGNORECASE)
|
||||
GIT_HOST_HINTS: tuple[str, ...] = ("github.com", "gitlab.com", "bitbucket.org", "codeberg.org", "gitea.")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceResolution:
|
||||
url: str
|
||||
kind: str
|
||||
probe_output: str
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
url = url.strip()
|
||||
if re.match(r"^git@[\w.-]+:", url):
|
||||
return url
|
||||
if not re.match(r"^[a-z]+://", url, re.IGNORECASE):
|
||||
url = f"https://{url}"
|
||||
return url
|
||||
|
||||
|
||||
def is_private_host(url: str) -> bool:
|
||||
parsed = urlparse(url if "://" in url else f"ssh://{url}")
|
||||
host = parsed.hostname or ""
|
||||
if not host:
|
||||
return True
|
||||
if host in ("localhost",):
|
||||
return True
|
||||
try:
|
||||
address = ipaddress.ip_address(host)
|
||||
return address.is_private or address.is_loopback or address.is_link_local or address.is_reserved
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
resolved = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
for entry in resolved:
|
||||
candidate = ipaddress.ip_address(entry[4][0])
|
||||
if candidate.is_private or candidate.is_loopback or candidate.is_link_local:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def probe_git(url: str) -> tuple[bool, str]:
|
||||
argv = ["git", "ls-remote", "--heads", "--exit-code", url]
|
||||
logger.info("Probing for git repository: %s", url)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env={"GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "true", "PATH": "/usr/bin:/bin:/usr/local/bin"},
|
||||
)
|
||||
except OSError as error:
|
||||
logger.error("git probe spawn failed: %s", error)
|
||||
return False, str(error)
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=GIT_PROBE_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
logger.warning("git probe timed out for %s", url)
|
||||
return False, "probe timeout"
|
||||
output = stdout.decode("utf-8", errors="replace")
|
||||
errors = stderr.decode("utf-8", errors="replace")
|
||||
is_git = proc.returncode == 0 and bool(output.strip())
|
||||
logger.info("git probe %s: is_git=%s refs=%d", url, is_git, len(output.splitlines()))
|
||||
return is_git, output if is_git else errors[:500]
|
||||
|
||||
|
||||
async def resolve_source(url: str) -> SourceResolution:
|
||||
normalized = normalize_url(url)
|
||||
hinted = bool(GIT_URL_PATTERN.search(normalized)) or any(hint in normalized.lower() for hint in GIT_HOST_HINTS)
|
||||
is_git, probe_output = await probe_git(normalized)
|
||||
if is_git:
|
||||
return SourceResolution(url=normalized, kind=KIND_GIT, probe_output=probe_output)
|
||||
if hinted:
|
||||
logger.info("URL hinted git but probe failed, treating as website: %s", normalized)
|
||||
return SourceResolution(url=normalized, kind=KIND_WEBSITE, probe_output=probe_output)
|
||||
315
devplacepy/services/jobs/isslop/acquisition/website.py
Normal file
315
devplacepy/services/jobs/isslop/acquisition/website.py
Normal file
@ -0,0 +1,315 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
from urllib.parse import urldefrag, urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.net_guard import BlockedAddressError, guard_public_url
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
from devplacepy.services.jobs.isslop.acquisition.domcapture import DomSnapshot
|
||||
from devplacepy.services.jobs.isslop.acquisition.workspace import safe_relative_path
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
DOM_ANALYSIS_MAX_PAGES,
|
||||
WEBSITE_MAX_DEPTH,
|
||||
WEBSITE_MAX_FILE_BYTES,
|
||||
WEBSITE_MAX_FILES,
|
||||
WEBSITE_MAX_PAGES,
|
||||
WEBSITE_MAX_TOTAL_BYTES,
|
||||
WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
||||
TEXT_CONTENT_HINTS: tuple[str, ...] = ("text/", "javascript", "json", "xml", "svg", "css")
|
||||
SKIP_SCHEMES: tuple[str, ...] = ("mailto:", "tel:", "javascript:", "data:", "#")
|
||||
|
||||
|
||||
class LinkExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.page_links: list[str] = []
|
||||
self.asset_links: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
mapping = dict(attrs)
|
||||
if tag == "a" and mapping.get("href"):
|
||||
self.page_links.append(str(mapping["href"]))
|
||||
if tag in ("script", "img", "source", "iframe") and mapping.get("src"):
|
||||
self.asset_links.append(str(mapping["src"]))
|
||||
if tag == "link" and mapping.get("href"):
|
||||
self.asset_links.append(str(mapping["href"]))
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlState:
|
||||
pages_fetched: int = 0
|
||||
files_saved: int = 0
|
||||
bytes_saved: int = 0
|
||||
visited: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
def _clean_url(base: str, href: str) -> str:
|
||||
joined = urljoin(base, href)
|
||||
return urldefrag(joined).url
|
||||
|
||||
|
||||
def _local_path_for(workspace: Path, url: str) -> Path:
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or "/"
|
||||
if path.endswith("/"):
|
||||
path += "index.html"
|
||||
name = path.lstrip("/") or "index.html"
|
||||
if "." not in name.split("/")[-1]:
|
||||
name += ".html"
|
||||
if parsed.query:
|
||||
name += "-" + "".join(ch for ch in parsed.query if ch.isalnum())[:40]
|
||||
return safe_relative_path(workspace, name)
|
||||
|
||||
|
||||
def _is_text_response(content_type: str) -> bool:
|
||||
lowered = content_type.lower()
|
||||
return any(hint in lowered for hint in TEXT_CONTENT_HINTS)
|
||||
|
||||
|
||||
def _same_site(candidate: str, root_host: str | None) -> bool:
|
||||
host = urlparse(candidate).hostname
|
||||
if host is None or root_host is None:
|
||||
return False
|
||||
return host == root_host or host.endswith("." + root_host) or root_host.endswith("." + host)
|
||||
|
||||
|
||||
def _extract_links(base: str, html: str, root_host: str | None) -> tuple[list[str], list[str]]:
|
||||
extractor = LinkExtractor()
|
||||
try:
|
||||
extractor.feed(html)
|
||||
except ValueError as error:
|
||||
logger.debug("HTML parse issue on %s: %s", base, error)
|
||||
pages: list[str] = []
|
||||
assets: list[str] = []
|
||||
for asset in extractor.asset_links:
|
||||
cleaned = _clean_url(base, asset)
|
||||
if _same_site(cleaned, root_host):
|
||||
assets.append(cleaned)
|
||||
for link in extractor.page_links:
|
||||
cleaned = _clean_url(base, link)
|
||||
if _same_site(cleaned, root_host):
|
||||
pages.append(cleaned)
|
||||
return pages, assets
|
||||
|
||||
|
||||
def _save_bytes(workspace: Path, url: str, body: bytes) -> Path | None:
|
||||
try:
|
||||
target = _local_path_for(workspace, url)
|
||||
except ValueError as error:
|
||||
logger.warning("Rejected path for %s: %s", url, error)
|
||||
return None
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
return target
|
||||
|
||||
|
||||
async def crawl_website(
|
||||
url: str,
|
||||
workspace: Path,
|
||||
*,
|
||||
dom_sink: list[DomSnapshot] | None = None,
|
||||
allow_private: bool = False,
|
||||
) -> AsyncIterator[str]:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import browser_available
|
||||
|
||||
if browser_available():
|
||||
try:
|
||||
async for progress in _crawl_with_browser(
|
||||
url, workspace, dom_sink=dom_sink, allow_private=allow_private
|
||||
):
|
||||
yield progress
|
||||
return
|
||||
except RuntimeError as error:
|
||||
logger.warning("Stealth browser crawl failed, falling back to HTTP client: %s", error)
|
||||
yield f"Stealth browser unavailable ({error}); falling back to HTTP client"
|
||||
async for progress in _crawl_with_http(url, workspace):
|
||||
yield progress
|
||||
|
||||
|
||||
async def _crawl_with_browser(
|
||||
url: str,
|
||||
workspace: Path,
|
||||
*,
|
||||
dom_sink: list[DomSnapshot] | None = None,
|
||||
allow_private: bool = False,
|
||||
) -> AsyncIterator[str]:
|
||||
from devplacepy.services.jobs.isslop.acquisition.browser import StealthBrowser
|
||||
|
||||
root_host = urlparse(url).hostname
|
||||
state = CrawlState()
|
||||
yield f"Launching stealth browser for {url} (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
|
||||
async with StealthBrowser(tab_concurrency=4) as browser:
|
||||
yield "Stealth browser ready; rendering pages with an undetectable Chromium fingerprint"
|
||||
current_level: list[str] = [url]
|
||||
for depth in range(WEBSITE_MAX_DEPTH + 1):
|
||||
pending = [
|
||||
candidate
|
||||
for candidate in current_level
|
||||
if candidate not in state.visited and _same_site(candidate, root_host)
|
||||
and not any(candidate.lower().startswith(scheme) for scheme in SKIP_SCHEMES)
|
||||
]
|
||||
if not pending or state.files_saved >= WEBSITE_MAX_FILES:
|
||||
break
|
||||
pending = pending[: max(0, WEBSITE_MAX_PAGES - state.pages_fetched)]
|
||||
safe_candidates: list[str] = []
|
||||
for candidate in pending:
|
||||
try:
|
||||
await guard_public_url(candidate, allow_private=allow_private)
|
||||
except BlockedAddressError:
|
||||
state.visited.add(candidate)
|
||||
yield f"Blocked private/local address: {candidate}"
|
||||
continue
|
||||
safe_candidates.append(candidate)
|
||||
for candidate in safe_candidates:
|
||||
state.visited.add(candidate)
|
||||
|
||||
async def _render_or_capture(index: int, candidate: str) -> object:
|
||||
if depth == 0 and dom_sink is not None and index < DOM_ANALYSIS_MAX_PAGES:
|
||||
snapshot = await browser.capture(candidate)
|
||||
dom_sink.append(snapshot)
|
||||
return snapshot.render
|
||||
return await browser.render(candidate)
|
||||
|
||||
renders = await asyncio.gather(
|
||||
*(_render_or_capture(index, candidate) for index, candidate in enumerate(safe_candidates))
|
||||
)
|
||||
asset_urls: set[str] = set()
|
||||
next_level: list[str] = []
|
||||
for render in renders:
|
||||
if not render.ok:
|
||||
yield f"Render failed: {render.url} ({render.error or 'no content'})"
|
||||
continue
|
||||
body = render.html.encode("utf-8", errors="replace")[:WEBSITE_MAX_FILE_BYTES]
|
||||
target = _save_bytes(workspace, render.final_url, body)
|
||||
if target is None:
|
||||
continue
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(body)
|
||||
state.pages_fetched += 1
|
||||
yield f"Rendered {target.relative_to(workspace)} ({len(body)} bytes, DOM after JS execution)"
|
||||
pages, assets = _extract_links(render.final_url, render.html, root_host)
|
||||
for asset in assets:
|
||||
if asset not in state.visited:
|
||||
asset_urls.add(asset)
|
||||
next_level.extend(pages)
|
||||
async for progress in _download_assets(browser, workspace, sorted(asset_urls), state):
|
||||
yield progress
|
||||
current_level = next_level
|
||||
if state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
|
||||
yield f"Byte budget reached: {state.bytes_saved / 1024:.1f} KB"
|
||||
break
|
||||
if state.files_saved == 0:
|
||||
raise RuntimeError(f"Stealth crawl produced no files for {url}")
|
||||
yield (
|
||||
f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, "
|
||||
f"{state.pages_fetched} rendered pages"
|
||||
)
|
||||
|
||||
|
||||
async def _download_assets(
|
||||
browser: object,
|
||||
workspace: Path,
|
||||
asset_urls: list[str],
|
||||
state: CrawlState,
|
||||
) -> AsyncIterator[str]:
|
||||
budget = min(len(asset_urls), max(0, WEBSITE_MAX_FILES - state.files_saved))
|
||||
if budget <= 0:
|
||||
return
|
||||
targets = [asset for asset in asset_urls if asset not in state.visited][:budget]
|
||||
for asset in targets:
|
||||
state.visited.add(asset)
|
||||
results = await asyncio.gather(*(browser.fetch(asset, WEBSITE_MAX_FILE_BYTES) for asset in targets))
|
||||
saved = 0
|
||||
for result in results:
|
||||
if not result.ok or not result.body:
|
||||
continue
|
||||
target = _save_bytes(workspace, result.url, result.body)
|
||||
if target is None:
|
||||
continue
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(result.body)
|
||||
saved += 1
|
||||
if saved:
|
||||
yield f"Downloaded {saved} assets concurrently (scripts, styles, sources)"
|
||||
|
||||
|
||||
async def _crawl_with_http(url: str, workspace: Path) -> AsyncIterator[str]:
|
||||
root = urlparse(url)
|
||||
state = CrawlState()
|
||||
queue: list[tuple[str, int]] = [(url, 0)]
|
||||
headers = {"user-agent": USER_AGENT, "accept": "*/*"}
|
||||
yield f"Crawling website {url} with HTTP client (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
|
||||
async with stealth_async_client(
|
||||
timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
) as client:
|
||||
while queue:
|
||||
if state.files_saved >= WEBSITE_MAX_FILES or state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
|
||||
yield f"Crawl limits reached: {state.files_saved} files, {state.bytes_saved} bytes"
|
||||
break
|
||||
current, depth = queue.pop(0)
|
||||
if current in state.visited:
|
||||
continue
|
||||
state.visited.add(current)
|
||||
parsed = urlparse(current)
|
||||
if parsed.hostname != root.hostname:
|
||||
continue
|
||||
if any(current.lower().startswith(scheme) for scheme in SKIP_SCHEMES):
|
||||
continue
|
||||
try:
|
||||
response = await client.get(current)
|
||||
except httpx.HTTPError as error:
|
||||
logger.warning("Fetch failed %s: %s", current, error)
|
||||
yield f"Fetch failed: {current} ({error})"
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
yield f"Skipped {current}: HTTP {response.status_code}"
|
||||
continue
|
||||
content_type = response.headers.get("content-type", "")
|
||||
body = response.content[:WEBSITE_MAX_FILE_BYTES]
|
||||
try:
|
||||
target = _local_path_for(workspace, current)
|
||||
except ValueError as error:
|
||||
logger.warning("Rejected path for %s: %s", current, error)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
state.files_saved += 1
|
||||
state.bytes_saved += len(body)
|
||||
yield f"Saved {target.relative_to(workspace)} ({len(body)} bytes, {content_type.split(';')[0] or 'unknown type'})"
|
||||
|
||||
is_page = "text/html" in content_type.lower()
|
||||
if is_page:
|
||||
state.pages_fetched += 1
|
||||
if depth < WEBSITE_MAX_DEPTH and state.pages_fetched < WEBSITE_MAX_PAGES:
|
||||
extractor = LinkExtractor()
|
||||
try:
|
||||
extractor.feed(body.decode("utf-8", errors="replace"))
|
||||
except ValueError as error:
|
||||
logger.debug("HTML parse issue on %s: %s", current, error)
|
||||
for asset in extractor.asset_links:
|
||||
cleaned = _clean_url(current, asset)
|
||||
if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
|
||||
queue.append((cleaned, depth + 1))
|
||||
for link in extractor.page_links:
|
||||
cleaned = _clean_url(current, link)
|
||||
if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
|
||||
queue.append((cleaned, depth + 1))
|
||||
if state.files_saved == 0:
|
||||
raise RuntimeError(f"Website download produced no files for {url}")
|
||||
yield f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, {state.pages_fetched} pages"
|
||||
85
devplacepy/services/jobs/isslop/acquisition/workspace.py
Normal file
85
devplacepy/services/jobs/isslop/acquisition/workspace.py
Normal file
@ -0,0 +1,85 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SLUG_MAX_LENGTH: int = 80
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
value = re.sub(r"^[a-z]+://", "", value.strip().lower())
|
||||
value = re.sub(r"[^a-z0-9._-]+", "-", value).strip("-._")
|
||||
if not value:
|
||||
value = "source"
|
||||
return value[:SLUG_MAX_LENGTH]
|
||||
|
||||
|
||||
def workspace_for(workspaces_dir: Path, source_url: str, analysis_uid: str) -> Path:
|
||||
digest = hashlib.sha256(source_url.encode("utf-8")).hexdigest()[:10]
|
||||
slug = slugify(source_url)
|
||||
workspace = (workspaces_dir / f"{slug}-{digest}-{analysis_uid}").resolve()
|
||||
if workspaces_dir.resolve() != workspace.parent:
|
||||
raise ValueError(f"Workspace path escapes workspaces directory: {workspace}")
|
||||
logger.info("Workspace resolved: %s", workspace)
|
||||
return workspace
|
||||
|
||||
|
||||
def reset_workspace(workspace: Path) -> Path:
|
||||
if workspace.exists():
|
||||
logger.info("Removing existing workspace %s", workspace)
|
||||
shutil.rmtree(workspace)
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
return workspace
|
||||
|
||||
|
||||
def safe_relative_path(workspace: Path, candidate: str) -> Path:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._/-]+", "-", candidate).strip("/")
|
||||
cleaned = re.sub(r"\.\.+", ".", cleaned)
|
||||
target = (workspace / cleaned).resolve()
|
||||
if workspace.resolve() not in target.parents and target != workspace.resolve():
|
||||
raise ValueError(f"Refusing path traversal: {candidate}")
|
||||
return target
|
||||
|
||||
|
||||
def remove_workspace(workspace: Path) -> bool:
|
||||
if not workspace.exists():
|
||||
return False
|
||||
try:
|
||||
shutil.rmtree(workspace)
|
||||
except OSError as error:
|
||||
logger.error("Workspace removal failed for %s: %s", workspace, error)
|
||||
return False
|
||||
logger.info("Workspace removed: %s", workspace)
|
||||
return True
|
||||
|
||||
|
||||
def directory_size_bytes(path: Path) -> int:
|
||||
total = 0
|
||||
for entry in path.rglob("*"):
|
||||
if entry.is_file() and not entry.is_symlink():
|
||||
try:
|
||||
total += entry.stat().st_size
|
||||
except OSError as error:
|
||||
logger.debug("Size stat failed for %s: %s", entry, error)
|
||||
return total
|
||||
|
||||
|
||||
def content_hash(workspace: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
entries: list[tuple[str, Path]] = []
|
||||
for entry in workspace.rglob("*"):
|
||||
if entry.is_file() and not entry.is_symlink() and ".git" not in entry.parts:
|
||||
entries.append((str(entry.relative_to(workspace)), entry))
|
||||
for relative, entry in sorted(entries):
|
||||
digest.update(relative.encode("utf-8"))
|
||||
try:
|
||||
digest.update(hashlib.sha256(entry.read_bytes()).digest())
|
||||
except OSError as error:
|
||||
logger.debug("Hash read failed for %s: %s", entry, error)
|
||||
return digest.hexdigest()
|
||||
1
devplacepy/services/jobs/isslop/agent/__init__.py
Normal file
1
devplacepy/services/jobs/isslop/agent/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
111
devplacepy/services/jobs/isslop/agent/classifier.py
Normal file
111
devplacepy/services/jobs/isslop/agent/classifier.py
Normal file
@ -0,0 +1,111 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError, extract_json_object
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import (
|
||||
CATEGORY_HUMAN_CLEAN,
|
||||
CATEGORY_HUMAN_MESSY,
|
||||
CATEGORY_SLOP,
|
||||
CATEGORY_SOPHISTICATED,
|
||||
CATEGORY_UNCERTAIN,
|
||||
FileScore,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_STRONG
|
||||
from devplacepy.services.jobs.isslop.config import AI_EXCERPT_CHARS, AI_SAMPLE_LIMIT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_CATEGORIES: frozenset[str] = frozenset(
|
||||
{CATEGORY_SLOP, CATEGORY_SOPHISTICATED, CATEGORY_HUMAN_CLEAN, CATEGORY_HUMAN_MESSY, CATEGORY_UNCERTAIN}
|
||||
)
|
||||
|
||||
SYSTEM_PROMPT: str = (
|
||||
"You are a deterministic source-code provenance and quality auditor. "
|
||||
"You classify code on two independent axes: origin (was it AI-generated) and "
|
||||
"curation quality (was it reviewed, integrated and production-worthy). "
|
||||
"The categories are: ai-slop (AI origin, low curation: the model's defaults shipped as-is), "
|
||||
"sophisticated-ai (AI origin, high curation: the model typed, the engineer decided), "
|
||||
"human-clean and human-messy (code no model would write that way), uncertain. "
|
||||
"Human messiness is not slop. Clean AI-assisted code is not slop. "
|
||||
"Judge only from the evidence given. Respond with a single JSON object and nothing else, using exactly these keys: "
|
||||
'{"origin_score": <0-100 integer, likelihood of AI generation>, '
|
||||
'"quality_deficit": <0-100 integer, density of anti-patterns and missing curation>, '
|
||||
'"ai_probability": <0-100 integer>, '
|
||||
'"category": <one of ai-slop|sophisticated-ai|human-clean|human-messy|uncertain>, '
|
||||
'"reasoning": <two to four factual sentences citing concrete evidence>, '
|
||||
'"notable_signals": [<up to five short strings>]}'
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AiVerdict:
|
||||
path: str
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
ai_probability: float
|
||||
category: str
|
||||
reasoning: str
|
||||
notable_signals: list[str]
|
||||
|
||||
|
||||
def select_samples(scores: list[FileScore], limit: int = AI_SAMPLE_LIMIT) -> list[FileScore]:
|
||||
def interest(entry: FileScore) -> tuple[float, str]:
|
||||
strong = sum(1 for signal in entry.signals if signal.severity == SEVERITY_STRONG)
|
||||
return (-(entry.sloc * (1.0 + strong) * entry.criticality), entry.relative)
|
||||
|
||||
ranked = sorted((entry for entry in scores if entry.sloc >= 10), key=interest)
|
||||
return ranked[:limit]
|
||||
|
||||
|
||||
def _clamp(value: object, fallback: float) -> float:
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return float(min(100.0, max(0.0, value)))
|
||||
return fallback
|
||||
|
||||
|
||||
async def classify_file(
|
||||
llm: LlmClient,
|
||||
score: FileScore,
|
||||
excerpt: str,
|
||||
) -> Optional[AiVerdict]:
|
||||
signal_lines = "\n".join(
|
||||
f"- [{signal.severity}] {signal.code} line {signal.line}: {signal.title}"
|
||||
for signal in score.signals[:15]
|
||||
) or "- none detected"
|
||||
user = (
|
||||
f"File: {score.relative}\n"
|
||||
f"Language: {score.language}\n"
|
||||
f"Static origin score: {score.origin_score}\n"
|
||||
f"Static quality deficit: {score.quality_deficit}\n"
|
||||
f"Static signals:\n{signal_lines}\n\n"
|
||||
f"Source excerpt (may be truncated):\n```\n{excerpt[:AI_EXCERPT_CHARS]}\n```"
|
||||
)
|
||||
try:
|
||||
answer = await llm.complete(SYSTEM_PROMPT, user)
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("AI classification unavailable for %s: %s", score.relative, error)
|
||||
return None
|
||||
parsed = extract_json_object(answer)
|
||||
if parsed is None:
|
||||
logger.warning("AI classification returned unparseable output for %s", score.relative)
|
||||
return None
|
||||
category = str(parsed.get("category", CATEGORY_UNCERTAIN)).strip().lower()
|
||||
if category not in VALID_CATEGORIES:
|
||||
category = CATEGORY_UNCERTAIN
|
||||
signals_field = parsed.get("notable_signals")
|
||||
notable = [str(item)[:120] for item in signals_field[:5]] if isinstance(signals_field, list) else []
|
||||
verdict = AiVerdict(
|
||||
path=score.relative,
|
||||
origin_score=_clamp(parsed.get("origin_score"), score.origin_score),
|
||||
quality_deficit=_clamp(parsed.get("quality_deficit"), score.quality_deficit),
|
||||
ai_probability=_clamp(parsed.get("ai_probability"), score.origin_score),
|
||||
category=category,
|
||||
reasoning=str(parsed.get("reasoning", ""))[:1200],
|
||||
notable_signals=notable,
|
||||
)
|
||||
logger.info("AI verdict for %s: %s (ai %d%%)", verdict.path, verdict.category, int(verdict.ai_probability))
|
||||
return verdict
|
||||
159
devplacepy/services/jobs/isslop/agent/llm.py
Normal file
159
devplacepy/services/jobs/isslop/agent/llm.py
Normal file
@ -0,0 +1,159 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.stealth import stealth_async_client
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
LLM_MAX_RETRIES,
|
||||
LLM_RETRY_BACKOFF_SECONDS,
|
||||
LLM_TEMPERATURE,
|
||||
LLM_TIMEOUT_SECONDS,
|
||||
WorkerSettings,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LlmUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LlmClient:
|
||||
def __init__(self, settings: WorkerSettings) -> None:
|
||||
self._endpoint = settings.llm_endpoint
|
||||
self._model = settings.llm_model
|
||||
self._api_key = settings.llm_api_key
|
||||
self._review_enabled = settings.ai_review_enabled
|
||||
self._vision_enabled = settings.image_review_enabled
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def active_backend_name(self) -> str:
|
||||
return self._model
|
||||
|
||||
@property
|
||||
def review_available(self) -> bool:
|
||||
return self._review_enabled and bool(self._endpoint and self._api_key)
|
||||
|
||||
@property
|
||||
def vision_available(self) -> bool:
|
||||
return self._vision_enabled and bool(self._endpoint and self._api_key)
|
||||
|
||||
async def _http(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = stealth_async_client(timeout=httpx.Timeout(LLM_TIMEOUT_SECONDS))
|
||||
return self._client
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _call(self, messages: list[dict[str, Any]], timeout: Optional[float] = None) -> str:
|
||||
if not self._endpoint or not self._api_key:
|
||||
raise LlmUnavailableError("AI gateway not configured")
|
||||
client = await self._http()
|
||||
payload = {
|
||||
"model": self._model,
|
||||
"messages": messages,
|
||||
"temperature": LLM_TEMPERATURE,
|
||||
}
|
||||
headers = {"authorization": f"Bearer {self._api_key}", "content-type": "application/json"}
|
||||
request_timeout = httpx.Timeout(timeout) if timeout else None
|
||||
last_error = "unknown"
|
||||
for attempt in range(1, LLM_MAX_RETRIES + 1):
|
||||
try:
|
||||
response = await client.post(
|
||||
self._endpoint, json=payload, headers=headers, timeout=request_timeout
|
||||
)
|
||||
except httpx.HTTPError as error:
|
||||
last_error = f"{type(error).__name__}: {error}"
|
||||
logger.warning("Gateway attempt %d transport failure: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
last_error = f"HTTP {response.status_code}: {response.text[:300]}"
|
||||
logger.warning("Gateway attempt %d failed: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
continue
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
last_error = "invalid JSON envelope"
|
||||
continue
|
||||
choices = body.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
content = choices[0].get("message", {}).get("content")
|
||||
if isinstance(content, str) and content.strip():
|
||||
return content
|
||||
last_error = json.dumps(body)[:300]
|
||||
logger.warning("Gateway attempt %d empty completion: %s", attempt, last_error)
|
||||
await asyncio.sleep(LLM_RETRY_BACKOFF_SECONDS * attempt)
|
||||
raise LlmUnavailableError(f"AI gateway failed after {LLM_MAX_RETRIES} attempts: {last_error}")
|
||||
|
||||
async def complete(self, system: str, user: str) -> str:
|
||||
if not self.review_available:
|
||||
raise LlmUnavailableError("AI review disabled")
|
||||
return await self._call(
|
||||
[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
)
|
||||
|
||||
async def describe_image(self, prompt: str, image_data_url: str, timeout: float) -> str:
|
||||
if not self.vision_available:
|
||||
raise LlmUnavailableError("vision backend unavailable")
|
||||
return await self._call(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": image_data_url}},
|
||||
],
|
||||
}
|
||||
],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def extract_json_object(text: str) -> Optional[dict[str, Any]]:
|
||||
start = text.find("{")
|
||||
while start != -1:
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for position in range(start, len(text)):
|
||||
char = text[position]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
candidate = text[start:position + 1]
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
break
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
break
|
||||
start = text.find("{", start + 1)
|
||||
return None
|
||||
246
devplacepy/services/jobs/isslop/agent/reporter.py
Normal file
246
devplacepy/services/jobs/isslop/agent/reporter.py
Normal file
@ -0,0 +1,246 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.classifier import AiVerdict
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError
|
||||
from devplacepy.services.jobs.isslop.agent.vision import ImageVerdict
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.aggregate import DomEvidence
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import FileScore, RepoScores
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_STRONG
|
||||
from devplacepy.services.jobs.isslop.analysis.templates import TemplateEvidence
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPORT_SYSTEM_PROMPT: str = (
|
||||
"You are a technical auditor writing the final report of a source-code AI usage classification. "
|
||||
"Write factual, professional markdown. No emoji, no marketing language, no hedging filler. "
|
||||
"Structure: '# AI Usage Classification Report' title, then sections 'Verdict', 'Scores', "
|
||||
"'Evidence', 'Notable Files', 'Image Review' (only if image data is present), 'Methodology Notes', 'Caveats'. "
|
||||
"Cite concrete files and signals from the data, and wrap EVERY file path you mention in backticks; "
|
||||
"when citing a specific line, write it as `path:line` (like `src/libs/Env.ts:12`) so the platform can "
|
||||
"link the reference to the exact annotated source line. "
|
||||
"Present the result as calibrated guidance, not accusation. "
|
||||
"Category definitions: ai-slop ships the model's defaults untouched; sophisticated-ai shows AI steered by "
|
||||
"an engineer who knows what they are doing; human is code no model would write that way. "
|
||||
"When template_provenance markers are present, explain that the project ships a starter template's "
|
||||
"defaults and name the template evidence; shipping scaffold defaults counts against authenticity. "
|
||||
"A near-certain unmodified starter template IS ai-slop by definition: the defaults were shipped as-is, "
|
||||
"and clean scaffold code does not change that - the curation belongs to the template author, not the "
|
||||
"presenter. Never call an untouched template sophisticated-ai. "
|
||||
"When dom_evidence.detected_builder is present, lead the Verdict section with a line stating the site was "
|
||||
"detected as built with that no-code builder, treating the rendered-page fingerprint match as near-certain "
|
||||
"evidence. "
|
||||
"Keep it under 900 words. Output only markdown."
|
||||
)
|
||||
|
||||
|
||||
def _summary_payload(
|
||||
source_url: str,
|
||||
source_kind: str,
|
||||
scores: RepoScores,
|
||||
files: list[FileScore],
|
||||
verdicts: list[AiVerdict],
|
||||
excluded_count: int,
|
||||
image_verdicts: list[ImageVerdict],
|
||||
image_stats: dict[str, Any],
|
||||
template: TemplateEvidence,
|
||||
dom_evidence: DomEvidence | None,
|
||||
) -> dict[str, Any]:
|
||||
worst = sorted(files, key=lambda entry: (-entry.quality_deficit, entry.relative))[:10]
|
||||
strongest = [
|
||||
{
|
||||
"path": entry.relative,
|
||||
"origin": entry.origin_score,
|
||||
"quality_deficit": entry.quality_deficit,
|
||||
"category": entry.category,
|
||||
"signals": [signal.to_dict() for signal in entry.signals if signal.severity == SEVERITY_STRONG][:4],
|
||||
}
|
||||
for entry in worst
|
||||
]
|
||||
return {
|
||||
"source_url": source_url,
|
||||
"source_kind": source_kind,
|
||||
"repo_scores": {
|
||||
"grade": scores.grade,
|
||||
"slop_score": scores.slop_score,
|
||||
"origin_score": scores.origin_score,
|
||||
"quality_deficit": scores.quality_deficit,
|
||||
"category": scores.category,
|
||||
"human_percent": scores.human_percent,
|
||||
"ai_percent": scores.ai_percent,
|
||||
"confidence": scores.confidence,
|
||||
"files_scored": scores.files_scored,
|
||||
"excluded_files": excluded_count,
|
||||
"strong_signals": scores.strong_signal_count,
|
||||
"medium_signals": scores.medium_signal_count,
|
||||
},
|
||||
"worst_files": strongest,
|
||||
"ai_verdicts": [
|
||||
{
|
||||
"path": verdict.path,
|
||||
"category": verdict.category,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"reasoning": verdict.reasoning,
|
||||
"notable_signals": verdict.notable_signals,
|
||||
}
|
||||
for verdict in verdicts
|
||||
],
|
||||
"template_provenance": {
|
||||
"score": template.score,
|
||||
"markers": template.markers,
|
||||
},
|
||||
"image_review": image_stats,
|
||||
"image_verdicts": [
|
||||
{
|
||||
"path": verdict.relative,
|
||||
"grade": verdict.grade,
|
||||
"verdict": verdict.verdict,
|
||||
"ai_probability": verdict.ai_probability,
|
||||
"image_kind": verdict.image_kind,
|
||||
"tells": verdict.tells,
|
||||
"description": verdict.description,
|
||||
}
|
||||
for verdict in sorted(image_verdicts, key=lambda item: -item.ai_probability)[:12]
|
||||
],
|
||||
"dom_evidence": (
|
||||
{
|
||||
"detected_builder": dom_evidence.detected_builder,
|
||||
"builder_confidence": dom_evidence.builder_confidence,
|
||||
"dom_slop_score": dom_evidence.score,
|
||||
"bucket": dom_evidence.bucket,
|
||||
"top_signals": [signal.to_dict() for signal in dom_evidence.signals[:6]],
|
||||
}
|
||||
if dom_evidence is not None
|
||||
else {"detected_builder": None, "builder_confidence": 0.0, "dom_slop_score": 0.0, "bucket": "none", "top_signals": []}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def fallback_report(payload: dict[str, Any]) -> str:
|
||||
repo = payload["repo_scores"]
|
||||
lines: list[str] = [
|
||||
"# AI Usage Classification Report",
|
||||
"",
|
||||
f"Source: `{payload['source_url']}` ({payload['source_kind']})",
|
||||
"",
|
||||
"## Verdict",
|
||||
"",
|
||||
f"Category **{repo['category']}** with grade **{repo['grade']}** "
|
||||
f"(slop score {repo['slop_score']}/100, confidence {repo['confidence']}).",
|
||||
]
|
||||
dom_evidence = payload.get("dom_evidence") or {}
|
||||
if dom_evidence.get("detected_builder"):
|
||||
lines.append(
|
||||
f"Detected: built with {dom_evidence['detected_builder']} (rendered-page fingerprint, "
|
||||
f"confidence {dom_evidence.get('builder_confidence', 0.0):.0%})."
|
||||
)
|
||||
lines.extend([
|
||||
f"Estimated composition: **{repo['human_percent']}% human**, **{repo['ai_percent']}% AI**.",
|
||||
"",
|
||||
"## Scores",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"| --- | --- |",
|
||||
f"| Grade | {repo['grade']} |",
|
||||
f"| Slop score | {repo['slop_score']} / 100 |",
|
||||
f"| Origin score | {repo['origin_score']} / 100 |",
|
||||
f"| Quality deficit | {repo['quality_deficit']} / 100 |",
|
||||
f"| Human share | {repo['human_percent']}% |",
|
||||
f"| AI share | {repo['ai_percent']}% |",
|
||||
f"| Files scored | {repo['files_scored']} |",
|
||||
f"| Files excluded | {repo['excluded_files']} |",
|
||||
f"| Strong signals | {repo['strong_signals']} |",
|
||||
f"| Medium signals | {repo['medium_signals']} |",
|
||||
"",
|
||||
"## Notable Files",
|
||||
"",
|
||||
])
|
||||
for entry in payload["worst_files"]:
|
||||
lines.append(
|
||||
f"- `{entry['path']}`: {entry['category']}, origin {entry['origin']}, quality deficit {entry['quality_deficit']}"
|
||||
)
|
||||
for signal in entry["signals"]:
|
||||
lines.append(f" - [{signal['severity']}] {signal['title']} (line {signal['line']})")
|
||||
template = payload.get("template_provenance", {})
|
||||
if template.get("markers"):
|
||||
lines.extend(["", "## Template Provenance", ""])
|
||||
lines.append(
|
||||
f"Starter template evidence score **{template.get('score')}/100**. The authenticity grade is "
|
||||
"reduced accordingly: an unmodified starter ships its scaffold's defaults rather than original work."
|
||||
)
|
||||
for marker in template["markers"]:
|
||||
lines.append(f"- {marker}")
|
||||
if payload["ai_verdicts"]:
|
||||
lines.extend(["", "## AI Review Findings", ""])
|
||||
for verdict in payload["ai_verdicts"]:
|
||||
lines.append(f"- `{verdict['path']}`: {verdict['category']} ({verdict['ai_probability']}% AI). {verdict['reasoning']}")
|
||||
image_review = payload.get("image_review", {})
|
||||
if payload.get("image_verdicts"):
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Image Review",
|
||||
"",
|
||||
f"Reviewed {image_review.get('count', 0)} images; "
|
||||
f"{image_review.get('ai_generated_count', 0)} appear AI-generated "
|
||||
f"(mean AI likelihood {image_review.get('mean_ai_probability', 0)}%, image grade {image_review.get('grade', 'n/a')}).",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for verdict in payload["image_verdicts"]:
|
||||
tells = f" Tells: {', '.join(verdict['tells'])}." if verdict["tells"] else ""
|
||||
lines.append(
|
||||
f"- `{verdict['path']}` ({verdict['image_kind']}): grade {verdict['grade']}, {verdict['verdict']} "
|
||||
f"({verdict['ai_probability']}% AI). {verdict['description']}{tells}"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Caveats",
|
||||
"",
|
||||
"No detector is definitive. This report combines weighted static signals with an AI review pass "
|
||||
"and should be read as calibrated guidance rather than proof of provenance.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def generate_report(
|
||||
llm: LlmClient,
|
||||
source_url: str,
|
||||
source_kind: str,
|
||||
scores: RepoScores,
|
||||
files: list[FileScore],
|
||||
verdicts: list[AiVerdict],
|
||||
excluded_count: int,
|
||||
image_verdicts: list[ImageVerdict],
|
||||
image_stats: dict[str, Any],
|
||||
template: TemplateEvidence,
|
||||
dom_evidence: DomEvidence | None,
|
||||
) -> tuple[str, str]:
|
||||
payload = _summary_payload(
|
||||
source_url,
|
||||
source_kind,
|
||||
scores,
|
||||
files,
|
||||
verdicts,
|
||||
excluded_count,
|
||||
image_verdicts,
|
||||
image_stats,
|
||||
template,
|
||||
dom_evidence,
|
||||
)
|
||||
user = "Write the final report for this classification data:\n" + json.dumps(payload, indent=2)[:24000]
|
||||
try:
|
||||
markdown = await llm.complete(REPORT_SYSTEM_PROMPT, user)
|
||||
if "# " not in markdown:
|
||||
raise LlmUnavailableError("report missing markdown structure")
|
||||
logger.info("Report generated via %s (%d chars)", llm.active_backend_name, len(markdown))
|
||||
return markdown.strip(), llm.active_backend_name
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("LLM report generation failed, using deterministic fallback: %s", error)
|
||||
return fallback_report(payload), "static-fallback"
|
||||
215
devplacepy/services/jobs/isslop/agent/vision.py
Normal file
215
devplacepy/services/jobs/isslop/agent/vision.py
Normal file
@ -0,0 +1,215 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from devplacepy.services.jobs.isslop.agent.llm import LlmClient, LlmUnavailableError, extract_json_object
|
||||
from devplacepy.services.jobs.isslop.config import (
|
||||
IMAGE_CONCURRENCY,
|
||||
IMAGE_EXTENSIONS,
|
||||
IMAGE_MAX_BYTES,
|
||||
IMAGE_MAX_COUNT,
|
||||
IMAGE_MIN_BYTES,
|
||||
IMAGE_MIN_DIMENSION,
|
||||
THUMBNAIL_MAX_DIMENSION,
|
||||
THUMBNAIL_QUALITY,
|
||||
VISION_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VISION_PROMPT: str = (
|
||||
"You are a forensic image analyst deciding whether an image is AI-generated (or heavily AI-edited) or a genuine "
|
||||
"human-made photograph, illustration or screenshot. Examine, in detail: hands and fingers, faces and teeth, skin "
|
||||
"texture (waxy or plastic over-smoothing), eyes and reflections, text and lettering (garbled or nonsensical), "
|
||||
"lighting and shadow direction consistency, background warping and impossible geometry, repeated or cloned patterns, "
|
||||
"over-saturated or airbrushed rendering, and the 'too perfect' mathematically smooth look typical of diffusion models. "
|
||||
"Screenshots, logos, diagrams, icons and real product photos are usually not AI-generated; judge accordingly and do "
|
||||
"not call an ordinary screenshot AI art. Respond with a single JSON object only, using exactly these keys: "
|
||||
'{"ai_probability": <integer 0-100, likelihood the image is AI-generated>, '
|
||||
'"verdict": <one of "ai-generated"|"likely-ai"|"uncertain"|"likely-human"|"human-made">, '
|
||||
'"image_kind": <short phrase, e.g. "photo of a person", "screenshot", "logo", "illustration">, '
|
||||
'"tells": [<up to five short strings naming concrete artifacts or reasons>], '
|
||||
'"description": <two to four factual sentences describing the image and the evidence>}'
|
||||
)
|
||||
|
||||
VERDICT_VALUES: frozenset[str] = frozenset(
|
||||
{"ai-generated", "likely-ai", "uncertain", "likely-human", "human-made"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageVerdict:
|
||||
relative: str
|
||||
ai_probability: float
|
||||
grade: str
|
||||
verdict: str
|
||||
image_kind: str
|
||||
tells: list[str]
|
||||
description: str
|
||||
|
||||
|
||||
def _grade_for(ai_probability: float) -> str:
|
||||
if ai_probability <= 15:
|
||||
return "A"
|
||||
if ai_probability <= 35:
|
||||
return "B"
|
||||
if ai_probability <= 55:
|
||||
return "C"
|
||||
if ai_probability <= 75:
|
||||
return "D"
|
||||
return "F"
|
||||
|
||||
|
||||
def _eligible(path: Path) -> bool:
|
||||
if path.suffix.lower() not in IMAGE_EXTENSIONS:
|
||||
return False
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return False
|
||||
if size < IMAGE_MIN_BYTES or size > IMAGE_MAX_BYTES:
|
||||
return False
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
width, height = image.size
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.debug("Skipping unreadable image %s: %s", path, error)
|
||||
return False
|
||||
if width < IMAGE_MIN_DIMENSION or height < IMAGE_MIN_DIMENSION:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def collect_images(workspace: Path, limit: int = IMAGE_MAX_COUNT) -> list[Path]:
|
||||
candidates = [
|
||||
path
|
||||
for path in sorted(workspace.rglob("*"))
|
||||
if path.is_file() and not path.is_symlink() and _eligible(path)
|
||||
]
|
||||
if len(candidates) <= limit:
|
||||
logger.info("Collected %d images for vision analysis", len(candidates))
|
||||
return candidates
|
||||
ranked = sorted(
|
||||
candidates,
|
||||
key=lambda path: hashlib.sha256(str(path.relative_to(workspace)).encode("utf-8")).hexdigest(),
|
||||
)
|
||||
sample = sorted(ranked[:limit])
|
||||
logger.info("Sampled %d of %d images deterministically for vision analysis", len(sample), len(candidates))
|
||||
return sample
|
||||
|
||||
|
||||
def _to_data_url(path: Path) -> Optional[str]:
|
||||
mime_type = mimetypes.guess_type(path.name)[0]
|
||||
if not mime_type or not mime_type.startswith("image/"):
|
||||
return None
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as error:
|
||||
logger.warning("Image read failed for %s: %s", path, error)
|
||||
return None
|
||||
return f"data:{mime_type};base64,{base64.b64encode(raw).decode('utf-8')}"
|
||||
|
||||
|
||||
async def classify_image(llm: LlmClient, workspace: Path, path: Path) -> Optional[ImageVerdict]:
|
||||
relative = str(path.relative_to(workspace))
|
||||
data_url = await asyncio.to_thread(_to_data_url, path)
|
||||
if data_url is None:
|
||||
return None
|
||||
try:
|
||||
answer = await llm.describe_image(VISION_PROMPT, data_url, VISION_TIMEOUT_SECONDS)
|
||||
except LlmUnavailableError as error:
|
||||
logger.warning("Vision unavailable for %s: %s", relative, error)
|
||||
return None
|
||||
parsed = extract_json_object(answer)
|
||||
if parsed is None:
|
||||
logger.warning("Vision returned unparseable output for %s", relative)
|
||||
return None
|
||||
probability = parsed.get("ai_probability")
|
||||
if not isinstance(probability, (int, float)) or isinstance(probability, bool):
|
||||
probability = 50.0
|
||||
probability = float(min(100.0, max(0.0, probability)))
|
||||
verdict = str(parsed.get("verdict", "uncertain")).strip().lower()
|
||||
if verdict not in VERDICT_VALUES:
|
||||
verdict = "uncertain"
|
||||
tells_field = parsed.get("tells")
|
||||
tells = [str(item)[:120] for item in tells_field[:5]] if isinstance(tells_field, list) else []
|
||||
return ImageVerdict(
|
||||
relative=relative,
|
||||
ai_probability=probability,
|
||||
grade=_grade_for(probability),
|
||||
verdict=verdict,
|
||||
image_kind=str(parsed.get("image_kind", "image"))[:80],
|
||||
tells=tells,
|
||||
description=str(parsed.get("description", ""))[:1000],
|
||||
)
|
||||
|
||||
|
||||
async def analyze_images(
|
||||
llm: LlmClient,
|
||||
workspace: Path,
|
||||
images: list[Path],
|
||||
) -> list[ImageVerdict]:
|
||||
semaphore = asyncio.Semaphore(IMAGE_CONCURRENCY)
|
||||
|
||||
async def worker(path: Path) -> Optional[ImageVerdict]:
|
||||
async with semaphore:
|
||||
return await classify_image(llm, workspace, path)
|
||||
|
||||
results = await asyncio.gather(*(worker(path) for path in images))
|
||||
verdicts = [result for result in results if result is not None]
|
||||
logger.info("Vision analysis produced %d verdicts from %d images", len(verdicts), len(images))
|
||||
return verdicts
|
||||
|
||||
|
||||
def make_thumbnail(source: Path, media_dir: Path, relative: str) -> Optional[str]:
|
||||
name = f"{hashlib.sha1(relative.encode('utf-8')).hexdigest()[:16]}.webp"
|
||||
try:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(source) as image:
|
||||
image.thumbnail((THUMBNAIL_MAX_DIMENSION, THUMBNAIL_MAX_DIMENSION))
|
||||
if image.mode not in ("RGB", "RGBA"):
|
||||
image = image.convert("RGBA")
|
||||
image.save(media_dir / name, format="WEBP", quality=THUMBNAIL_QUALITY)
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.warning("Thumbnail failed for %s: %s", relative, error)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def persist_screenshot(raw_bytes: bytes, media_dir: Path, relative: str) -> Optional[str]:
|
||||
name = f"{hashlib.sha1(relative.encode('utf-8')).hexdigest()[:16]}.webp"
|
||||
try:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(io.BytesIO(raw_bytes)) as image:
|
||||
image.thumbnail((THUMBNAIL_MAX_DIMENSION, THUMBNAIL_MAX_DIMENSION))
|
||||
if image.mode not in ("RGB", "RGBA"):
|
||||
image = image.convert("RGBA")
|
||||
image.save(media_dir / name, format="WEBP", quality=THUMBNAIL_QUALITY)
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
logger.warning("Screenshot persist failed for %s: %s", relative, error)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def image_summary(verdicts: list[ImageVerdict]) -> dict[str, float | int | str]:
|
||||
if not verdicts:
|
||||
return {"count": 0, "mean_ai_probability": 0.0, "grade": "n/a", "ai_generated_count": 0}
|
||||
mean = sum(verdict.ai_probability for verdict in verdicts) / len(verdicts)
|
||||
ai_generated = sum(1 for verdict in verdicts if verdict.ai_probability >= 60.0)
|
||||
return {
|
||||
"count": len(verdicts),
|
||||
"mean_ai_probability": round(mean, 1),
|
||||
"grade": _grade_for(mean),
|
||||
"ai_generated_count": ai_generated,
|
||||
}
|
||||
1
devplacepy/services/jobs/isslop/analysis/__init__.py
Normal file
1
devplacepy/services/jobs/isslop/analysis/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
74
devplacepy/services/jobs/isslop/analysis/baselines.py
Normal file
74
devplacepy/services/jobs/isslop/analysis/baselines.py
Normal file
@ -0,0 +1,74 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_ORIGIN, SEVERITY_MEDIUM, Signal
|
||||
|
||||
MIN_FILES_FOR_BASELINE: int = 8
|
||||
DEVIATION_Z_THRESHOLD: float = 2.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoBaselines:
|
||||
file_count: int
|
||||
indent_variance_mean: float
|
||||
indent_variance_std: float
|
||||
comment_ratio_mean: float
|
||||
comment_ratio_std: float
|
||||
blank_ratio_mean: float
|
||||
blank_ratio_std: float
|
||||
|
||||
|
||||
def _mean_std(values: list[float]) -> tuple[float, float]:
|
||||
if not values:
|
||||
return 0.0, 0.0
|
||||
mean = sum(values) / len(values)
|
||||
variance = sum((value - mean) ** 2 for value in values) / len(values)
|
||||
return mean, math.sqrt(variance)
|
||||
|
||||
|
||||
def compute_baselines(metrics_by_file: dict[str, FileMetrics]) -> RepoBaselines:
|
||||
indent = [metrics.indent_variance for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
comment = [metrics.comment_ratio for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
blank = [metrics.blank_ratio for metrics in metrics_by_file.values() if metrics.sloc > 10]
|
||||
indent_mean, indent_std = _mean_std(indent)
|
||||
comment_mean, comment_std = _mean_std(comment)
|
||||
blank_mean, blank_std = _mean_std(blank)
|
||||
return RepoBaselines(
|
||||
file_count=len(indent),
|
||||
indent_variance_mean=indent_mean,
|
||||
indent_variance_std=indent_std,
|
||||
comment_ratio_mean=comment_mean,
|
||||
comment_ratio_std=comment_std,
|
||||
blank_ratio_mean=blank_mean,
|
||||
blank_ratio_std=blank_std,
|
||||
)
|
||||
|
||||
|
||||
def deviation_signals(relative: str, metrics: FileMetrics, baselines: RepoBaselines) -> list[Signal]:
|
||||
if baselines.file_count < MIN_FILES_FOR_BASELINE or metrics.sloc <= 10:
|
||||
return []
|
||||
scores: list[float] = []
|
||||
if baselines.indent_variance_std > 0.01:
|
||||
scores.append((baselines.indent_variance_mean - metrics.indent_variance) / baselines.indent_variance_std)
|
||||
if baselines.comment_ratio_std > 0.01:
|
||||
scores.append((metrics.comment_ratio - baselines.comment_ratio_mean) / baselines.comment_ratio_std)
|
||||
if not scores:
|
||||
return []
|
||||
composite = sum(scores) / len(scores)
|
||||
if composite >= DEVIATION_Z_THRESHOLD:
|
||||
return [
|
||||
Signal(
|
||||
code="CONVENTION_DEVIATION",
|
||||
title=f"File is markedly cleaner and more regular than repo baseline (z={composite:.1f})",
|
||||
severity=SEVERITY_MEDIUM,
|
||||
axis=AXIS_ORIGIN,
|
||||
weight=3.0,
|
||||
line=1,
|
||||
evidence=f"indent deviation {metrics.indent_variance:.2f} vs repo mean {baselines.indent_variance_mean:.2f}",
|
||||
)
|
||||
]
|
||||
return []
|
||||
@ -0,0 +1,2 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_QUALITY, SEVERITY_WEAK, Signal
|
||||
|
||||
GENERIC_ALT_PATTERN = re.compile(r"^(image of|photo of|picture of)\b", re.IGNORECASE)
|
||||
DUPLICATE_ALT_THRESHOLD: int = 3
|
||||
EMPTY_ALT_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _detect_generic_alt_text(images: list[dict]) -> Signal | None:
|
||||
empty_count = 0
|
||||
for image in images:
|
||||
alt = str(image.get("alt", "")).strip() if isinstance(image, dict) else ""
|
||||
if not alt:
|
||||
empty_count += 1
|
||||
continue
|
||||
if GENERIC_ALT_PATTERN.match(alt):
|
||||
return Signal(
|
||||
"GENERIC_ALT_TEXT",
|
||||
"Generic placeholder-style alt text on an image",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
alt[:100],
|
||||
)
|
||||
if empty_count >= EMPTY_ALT_THRESHOLD and len(images) >= EMPTY_ALT_THRESHOLD:
|
||||
return Signal(
|
||||
"GENERIC_ALT_TEXT",
|
||||
f"{empty_count} images with empty alt text",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
f"{empty_count} empty alt attributes",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_duplicate_alt_text(images: list[dict]) -> Signal | None:
|
||||
counts: dict[str, int] = {}
|
||||
for image in images:
|
||||
alt = str(image.get("alt", "")).strip() if isinstance(image, dict) else ""
|
||||
if alt:
|
||||
counts[alt] = counts.get(alt, 0) + 1
|
||||
for alt, count in counts.items():
|
||||
if count >= DUPLICATE_ALT_THRESHOLD:
|
||||
return Signal(
|
||||
"DUPLICATE_ALT_TEXT",
|
||||
f"{count} images share the exact same alt text",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
alt[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_skipped_heading_level(headings: list[dict]) -> Signal | None:
|
||||
levels = [
|
||||
heading.get("level")
|
||||
for heading in headings
|
||||
if isinstance(heading, dict) and isinstance(heading.get("level"), int)
|
||||
]
|
||||
if len(levels) < 2 or levels[0] != 1:
|
||||
return None
|
||||
next_level = levels[1]
|
||||
if isinstance(next_level, int) and next_level > 2:
|
||||
return Signal(
|
||||
"SKIPPED_HEADING_LEVEL",
|
||||
f"H1 followed directly by H{next_level} with no H2 in between",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
0.5,
|
||||
0,
|
||||
f"H1 -> H{next_level}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_accessibility_signals(page: DomPageContext) -> list[Signal]:
|
||||
images = page.dom.get("images", []) or []
|
||||
headings = page.dom.get("headings", []) or []
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_generic_alt_text(images),
|
||||
_detect_duplicate_alt_text(images),
|
||||
_detect_skipped_heading_level(headings),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@ -0,0 +1,58 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, DomSiteContext
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.builders import detected_builder
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.registry import run_dom_checks, run_dom_site_checks
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import SEVERITY_FACTORS, Signal
|
||||
|
||||
DOM_SATURATION: float = 16.0
|
||||
DOM_FINDINGS_CAP: int = 14
|
||||
|
||||
BUCKET_NONE: str = "none"
|
||||
BUCKET_LIGHT: str = "light"
|
||||
BUCKET_MODERATE: str = "moderate"
|
||||
BUCKET_HEAVY: str = "heavy"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomEvidence:
|
||||
score: float
|
||||
bucket: str
|
||||
signals: list[Signal] = field(default_factory=list)
|
||||
detected_builder: str | None = None
|
||||
builder_confidence: float = 0.0
|
||||
|
||||
|
||||
def _bucket_for(score: float) -> str:
|
||||
if score < 10.0:
|
||||
return BUCKET_NONE
|
||||
if score < 35.0:
|
||||
return BUCKET_LIGHT
|
||||
if score < 65.0:
|
||||
return BUCKET_MODERATE
|
||||
return BUCKET_HEAVY
|
||||
|
||||
|
||||
def aggregate_dom_evidence(pages: list[DomPageContext]) -> DomEvidence:
|
||||
if not pages:
|
||||
return DomEvidence(score=0.0, bucket=BUCKET_NONE, signals=[], detected_builder=None, builder_confidence=0.0)
|
||||
signals: list[Signal] = []
|
||||
for page in pages:
|
||||
signals.extend(run_dom_checks(page)[:DOM_FINDINGS_CAP])
|
||||
site = DomSiteContext(pages=pages, root_host=urlparse(pages[0].url).hostname or "")
|
||||
signals.extend(run_dom_site_checks(site))
|
||||
weighted = sum(signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3) for signal in signals)
|
||||
score = round(100.0 * (1.0 - math.exp(-weighted / DOM_SATURATION)), 1)
|
||||
builder_name, builder_confidence = detected_builder(pages)
|
||||
return DomEvidence(
|
||||
score=score,
|
||||
bucket=_bucket_for(score),
|
||||
signals=signals,
|
||||
detected_builder=builder_name,
|
||||
builder_confidence=builder_confidence,
|
||||
)
|
||||
41
devplacepy/services/jobs/isslop/analysis/domsignals/base.py
Normal file
41
devplacepy/services/jobs/isslop/analysis/domsignals/base.py
Normal file
@ -0,0 +1,41 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Signal
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomPageContext:
|
||||
url: str
|
||||
dom: dict = field(default_factory=dict)
|
||||
console_warnings: list[str] = field(default_factory=list)
|
||||
console_errors: list[str] = field(default_factory=list)
|
||||
response_headers: dict[str, str] = field(default_factory=dict)
|
||||
resource_hosts: list[str] = field(default_factory=list)
|
||||
screenshot_bytes: bytes | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomSiteContext:
|
||||
pages: list[DomPageContext] = field(default_factory=list)
|
||||
root_host: str = ""
|
||||
|
||||
|
||||
DomPageDetector = Callable[[DomPageContext], list[Signal]]
|
||||
DomSiteDetector = Callable[[DomSiteContext], list[Signal]]
|
||||
|
||||
DOM_CHECKS: list[DomPageDetector] = []
|
||||
DOM_SITE_CHECKS: list[DomSiteDetector] = []
|
||||
|
||||
|
||||
def dom_check(func: DomPageDetector) -> DomPageDetector:
|
||||
DOM_CHECKS.append(func)
|
||||
return func
|
||||
|
||||
|
||||
def dom_site_check(func: DomSiteDetector) -> DomSiteDetector:
|
||||
DOM_SITE_CHECKS.append(func)
|
||||
return func
|
||||
204
devplacepy/services/jobs/isslop/analysis/domsignals/builders.py
Normal file
204
devplacepy/services/jobs/isslop/analysis/domsignals/builders.py
Normal file
@ -0,0 +1,204 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CLAUDE_ARTIFACT_BADGE_KEY: str = 'iframe[src*="claudeusercontent.com"]'
|
||||
|
||||
FRAMER_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"events.framer.com", "framerusercontent.com", "framercdn.com", "framercanvas.com"}
|
||||
)
|
||||
WIX_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"static.wixstatic.com", "static.parastorage.com", "siteassets.parastorage.com"}
|
||||
)
|
||||
WEBFLOW_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"assets-global.website-files.com", "uploads-ssl.webflow.com", "assets.website-files.com"}
|
||||
)
|
||||
SQUARESPACE_RESOURCE_HOSTS: frozenset[str] = frozenset(
|
||||
{"static1.squarespace.com", "static.squarespace.com", "images.squarespace-cdn.com"}
|
||||
)
|
||||
HOSTING_HEADER_KEYS: frozenset[str] = frozenset({"x-vercel-id", "x-nf-request-id"})
|
||||
|
||||
|
||||
def _host_of(url: str) -> str:
|
||||
return (urlparse(url).hostname or "").lower()
|
||||
|
||||
|
||||
def _resource_text(page: DomPageContext) -> str:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
links = page.dom.get("links", []) or []
|
||||
hosts = page.resource_hosts or []
|
||||
return " ".join(str(item) for item in (*scripts, *links, *hosts)).lower()
|
||||
|
||||
|
||||
def _generator(page: DomPageContext) -> str:
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
return str(meta.get("generator") or "")
|
||||
|
||||
|
||||
def _class_names(page: DomPageContext) -> list[str]:
|
||||
return list((page.dom.get("classNames") or {}).keys())
|
||||
|
||||
|
||||
def _badge(page: DomPageContext, selector: str) -> bool:
|
||||
return bool((page.dom.get("badgeHits") or {}).get(selector))
|
||||
|
||||
|
||||
def _detect_lovable(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, "#lovable-badge")
|
||||
resource_hit = "cdn.gpteng.co" in _resource_text(page) or "gptengineer.js" in _resource_text(page)
|
||||
generator_hit = "lovable" in _generator(page).lower()
|
||||
host_hit = _host_of(page.url).endswith(".lovable.app")
|
||||
if not (badge or resource_hit or generator_hit or host_hit):
|
||||
return None
|
||||
evidence = "lovable-badge" if badge else "cdn.gpteng.co" if resource_hit else "generator meta" if generator_hit else "lovable.app host"
|
||||
return Signal("AI_BUILDER_LOVABLE", "Lovable / GPT Engineer builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_bolt(page: DomPageContext) -> Signal | None:
|
||||
haystack = _resource_text(page) + " " + page.url.lower()
|
||||
if "bolt.new" not in haystack:
|
||||
return None
|
||||
return Signal("AI_BUILDER_BOLT", "Bolt.new builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "bolt.new")
|
||||
|
||||
|
||||
def _detect_replit(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, ".replit-badge")
|
||||
host = _host_of(page.url)
|
||||
host_hit = host.endswith(".replit.app") or host.endswith(".repl.co")
|
||||
if not (badge or host_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_REPLIT", "Replit builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "replit-badge" if badge else host)
|
||||
|
||||
|
||||
def _detect_base44(page: DomPageContext) -> Signal | None:
|
||||
if not _badge(page, "#base44-badge"):
|
||||
return None
|
||||
return Signal("AI_BUILDER_BASE44", "Base44 builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "base44-badge")
|
||||
|
||||
|
||||
def _detect_framer(page: DomPageContext) -> Signal | None:
|
||||
badge = _badge(page, "#__framer-badge-container")
|
||||
generator_hit = "framer" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & FRAMER_RESOURCE_HOSTS)
|
||||
host_hit = _host_of(page.url).endswith(".framer.website")
|
||||
if not (badge or generator_hit or resource_hit or host_hit):
|
||||
return None
|
||||
evidence = "framer-badge" if badge else "generator meta" if generator_hit else "framer resource host" if resource_hit else "framer.website host"
|
||||
return Signal("AI_BUILDER_FRAMER", "Framer builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_wix(page: DomPageContext) -> Signal | None:
|
||||
generator_hit = "wix.com website builder" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & WIX_RESOURCE_HOSTS)
|
||||
class_hit = any("wix-bolt" in name.lower() for name in _class_names(page))
|
||||
if not (generator_hit or resource_hit or class_hit):
|
||||
return None
|
||||
evidence = "generator meta" if generator_hit else "wix resource host" if resource_hit else "wix-bolt class"
|
||||
return Signal("AI_BUILDER_WIX", "Wix builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, evidence)
|
||||
|
||||
|
||||
def _detect_webflow(page: DomPageContext) -> Signal | None:
|
||||
generator_hit = "webflow" in _generator(page).lower()
|
||||
resource_hit = bool(set(page.resource_hosts or []) & WEBFLOW_RESOURCE_HOSTS)
|
||||
if not (generator_hit or resource_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_WEBFLOW", "Webflow builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "generator meta" if generator_hit else "webflow resource host")
|
||||
|
||||
|
||||
def _detect_godaddy(page: DomPageContext) -> Signal | None:
|
||||
if "go daddy website builder" not in _generator(page).lower():
|
||||
return None
|
||||
return Signal("AI_BUILDER_GODADDY", "GoDaddy Website Builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "generator meta")
|
||||
|
||||
|
||||
def _detect_squarespace(page: DomPageContext) -> Signal | None:
|
||||
resource_hit = bool(set(page.resource_hosts or []) & SQUARESPACE_RESOURCE_HOSTS)
|
||||
class_hit = any(name.lower().startswith("sqs-block") or name.lower().startswith("yui3-") for name in _class_names(page))
|
||||
if not (resource_hit or class_hit):
|
||||
return None
|
||||
return Signal("AI_BUILDER_SQUARESPACE", "Squarespace builder fingerprint", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "squarespace resource host" if resource_hit else "sqs-block class")
|
||||
|
||||
|
||||
def _detect_claude_artifact(page: DomPageContext) -> Signal | None:
|
||||
if not _badge(page, CLAUDE_ARTIFACT_BADGE_KEY):
|
||||
return None
|
||||
return Signal("AI_BUILDER_CLAUDE_ARTIFACT", "Claude Artifacts iframe embed detected", SEVERITY_STRONG, AXIS_ORIGIN, 4.0, 0, "claudeusercontent.com iframe")
|
||||
|
||||
|
||||
def _detect_shadcn_radix_cluster(page: DomPageContext) -> Signal | None:
|
||||
indicators: list[str] = []
|
||||
if _badge(page, "[data-radix-root]"):
|
||||
indicators.append("data-radix-root")
|
||||
if any("radix" in name.lower() for name in _class_names(page)):
|
||||
indicators.append("radix class token")
|
||||
if "lucide" in _resource_text(page):
|
||||
indicators.append("lucide icons")
|
||||
if len(indicators) < 2:
|
||||
return None
|
||||
severity = SEVERITY_STRONG if len(indicators) >= 3 else SEVERITY_MEDIUM
|
||||
return Signal("SHADCN_RADIX_CLUSTER", f"shadcn/ui and Radix primitives cluster ({len(indicators)} indicators)", severity, AXIS_ORIGIN, 3.0, 0, ", ".join(indicators))
|
||||
|
||||
|
||||
def _detect_hosting_subdomain(page: DomPageContext) -> Signal | None:
|
||||
host = _host_of(page.url)
|
||||
host_hit = host.endswith(".vercel.app") or host.endswith(".netlify.app") or host.endswith(".databutton.app")
|
||||
headers = {key.lower() for key in (page.response_headers or {}).keys()}
|
||||
header_hit = bool(headers & HOSTING_HEADER_KEYS)
|
||||
if not (host_hit or header_hit):
|
||||
return None
|
||||
evidence = host if host_hit else ", ".join(sorted(headers & HOSTING_HEADER_KEYS))
|
||||
return Signal("BUILDER_HOSTING_SUBDOMAIN", "Hosted on a default builder/PaaS subdomain", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, 0, evidence)
|
||||
|
||||
|
||||
_VENDOR_DETECTORS: tuple[tuple[Callable[[DomPageContext], Signal | None], str], ...] = (
|
||||
(_detect_lovable, "Lovable"),
|
||||
(_detect_bolt, "Bolt.new"),
|
||||
(_detect_replit, "Replit"),
|
||||
(_detect_base44, "Base44"),
|
||||
(_detect_framer, "Framer"),
|
||||
(_detect_wix, "Wix"),
|
||||
(_detect_webflow, "Webflow"),
|
||||
(_detect_godaddy, "GoDaddy"),
|
||||
(_detect_squarespace, "Squarespace"),
|
||||
(_detect_claude_artifact, "Claude Artifacts"),
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_builders(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for detector, _label in _VENDOR_DETECTORS:
|
||||
signal = detector(page)
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
cluster = _detect_shadcn_radix_cluster(page)
|
||||
if cluster is not None:
|
||||
findings.append(cluster)
|
||||
hosting = _detect_hosting_subdomain(page)
|
||||
if hosting is not None:
|
||||
findings.append(hosting)
|
||||
return findings
|
||||
|
||||
|
||||
def detected_builder(pages: list[DomPageContext]) -> tuple[str | None, float]:
|
||||
cluster_hit = False
|
||||
for page in pages:
|
||||
for detector, label in _VENDOR_DETECTORS:
|
||||
if detector(page) is not None:
|
||||
return label, 1.0
|
||||
if _detect_shadcn_radix_cluster(page) is not None:
|
||||
cluster_hit = True
|
||||
if cluster_hit:
|
||||
return "shadcn/ui + Radix", 0.5
|
||||
return None, 0.0
|
||||
@ -0,0 +1,185 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HASHED_SEGMENT_PATTERN = re.compile(r"[.-][0-9a-f]{6,}\.js", re.IGNORECASE)
|
||||
UNHASHED_BUNDLE_PATTERN = re.compile(r"/(main|bundle|index)\.js(\?|$)", re.IGNORECASE)
|
||||
PLACEHOLDER_TEXT_PATTERN = re.compile(
|
||||
r"lorem ipsum|coming soon|your headline here|add your description|placeholder", re.IGNORECASE
|
||||
)
|
||||
PLACEHOLDER_IMAGE_HOSTS: tuple[str, ...] = ("via.placeholder.com", "placehold.co", "picsum.photos")
|
||||
TAILWIND_PRODUCTION_WARNING: str = "cdn.tailwindcss.com should not be used in production"
|
||||
REACT_DEV_WARNING_MARKERS: tuple[str, ...] = ("development build of React", "Each child in a list should have a unique")
|
||||
|
||||
|
||||
def _text_blob(page: DomPageContext) -> str:
|
||||
parts = [
|
||||
str(heading.get("text", ""))
|
||||
for heading in (page.dom.get("headings") or [])
|
||||
if isinstance(heading, dict) and heading.get("text")
|
||||
]
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
description = meta.get("description") if isinstance(meta, dict) else ""
|
||||
if description:
|
||||
parts.append(str(description))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _looks_unhashed(src: str) -> bool:
|
||||
if not src:
|
||||
return False
|
||||
if HASHED_SEGMENT_PATTERN.search(src):
|
||||
return False
|
||||
return bool(UNHASHED_BUNDLE_PATTERN.search(src))
|
||||
|
||||
|
||||
def _detect_tailwind_cdn(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
if not any("cdn.tailwindcss.com" in str(src) for src in scripts):
|
||||
return None
|
||||
console_text = " ".join(page.console_warnings + page.console_errors)
|
||||
if TAILWIND_PRODUCTION_WARNING in console_text:
|
||||
return Signal(
|
||||
"TAILWIND_CDN_DOM",
|
||||
"Tailwind CDN build flagged unsuitable for production by the browser console",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.5,
|
||||
0,
|
||||
"cdn.tailwindcss.com + production warning",
|
||||
)
|
||||
return Signal(
|
||||
"TAILWIND_CDN_DOM",
|
||||
"Tailwind loaded from the CDN build in the rendered page",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
"cdn.tailwindcss.com",
|
||||
)
|
||||
|
||||
|
||||
def _detect_cdn_react(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
for src in scripts:
|
||||
text = str(src)
|
||||
if "unpkg.com/react" in text or "esm.sh/react" in text or "esm.sh/tsx" in text:
|
||||
return Signal(
|
||||
"CDN_REACT_UNBUNDLED",
|
||||
"React loaded unbundled directly from a CDN",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
text[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_react_dev_warning(page: DomPageContext) -> Signal | None:
|
||||
console_text = " ".join(page.console_warnings + page.console_errors)
|
||||
if not any(marker in console_text for marker in REACT_DEV_WARNING_MARKERS):
|
||||
return None
|
||||
return Signal(
|
||||
"REACT_DEV_BUILD_WARNING",
|
||||
"React development-build or missing-key warnings present in the console",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
console_text[:150],
|
||||
)
|
||||
|
||||
|
||||
def _detect_unhashed_bundle(page: DomPageContext) -> Signal | None:
|
||||
scripts = page.dom.get("scripts", []) or []
|
||||
for src in scripts:
|
||||
if _looks_unhashed(str(src)):
|
||||
return Signal(
|
||||
"UNHASHED_BUNDLE_FILENAME",
|
||||
"Script bundle filename with no content-hash segment",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
0.5,
|
||||
0,
|
||||
str(src),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_placeholder_content(page: DomPageContext) -> Signal | None:
|
||||
match = PLACEHOLDER_TEXT_PATTERN.search(_text_blob(page))
|
||||
if not match:
|
||||
return None
|
||||
return Signal(
|
||||
"PLACEHOLDER_CONTENT_DOM",
|
||||
"Placeholder or unedited template copy left in the rendered page",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
0,
|
||||
match.group(0),
|
||||
)
|
||||
|
||||
|
||||
def _detect_placeholder_image_host(page: DomPageContext) -> Signal | None:
|
||||
images = page.dom.get("images", []) or []
|
||||
for image in images:
|
||||
if not isinstance(image, dict):
|
||||
continue
|
||||
src = str(image.get("src", ""))
|
||||
if any(host in src for host in PLACEHOLDER_IMAGE_HOSTS):
|
||||
return Signal(
|
||||
"PLACEHOLDER_IMAGE_HOST",
|
||||
"Image served from a generic placeholder image host",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
src,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_default_framework_title(page: DomPageContext) -> Signal | None:
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
title = str(meta.get("title", ""))
|
||||
if title == "Vite + React" or "Create Next App" in title:
|
||||
return Signal(
|
||||
"DEFAULT_FRAMEWORK_TITLE",
|
||||
"Unedited default framework document title",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
title,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_build_signals(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_tailwind_cdn(page),
|
||||
_detect_cdn_react(page),
|
||||
_detect_react_dev_warning(page),
|
||||
_detect_unhashed_bundle(page),
|
||||
_detect_placeholder_content(page),
|
||||
_detect_placeholder_image_host(page),
|
||||
_detect_default_framework_title(page),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
245
devplacepy/services/jobs/isslop/analysis/domsignals/color.py
Normal file
245
devplacepy/services/jobs/isslop/analysis/domsignals/color.py
Normal file
@ -0,0 +1,245 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import colorsys
|
||||
import io
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HEX_COLOR_PATTERN = re.compile(r"#[0-9a-fA-F]{6}\b")
|
||||
RGB_COLOR_PATTERN = re.compile(r"rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)")
|
||||
GRADIENT_PATTERN = re.compile(r"linear-gradient\([^)]*\)", re.IGNORECASE)
|
||||
|
||||
TAILWIND_DEFAULT_ACCENTS: frozenset[str] = frozenset(
|
||||
{"#6366f1", "#4f46e5", "#8b5cf6", "#a855f7", "#7c3aed", "#818cf8"}
|
||||
)
|
||||
SIGNATURE_GRADIENT_PATTERN = re.compile(r"#667eea.{0,80}#764ba2|#764ba2.{0,80}#667eea", re.IGNORECASE | re.DOTALL)
|
||||
OKLCH_PATTERN = re.compile(r"oklch\(\s*([\d.]+)", re.IGNORECASE)
|
||||
BOX_SHADOW_PURPLE_PATTERN = re.compile(
|
||||
r"(rgba?\(\s*(?:99|79|139|168|124|129)\s*,\s*\d+\s*,\s*\d+|#(?:6366f1|4f46e5|8b5cf6|a855f7|7c3aed|818cf8))"
|
||||
r"[\s\S]{0,60}?(\d+)px",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DARK_LUMINANCE_THRESHOLD: int = 40
|
||||
|
||||
|
||||
def _hue_from_hex(hex_code: str) -> float | None:
|
||||
try:
|
||||
r = int(hex_code[1:3], 16) / 255.0
|
||||
g = int(hex_code[3:5], 16) / 255.0
|
||||
b = int(hex_code[5:7], 16) / 255.0
|
||||
except ValueError:
|
||||
return None
|
||||
hue, _lightness, _saturation = colorsys.rgb_to_hls(r, g, b)
|
||||
return hue * 360.0
|
||||
|
||||
|
||||
def _hue_from_rgb(r: int, g: int, b: int) -> float:
|
||||
hue, _lightness, _saturation = colorsys.rgb_to_hls(r / 255.0, g / 255.0, b / 255.0)
|
||||
return hue * 360.0
|
||||
|
||||
|
||||
def _is_blue_or_purple(hue: float) -> bool:
|
||||
return 210.0 <= hue <= 300.0
|
||||
|
||||
|
||||
def _sample_text(sample: dict) -> str:
|
||||
return " ".join(
|
||||
str(sample.get(field, ""))
|
||||
for field in ("backgroundImage", "backgroundColor", "color", "boxShadow")
|
||||
)
|
||||
|
||||
|
||||
def _detect_gradient_hue_combo(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
gradient = GRADIENT_PATTERN.search(str(sample.get("backgroundImage", "")))
|
||||
if not gradient:
|
||||
continue
|
||||
text = gradient.group(0)
|
||||
hues: list[float] = []
|
||||
for match in HEX_COLOR_PATTERN.finditer(text):
|
||||
hue = _hue_from_hex(match.group(0))
|
||||
if hue is not None:
|
||||
hues.append(hue)
|
||||
for match in RGB_COLOR_PATTERN.finditer(text):
|
||||
hues.append(_hue_from_rgb(int(match.group(1)), int(match.group(2)), int(match.group(3))))
|
||||
if sum(1 for hue in hues if _is_blue_or_purple(hue)) >= 2:
|
||||
return Signal(
|
||||
"SIGNATURE_GRADIENT_HUE_COMBO",
|
||||
"Gradient stops both land in the default LLM blue/purple hue range",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
text[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_tailwind_accent(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
text = _sample_text(sample).lower()
|
||||
for accent in TAILWIND_DEFAULT_ACCENTS:
|
||||
if accent in text:
|
||||
return Signal(
|
||||
"TAILWIND_DEFAULT_ACCENT",
|
||||
"Literal Tailwind default accent color used verbatim",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
accent,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_signature_gradient(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = SIGNATURE_GRADIENT_PATTERN.search(_sample_text(sample))
|
||||
if match:
|
||||
return Signal(
|
||||
"SIGNATURE_GRADIENT",
|
||||
"Canonical LLM purple gradient #667eea to #764ba2",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
0,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_oklch_token(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = OKLCH_PATTERN.search(_sample_text(sample))
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
lightness = float(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if 0.55 <= lightness <= 0.75:
|
||||
return Signal(
|
||||
"OKLCH_SHADCN_TOKEN",
|
||||
"oklch() color token in the shadcn/ui default lightness band",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
match.group(0),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_purple_glow_shadow(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
match = BOX_SHADOW_PURPLE_PATTERN.search(str(sample.get("boxShadow", "")))
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
blur = int(match.group(2))
|
||||
except ValueError:
|
||||
continue
|
||||
if blur >= 40:
|
||||
return Signal(
|
||||
"PURPLE_GLOW_SHADOW",
|
||||
"Large-blur purple/blue glow box-shadow",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_dark_mode_default(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
if sample.get("tag") != "body":
|
||||
continue
|
||||
match = RGB_COLOR_PATTERN.search(str(sample.get("backgroundColor", "")))
|
||||
if not match:
|
||||
continue
|
||||
channels = [int(match.group(1)), int(match.group(2)), int(match.group(3))]
|
||||
if sum(channels) / 3.0 < DARK_LUMINANCE_THRESHOLD:
|
||||
return Signal(
|
||||
"DARK_MODE_DEFAULT",
|
||||
"Dark background color by default",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
str(sample.get("backgroundColor", "")),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _screenshot_hue_buckets(screenshot_bytes: bytes) -> dict[int, int]:
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(io.BytesIO(screenshot_bytes)).convert("RGB")
|
||||
image = image.resize((64, 64))
|
||||
buckets: dict[int, int] = {}
|
||||
for r, g, b in image.getdata():
|
||||
hue, lightness, saturation = colorsys.rgb_to_hls(r / 255.0, g / 255.0, b / 255.0)
|
||||
if saturation < 0.15 or lightness < 0.05 or lightness > 0.95:
|
||||
continue
|
||||
bucket = int((hue * 360.0) // 20) * 20
|
||||
buckets[bucket] = buckets.get(bucket, 0) + 1
|
||||
return buckets
|
||||
|
||||
|
||||
def _detect_screenshot_gradient_hero(screenshot_bytes: bytes | None) -> Signal | None:
|
||||
if not screenshot_bytes:
|
||||
return None
|
||||
try:
|
||||
buckets = _screenshot_hue_buckets(screenshot_bytes)
|
||||
except Exception:
|
||||
return None
|
||||
total = sum(buckets.values())
|
||||
if total <= 0:
|
||||
return None
|
||||
ranked = sorted(buckets.items(), key=lambda item: item[1], reverse=True)[:2]
|
||||
if len(ranked) < 2 or not all(_is_blue_or_purple(bucket) for bucket, _count in ranked):
|
||||
return None
|
||||
coverage = sum(count for _bucket, count in ranked) / total
|
||||
if coverage <= 0.3:
|
||||
return None
|
||||
return Signal(
|
||||
"SCREENSHOT_GRADIENT_HERO",
|
||||
"Screenshot dominated by a blue/purple hero gradient",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
f"{coverage:.0%} of sampled pixels in blue/purple hues",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_color_signals(page: DomPageContext) -> list[Signal]:
|
||||
samples = page.dom.get("colors", []) or []
|
||||
findings: list[Signal] = []
|
||||
for detector in (
|
||||
_detect_gradient_hue_combo,
|
||||
_detect_tailwind_accent,
|
||||
_detect_signature_gradient,
|
||||
_detect_oklch_token,
|
||||
_detect_purple_glow_shadow,
|
||||
_detect_dark_mode_default,
|
||||
):
|
||||
signal = detector(samples)
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
screenshot_signal = _detect_screenshot_gradient_hero(page.screenshot_bytes)
|
||||
if screenshot_signal is not None:
|
||||
findings.append(screenshot_signal)
|
||||
return findings
|
||||
157
devplacepy/services/jobs/isslop/analysis/domsignals/copy.py
Normal file
157
devplacepy/services/jobs/isslop/analysis/domsignals/copy.py
Normal file
@ -0,0 +1,157 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CLICHE_PHRASES: tuple[str, ...] = (
|
||||
r"unlock (the )?(power|potential) of",
|
||||
r"unleash the power of",
|
||||
r"take your .{1,40} to the next level",
|
||||
r"elevate your",
|
||||
r"seamlessly integrat(e|es|ion)",
|
||||
r"revolutioniz(e|ing) the way",
|
||||
r"whether you('re| are) a .{1,30} or (a )?.{1,30}",
|
||||
r"delve into",
|
||||
r"dive into the world of",
|
||||
r"embark on a journey",
|
||||
r"push(ing)? the boundaries of",
|
||||
r"at the forefront of",
|
||||
r"game[- ]chang(er|ing)",
|
||||
r"pave the way for",
|
||||
r"bridging the gap between",
|
||||
r"navigate the complexities of",
|
||||
r"foster a culture of",
|
||||
r"harness the power of",
|
||||
r"cutting[- ]edge",
|
||||
r"future[- ]proof",
|
||||
r"world[- ]class",
|
||||
r"enterprise[- ]grade",
|
||||
r"streamline(d)?",
|
||||
r"empower(ing)?",
|
||||
r"supercharg(e|ed|ing)",
|
||||
)
|
||||
CLICHE_PATTERN = re.compile("|".join(CLICHE_PHRASES), re.IGNORECASE)
|
||||
|
||||
BUZZWORDS: tuple[str, ...] = ("delve", "crucial", "intricate", "nuanced", "myriad", "realm", "tapestry", "landscape")
|
||||
BUZZWORD_THRESHOLD: int = 3
|
||||
BUZZWORD_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
||||
re.compile(rf"\b{word}\b", re.IGNORECASE) for word in BUZZWORDS
|
||||
)
|
||||
|
||||
HEADING_EMOJI_PATTERN = re.compile(r"[\U0001F300-\U0001FAFF✅⭐✨\U0001F680]")
|
||||
|
||||
FAQ_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"^what is\b", re.IGNORECASE),
|
||||
re.compile(r"^how does .{1,30} work\??$", re.IGNORECASE),
|
||||
re.compile(r"^is (it|.{1,20}) secure\??$", re.IGNORECASE),
|
||||
re.compile(r"^can i cancel", re.IGNORECASE),
|
||||
re.compile(r"^do you offer a? ?(free trial|refund)", re.IGNORECASE),
|
||||
)
|
||||
FAQ_PATTERN_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _heading_texts(page: DomPageContext) -> list[str]:
|
||||
return [
|
||||
str(heading.get("text", ""))
|
||||
for heading in (page.dom.get("headings") or [])
|
||||
if isinstance(heading, dict) and heading.get("text")
|
||||
]
|
||||
|
||||
|
||||
def _text_blob(page: DomPageContext) -> str:
|
||||
parts = list(_heading_texts(page))
|
||||
for image in page.dom.get("images", []) or []:
|
||||
if isinstance(image, dict) and image.get("alt"):
|
||||
parts.append(str(image["alt"]))
|
||||
meta = page.dom.get("meta", {}) or {}
|
||||
description = meta.get("description") if isinstance(meta, dict) else ""
|
||||
if description:
|
||||
parts.append(str(description))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _detect_template_copy(blob: str) -> Signal | None:
|
||||
match = CLICHE_PATTERN.search(blob)
|
||||
if not match:
|
||||
return None
|
||||
return Signal(
|
||||
"TEMPLATE_COPY_DOM",
|
||||
"Stock AI landing-page cliche phrase in rendered copy",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
match.group(0)[:150],
|
||||
)
|
||||
|
||||
|
||||
def _detect_buzzword_cluster(blob: str) -> Signal | None:
|
||||
hits = [pattern.pattern for pattern in BUZZWORD_PATTERNS if pattern.search(blob)]
|
||||
if len(hits) < BUZZWORD_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"AI_BUZZWORD_CLUSTER",
|
||||
f"Elevated buzzword cluster ({len(hits)} distinct terms)",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
", ".join(hits),
|
||||
)
|
||||
|
||||
|
||||
def _detect_emoji_heading(page: DomPageContext) -> Signal | None:
|
||||
for text in _heading_texts(page):
|
||||
if HEADING_EMOJI_PATTERN.search(text):
|
||||
return Signal(
|
||||
"EMOJI_HEADING_DOM",
|
||||
"Emoji embedded inside a rendered heading",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
text[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_generic_faq_template(page: DomPageContext) -> Signal | None:
|
||||
texts = _heading_texts(page)
|
||||
matched = 0
|
||||
for pattern in FAQ_PATTERNS:
|
||||
if any(pattern.search(text.strip()) for text in texts):
|
||||
matched += 1
|
||||
if matched < FAQ_PATTERN_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"GENERIC_FAQ_TEMPLATE",
|
||||
f"Generic templated FAQ question set ({matched} canonical patterns matched)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
f"{matched} FAQ patterns matched",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_copy_signals(page: DomPageContext) -> list[Signal]:
|
||||
blob = _text_blob(page)
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_template_copy(blob),
|
||||
_detect_buzzword_cluster(blob),
|
||||
_detect_emoji_heading(page),
|
||||
_detect_generic_faq_template(page),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
104
devplacepy/services/jobs/isslop/analysis/domsignals/layout.py
Normal file
104
devplacepy/services/jobs/isslop/analysis/domsignals/layout.py
Normal file
@ -0,0 +1,104 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
LANDING_SECTION_TOKENS: tuple[str, ...] = ("hero", "features", "testimonials", "pricing", "faq", "cta")
|
||||
LANDING_SECTION_THRESHOLD: int = 4
|
||||
DEAD_ANCHOR_THRESHOLD: int = 5
|
||||
REPEATED_CARD_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def _detect_landing_template(class_names: dict) -> Signal | None:
|
||||
matched = {
|
||||
token
|
||||
for token in LANDING_SECTION_TOKENS
|
||||
if any(token in name.lower() for name in class_names)
|
||||
}
|
||||
if len(matched) < LANDING_SECTION_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"LANDING_TEMPLATE_DOM",
|
||||
"Canonical hero-features-testimonials-pricing landing structure in the rendered DOM",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
0,
|
||||
", ".join(sorted(matched)),
|
||||
)
|
||||
|
||||
|
||||
def _detect_glassmorphic_navbar(samples: list[dict]) -> Signal | None:
|
||||
for sample in samples:
|
||||
tag = str(sample.get("tag", ""))
|
||||
if tag not in ("nav", "header"):
|
||||
continue
|
||||
blob = " ".join(
|
||||
str(sample.get(field, "")) for field in ("backgroundImage", "backgroundColor", "boxShadow")
|
||||
).lower()
|
||||
if "blur(" not in blob:
|
||||
continue
|
||||
if "rgba(" in blob or "hsla(" in blob:
|
||||
return Signal(
|
||||
"GLASSMORPHIC_NAVBAR",
|
||||
"Glassmorphic translucent-blur navigation bar",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
blob[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_repeated_card_class(class_names: dict) -> Signal | None:
|
||||
for name, count in class_names.items():
|
||||
if count >= REPEATED_CARD_THRESHOLD:
|
||||
return Signal(
|
||||
"REPEATED_CARD_CLASS",
|
||||
f"Class token repeated across {count} elements, a generated card-grid shape",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
f"{name} x{count}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_dead_anchor_links(dead_anchor_count: int) -> Signal | None:
|
||||
if dead_anchor_count < DEAD_ANCHOR_THRESHOLD:
|
||||
return None
|
||||
return Signal(
|
||||
"DEAD_ANCHOR_LINKS_DOM",
|
||||
f'{dead_anchor_count} rendered links pointing to href="#"',
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
0,
|
||||
f"{dead_anchor_count} placeholder links",
|
||||
)
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_layout_signals(page: DomPageContext) -> list[Signal]:
|
||||
class_names = page.dom.get("classNames", {}) or {}
|
||||
colors = page.dom.get("colors", []) or []
|
||||
dead_anchor_count = int(page.dom.get("deadAnchorCount", 0) or 0)
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_landing_template(class_names),
|
||||
_detect_glassmorphic_navbar(colors),
|
||||
_detect_repeated_card_class(class_names),
|
||||
_detect_dead_anchor_links(dead_anchor_count),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
@ -0,0 +1,66 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, DomSiteContext, dom_check, dom_site_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import AXIS_QUALITY, SEVERITY_MEDIUM, SEVERITY_WEAK, Signal
|
||||
|
||||
|
||||
def _meta(page: DomPageContext) -> dict:
|
||||
meta = page.dom.get("meta", {})
|
||||
return meta if isinstance(meta, dict) else {}
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_metaseo_signals(page: DomPageContext) -> list[Signal]:
|
||||
meta = _meta(page)
|
||||
findings: list[Signal] = []
|
||||
if not str(meta.get("description") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_META_DESCRIPTION", "No meta description present", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 0, "")
|
||||
)
|
||||
if int(page.dom.get("jsonld", 0) or 0) == 0:
|
||||
findings.append(
|
||||
Signal("MISSING_STRUCTURED_DATA", "No JSON-LD structured data present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("ogImage") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_OG_IMAGE", "No og:image meta tag present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not meta.get("favicon"):
|
||||
findings.append(
|
||||
Signal("MISSING_FAVICON", "No favicon link present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("htmlLang") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_HTML_LANG", "No html lang attribute present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
if not str(meta.get("canonical") or "").strip():
|
||||
findings.append(
|
||||
Signal("MISSING_CANONICAL", "No canonical link present", SEVERITY_WEAK, AXIS_QUALITY, 0.5, 0, "")
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
@dom_site_check
|
||||
def detect_duplicate_meta_description(site: DomSiteContext) -> list[Signal]:
|
||||
if len(site.pages) < 2:
|
||||
return []
|
||||
counts: dict[str, int] = {}
|
||||
for page in site.pages:
|
||||
description = str(_meta(page).get("description") or "").strip()
|
||||
if description:
|
||||
counts[description] = counts.get(description, 0) + 1
|
||||
for description, count in counts.items():
|
||||
if count >= 2:
|
||||
return [
|
||||
Signal(
|
||||
"DUPLICATE_META_DESCRIPTION",
|
||||
"Identical meta description reused across multiple pages",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.5,
|
||||
0,
|
||||
description[:150],
|
||||
)
|
||||
]
|
||||
return []
|
||||
@ -0,0 +1,35 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import (
|
||||
DOM_CHECKS,
|
||||
DOM_SITE_CHECKS,
|
||||
DomPageContext,
|
||||
DomSiteContext,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Signal
|
||||
from . import accessibility, builders, buildsignals, color, copy, layout, metaseo, typography
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_dom_checks(page: DomPageContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for func in DOM_CHECKS:
|
||||
try:
|
||||
findings.extend(func(page))
|
||||
except Exception as error:
|
||||
logger.warning("DOM check %s failed for %s: %s", func.__name__, page.url, error)
|
||||
return findings
|
||||
|
||||
|
||||
def run_dom_site_checks(site: DomSiteContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for func in DOM_SITE_CHECKS:
|
||||
try:
|
||||
findings.extend(func(site))
|
||||
except Exception as error:
|
||||
logger.warning("DOM site check %s failed: %s", func.__name__, error)
|
||||
return findings
|
||||
@ -0,0 +1,128 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.base import DomPageContext, dom_check
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DEFAULT_AI_FONTS: tuple[str, ...] = (
|
||||
"Inter",
|
||||
"Poppins",
|
||||
"Manrope",
|
||||
"Geist",
|
||||
"Space Grotesk",
|
||||
"DM Sans",
|
||||
"Plus Jakarta Sans",
|
||||
)
|
||||
DECORATIVE_MONOSPACE_FONTS: tuple[str, ...] = ("JetBrains Mono", "Space Mono")
|
||||
GOOGLE_FONTS_WEIGHTSET_PATTERN = re.compile(
|
||||
r"fonts\.googleapis\.com.*family=(Inter|Poppins|Manrope)[^&]*wght@400;500;600;700",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _detect_default_font(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
for candidate in DEFAULT_AI_FONTS:
|
||||
if candidate.lower() in family.lower():
|
||||
return Signal(
|
||||
"DEFAULT_AI_FONT",
|
||||
f"Default AI-tool font family in use ({candidate})",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_instrument_serif(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
if "instrument serif" in family.lower():
|
||||
return Signal(
|
||||
"INSTRUMENT_SERIF_ACCENT",
|
||||
"Instrument Serif accent font pairing",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_single_font_family(fonts: list[dict]) -> Signal | None:
|
||||
families = {str(sample.get("fontFamily", "")).strip() for sample in fonts if sample.get("fontFamily")}
|
||||
if len(fonts) >= 2 and len(families) == 1:
|
||||
return Signal(
|
||||
"SINGLE_FONT_FAMILY",
|
||||
"Every sampled element shares one exact font-family declaration",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
next(iter(families))[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_google_fonts_weightset(links: list[str]) -> Signal | None:
|
||||
for href in links:
|
||||
match = GOOGLE_FONTS_WEIGHTSET_PATTERN.search(str(href))
|
||||
if match:
|
||||
return Signal(
|
||||
"GOOGLE_FONTS_DEFAULT_WEIGHTSET",
|
||||
"Google Fonts request for the canonical AI-tool weight set (400;500;600;700)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.5,
|
||||
0,
|
||||
str(href)[:150],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _detect_decorative_monospace(fonts: list[dict]) -> Signal | None:
|
||||
for sample in fonts:
|
||||
tag = str(sample.get("tag", ""))
|
||||
if tag not in ("h1", "h2", "h3", "button"):
|
||||
continue
|
||||
family = str(sample.get("fontFamily", ""))
|
||||
for candidate in DECORATIVE_MONOSPACE_FONTS:
|
||||
if candidate.lower() in family.lower():
|
||||
return Signal(
|
||||
"DECORATIVE_MONOSPACE",
|
||||
f"Decorative monospace font used outside code ({candidate})",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_ORIGIN,
|
||||
0.5,
|
||||
0,
|
||||
family[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dom_check
|
||||
def detect_typography_signals(page: DomPageContext) -> list[Signal]:
|
||||
fonts = page.dom.get("fonts", []) or []
|
||||
links = page.dom.get("links", []) or []
|
||||
findings: list[Signal] = []
|
||||
for signal in (
|
||||
_detect_default_font(fonts),
|
||||
_detect_instrument_serif(fonts),
|
||||
_detect_single_font_family(fonts),
|
||||
_detect_google_fonts_weightset(links),
|
||||
_detect_decorative_monospace(fonts),
|
||||
):
|
||||
if signal is not None:
|
||||
findings.append(signal)
|
||||
return findings
|
||||
323
devplacepy/services/jobs/isslop/analysis/engine.py
Normal file
323
devplacepy/services/jobs/isslop/analysis/engine.py
Normal file
@ -0,0 +1,323 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.baselines import RepoBaselines, compute_baselines, deviation_signals
|
||||
from devplacepy.services.jobs.isslop.analysis.exclusion import exclusion_reason, is_excluded_directory, looks_minified
|
||||
from devplacepy.services.jobs.isslop.analysis.languages import CODE_LANGUAGES, DOCUMENT_LANGUAGES, extract_comments, language_for
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics, compute_metrics
|
||||
from devplacepy.services.jobs.isslop.analysis.scoring import (
|
||||
FileScore,
|
||||
categorize,
|
||||
criticality_for,
|
||||
fingerprint_origin_score,
|
||||
origin_score,
|
||||
quality_deficit_score,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import Detector, FileContext, RepoContext, run_detectors
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.documentation import detect_documentation
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.errors import detect_error_handling
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.hallucination import detect_hallucination
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.infrastructure import detect_infrastructure
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.language import detect_language_tells
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.llmdefaults import detect_llm_defaults
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.naming import detect_naming
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.scaffolds import detect_scaffolds
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.security import detect_security
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.structure import detect_structure
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.textual import detect_textual
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.vibeerrors import detect_vibe_errors
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.webtells import detect_web_tells
|
||||
from devplacepy.services.jobs.isslop.config import ANALYSIS_MAX_FILES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTORS: list[Detector] = [
|
||||
detect_textual,
|
||||
detect_naming,
|
||||
detect_structure,
|
||||
detect_hallucination,
|
||||
detect_error_handling,
|
||||
detect_security,
|
||||
detect_documentation,
|
||||
detect_language_tells,
|
||||
detect_web_tells,
|
||||
detect_llm_defaults,
|
||||
detect_infrastructure,
|
||||
detect_vibe_errors,
|
||||
detect_scaffolds,
|
||||
]
|
||||
|
||||
FINGERPRINT_DETECTORS: list[Detector] = [
|
||||
detect_textual,
|
||||
detect_web_tells,
|
||||
detect_llm_defaults,
|
||||
detect_infrastructure,
|
||||
detect_security,
|
||||
detect_vibe_errors,
|
||||
detect_scaffolds,
|
||||
]
|
||||
|
||||
PYPROJECT_DEPENDENCY_PATTERN = re.compile(r"[\"']([A-Za-z0-9._-]+)\s*(?:[><=!~\[;].*)?[\"']")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InventoryEntry:
|
||||
path: Path
|
||||
relative: str
|
||||
language: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Inventory:
|
||||
analyzable: list[InventoryEntry]
|
||||
excluded: list[tuple[str, str]]
|
||||
repo: RepoContext
|
||||
|
||||
|
||||
def _python_dependencies(root: Path) -> tuple[frozenset[str], bool]:
|
||||
dependencies: set[str] = set()
|
||||
found = False
|
||||
for name in ("requirements.txt", "requirements-dev.txt"):
|
||||
manifest = root / name
|
||||
if manifest.exists():
|
||||
found = True
|
||||
for line in manifest.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
cleaned = line.split("#")[0].strip()
|
||||
if cleaned and not cleaned.startswith("-"):
|
||||
dependencies.add(re.split(r"[><=!~\[;@ ]", cleaned)[0])
|
||||
pyproject = root / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
text = pyproject.read_text(encoding="utf-8", errors="replace")
|
||||
if "dependencies" in text or "[tool.poetry" in text:
|
||||
found = True
|
||||
block = re.search(r"dependencies\s*=\s*\[(.*?)\]", text, re.DOTALL)
|
||||
if block:
|
||||
for match in PYPROJECT_DEPENDENCY_PATTERN.finditer(block.group(1)):
|
||||
dependencies.add(match.group(1))
|
||||
poetry = re.search(r"\[tool\.poetry\.dependencies\](.*?)(?:\n\[|\Z)", text, re.DOTALL)
|
||||
if poetry:
|
||||
for line in poetry.group(1).splitlines():
|
||||
key = line.split("=")[0].strip().strip('"')
|
||||
if key and key != "python":
|
||||
dependencies.add(key)
|
||||
project_name = re.search(r"^\s*name\s*=\s*[\"']([\w.-]+)[\"']", text, re.MULTILINE)
|
||||
if project_name:
|
||||
dependencies.add(project_name.group(1))
|
||||
return frozenset(dependencies), found
|
||||
|
||||
|
||||
def _javascript_dependencies(root: Path) -> tuple[frozenset[str], bool]:
|
||||
manifest = root / "package.json"
|
||||
if not manifest.exists():
|
||||
return frozenset(), False
|
||||
try:
|
||||
body = json.loads(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError) as error:
|
||||
logger.warning("package.json unreadable: %s", error)
|
||||
return frozenset(), False
|
||||
dependencies: set[str] = set()
|
||||
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
||||
section = body.get(key)
|
||||
if isinstance(section, dict):
|
||||
dependencies.update(section.keys())
|
||||
name = body.get("name")
|
||||
if isinstance(name, str):
|
||||
dependencies.add(name)
|
||||
return frozenset(dependencies), True
|
||||
|
||||
|
||||
def _local_python_modules(root: Path) -> frozenset[str]:
|
||||
modules: set[str] = set()
|
||||
for entry in root.iterdir():
|
||||
if entry.is_dir() and not is_excluded_directory(entry.name):
|
||||
modules.add(entry.name)
|
||||
if (entry / "__init__.py").exists():
|
||||
modules.add(entry.name)
|
||||
elif entry.suffix == ".py":
|
||||
modules.add(entry.stem)
|
||||
for src in (root / "src", root / "lib"):
|
||||
if src.is_dir():
|
||||
for entry in src.iterdir():
|
||||
if entry.is_dir() or entry.suffix == ".py":
|
||||
modules.add(entry.stem if entry.is_file() else entry.name)
|
||||
return frozenset(modules)
|
||||
|
||||
|
||||
JSONC_TRAILING_COMMA_PATTERN = re.compile(r",(\s*[}\]])")
|
||||
|
||||
|
||||
def _strip_jsonc_comments(text: str) -> str:
|
||||
out: list[str] = []
|
||||
position = 0
|
||||
length = len(text)
|
||||
in_string = False
|
||||
escaped = False
|
||||
while position < length:
|
||||
char = text[position]
|
||||
if in_string:
|
||||
out.append(char)
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
position += 1
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
out.append(char)
|
||||
position += 1
|
||||
continue
|
||||
if char == "/" and text[position + 1 : position + 2] == "/":
|
||||
while position < length and text[position] != "\n":
|
||||
position += 1
|
||||
continue
|
||||
if char == "/" and text[position + 1 : position + 2] == "*":
|
||||
position += 2
|
||||
while position + 1 < length and not (text[position] == "*" and text[position + 1] == "/"):
|
||||
position += 1
|
||||
position += 2
|
||||
continue
|
||||
out.append(char)
|
||||
position += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _parse_jsonc(text: str) -> dict:
|
||||
cleaned = _strip_jsonc_comments(text)
|
||||
cleaned = JSONC_TRAILING_COMMA_PATTERN.sub(r"\1", cleaned)
|
||||
parsed = json.loads(cleaned)
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _javascript_alias_prefixes(root: Path) -> frozenset[str]:
|
||||
prefixes: set[str] = set()
|
||||
for name in ("tsconfig.json", "jsconfig.json"):
|
||||
manifest = root / name
|
||||
if not manifest.exists():
|
||||
continue
|
||||
try:
|
||||
body = _parse_jsonc(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError) as error:
|
||||
logger.warning("%s unreadable: %s", name, error)
|
||||
continue
|
||||
options = body.get("compilerOptions")
|
||||
paths = options.get("paths") if isinstance(options, dict) else None
|
||||
if not isinstance(paths, dict):
|
||||
continue
|
||||
for alias in paths:
|
||||
prefix = str(alias).rstrip("*")
|
||||
if prefix:
|
||||
prefixes.add(prefix)
|
||||
return frozenset(prefixes)
|
||||
|
||||
|
||||
def build_repo_context(root: Path) -> RepoContext:
|
||||
python_deps, has_python = _python_dependencies(root)
|
||||
javascript_deps, has_javascript = _javascript_dependencies(root)
|
||||
return RepoContext(
|
||||
root=root,
|
||||
python_dependencies=python_deps,
|
||||
javascript_dependencies=javascript_deps,
|
||||
local_python_modules=_local_python_modules(root),
|
||||
has_python_manifest=has_python,
|
||||
has_javascript_manifest=has_javascript,
|
||||
javascript_alias_prefixes=_javascript_alias_prefixes(root),
|
||||
)
|
||||
|
||||
|
||||
def build_inventory(workspace: Path) -> Inventory:
|
||||
analyzable: list[InventoryEntry] = []
|
||||
excluded: list[tuple[str, str]] = []
|
||||
candidates = sorted(
|
||||
entry for entry in workspace.rglob("*") if entry.is_file() and not entry.is_symlink()
|
||||
)
|
||||
for path in candidates:
|
||||
relative = str(path.relative_to(workspace))
|
||||
reason = exclusion_reason(path, workspace)
|
||||
if reason:
|
||||
excluded.append((relative, reason))
|
||||
continue
|
||||
language = language_for(path.name)
|
||||
if language not in CODE_LANGUAGES and language not in DOCUMENT_LANGUAGES:
|
||||
excluded.append((relative, f"unsupported language ({language})"))
|
||||
continue
|
||||
analyzable.append(InventoryEntry(path=path, relative=relative, language=language))
|
||||
if len(analyzable) >= ANALYSIS_MAX_FILES:
|
||||
logger.warning("File cap of %d reached, remaining files skipped", ANALYSIS_MAX_FILES)
|
||||
break
|
||||
repo = build_repo_context(workspace)
|
||||
logger.info("Inventory: %d analyzable, %d excluded", len(analyzable), len(excluded))
|
||||
return Inventory(analyzable=analyzable, excluded=excluded, repo=repo)
|
||||
|
||||
|
||||
def load_context(entry: InventoryEntry, repo: RepoContext) -> FileContext | None:
|
||||
try:
|
||||
text = entry.path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as error:
|
||||
logger.warning("Read failed for %s: %s", entry.relative, error)
|
||||
return None
|
||||
fingerprint_only = looks_minified(text)
|
||||
lines = text.splitlines()
|
||||
context = FileContext(
|
||||
path=entry.path,
|
||||
relative=entry.relative,
|
||||
language=entry.language,
|
||||
text=text,
|
||||
lines=lines,
|
||||
comments=extract_comments(entry.language, lines),
|
||||
repo=repo,
|
||||
fingerprint_only=fingerprint_only,
|
||||
)
|
||||
context.metrics = compute_metrics(entry.language, text)
|
||||
return context
|
||||
|
||||
|
||||
def score_file(context: FileContext, baselines: RepoBaselines) -> FileScore:
|
||||
metrics = context.metrics
|
||||
if metrics is None:
|
||||
metrics = compute_metrics(context.language, context.text)
|
||||
context.metrics = metrics
|
||||
if context.fingerprint_only:
|
||||
signals = run_detectors(context, FINGERPRINT_DETECTORS)
|
||||
origin = fingerprint_origin_score(signals)
|
||||
quality = quality_deficit_score(signals, metrics)
|
||||
effective_sloc = max(metrics.sloc, min(metrics.total_lines, 20))
|
||||
return FileScore(
|
||||
relative=context.relative,
|
||||
language=context.language,
|
||||
sloc=effective_sloc,
|
||||
origin_score=origin,
|
||||
quality_deficit=quality,
|
||||
category=categorize(origin, quality),
|
||||
criticality=criticality_for(context.relative),
|
||||
signals=signals,
|
||||
)
|
||||
signals = run_detectors(context, DETECTORS)
|
||||
signals.extend(deviation_signals(context.relative, metrics, baselines))
|
||||
origin = origin_score(context.text, metrics, signals)
|
||||
quality = quality_deficit_score(signals, metrics)
|
||||
return FileScore(
|
||||
relative=context.relative,
|
||||
language=context.language,
|
||||
sloc=metrics.sloc,
|
||||
origin_score=origin,
|
||||
quality_deficit=quality,
|
||||
category=categorize(origin, quality),
|
||||
criticality=criticality_for(context.relative),
|
||||
signals=signals,
|
||||
)
|
||||
|
||||
|
||||
def compute_repo_baselines(contexts: list[FileContext]) -> RepoBaselines:
|
||||
metrics_by_file: dict[str, FileMetrics] = {
|
||||
context.relative: context.metrics for context in contexts if context.metrics is not None
|
||||
}
|
||||
return compute_baselines(metrics_by_file)
|
||||
142
devplacepy/services/jobs/isslop/analysis/exclusion.py
Normal file
142
devplacepy/services/jobs/isslop/analysis/exclusion.py
Normal file
@ -0,0 +1,142 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from devplacepy.services.jobs.isslop.config import ANALYSIS_FILE_CAP_BYTES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EXCLUDED_DIRECTORIES: frozenset[str] = frozenset(
|
||||
{
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
"node_modules",
|
||||
"bower_components",
|
||||
"vendor",
|
||||
"third_party",
|
||||
"thirdparty",
|
||||
"__pycache__",
|
||||
".venv",
|
||||
"venv",
|
||||
"env",
|
||||
".env",
|
||||
"dist",
|
||||
"build",
|
||||
"out",
|
||||
"target",
|
||||
"coverage",
|
||||
".next",
|
||||
".nuxt",
|
||||
".cache",
|
||||
"site-packages",
|
||||
".idea",
|
||||
".vscode",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
".pytest_cache",
|
||||
".terraform",
|
||||
"migrations",
|
||||
}
|
||||
)
|
||||
|
||||
LOCKFILE_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
"package-lock.json",
|
||||
"yarn.lock",
|
||||
"pnpm-lock.yaml",
|
||||
"poetry.lock",
|
||||
"pipfile.lock",
|
||||
"gemfile.lock",
|
||||
"go.sum",
|
||||
"cargo.lock",
|
||||
"composer.lock",
|
||||
"uv.lock",
|
||||
"bun.lockb",
|
||||
}
|
||||
)
|
||||
|
||||
GENERATED_SUFFIXES: tuple[str, ...] = (
|
||||
".min.js",
|
||||
".min.css",
|
||||
".map",
|
||||
".pb.go",
|
||||
"_pb2.py",
|
||||
"_pb2_grpc.py",
|
||||
".bundle.js",
|
||||
".chunk.js",
|
||||
".d.ts",
|
||||
)
|
||||
|
||||
BINARY_EXTENSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".bmp", ".ico", ".tiff", ".svgz",
|
||||
".woff", ".woff2", ".ttf", ".otf", ".eot",
|
||||
".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
|
||||
".mp3", ".mp4", ".wav", ".ogg", ".webm", ".avi", ".mov", ".flac",
|
||||
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||
".exe", ".dll", ".so", ".dylib", ".bin", ".dat", ".db", ".sqlite", ".sqlite3",
|
||||
".pyc", ".pyo", ".class", ".jar", ".war", ".wasm", ".o", ".a",
|
||||
}
|
||||
)
|
||||
|
||||
GENERATED_MARKERS: tuple[str, ...] = (
|
||||
"do not edit",
|
||||
"auto-generated",
|
||||
"autogenerated",
|
||||
"@generated",
|
||||
"code generated by",
|
||||
"this file was generated",
|
||||
)
|
||||
|
||||
MINIFIED_AVG_LINE_LENGTH: int = 300
|
||||
|
||||
|
||||
def is_excluded_directory(part: str) -> bool:
|
||||
return part.lower() in EXCLUDED_DIRECTORIES
|
||||
|
||||
|
||||
def exclusion_reason(path: Path, workspace: Path) -> Optional[str]:
|
||||
relative = path.relative_to(workspace)
|
||||
for part in relative.parts[:-1]:
|
||||
if is_excluded_directory(part):
|
||||
return f"vendored or generated directory: {part}"
|
||||
name = path.name.lower()
|
||||
if name in LOCKFILE_NAMES:
|
||||
return "lockfile"
|
||||
for suffix in GENERATED_SUFFIXES:
|
||||
if name.endswith(suffix):
|
||||
return f"generated artifact ({suffix})"
|
||||
if path.suffix.lower() in BINARY_EXTENSIONS:
|
||||
return "binary file"
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError as error:
|
||||
return f"unreadable: {error}"
|
||||
if size == 0:
|
||||
return "empty file"
|
||||
if size > ANALYSIS_FILE_CAP_BYTES:
|
||||
return f"oversize ({size} bytes)"
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
head = handle.read(8192)
|
||||
except OSError as error:
|
||||
return f"unreadable: {error}"
|
||||
if b"\x00" in head:
|
||||
return "binary content"
|
||||
text_head = head.decode("utf-8", errors="replace").lower()
|
||||
for marker in GENERATED_MARKERS:
|
||||
if marker in text_head[:600]:
|
||||
return f"generated marker: {marker}"
|
||||
return None
|
||||
|
||||
|
||||
def looks_minified(text: str) -> bool:
|
||||
lines = [line for line in text.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return False
|
||||
average = sum(len(line) for line in lines) / len(lines)
|
||||
return average > MINIFIED_AVG_LINE_LENGTH
|
||||
135
devplacepy/services/jobs/isslop/analysis/languages.py
Normal file
135
devplacepy/services/jobs/isslop/analysis/languages.py
Normal file
@ -0,0 +1,135 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
LANGUAGE_BY_EXTENSION: dict[str, str] = {
|
||||
".py": "python",
|
||||
".pyw": "python",
|
||||
".js": "javascript",
|
||||
".mjs": "javascript",
|
||||
".cjs": "javascript",
|
||||
".jsx": "javascript",
|
||||
".ts": "typescript",
|
||||
".tsx": "typescript",
|
||||
".html": "html",
|
||||
".htm": "html",
|
||||
".css": "css",
|
||||
".scss": "css",
|
||||
".less": "css",
|
||||
".php": "php",
|
||||
".go": "go",
|
||||
".java": "java",
|
||||
".kt": "java",
|
||||
".c": "c",
|
||||
".h": "c",
|
||||
".cpp": "cpp",
|
||||
".cc": "cpp",
|
||||
".hpp": "cpp",
|
||||
".cs": "csharp",
|
||||
".rb": "ruby",
|
||||
".rs": "rust",
|
||||
".sh": "shell",
|
||||
".bash": "shell",
|
||||
".lua": "lua",
|
||||
".sql": "sql",
|
||||
".md": "markdown",
|
||||
".rst": "markdown",
|
||||
".txt": "text",
|
||||
".json": "json",
|
||||
".yaml": "yaml",
|
||||
".yml": "yaml",
|
||||
".toml": "toml",
|
||||
".xml": "xml",
|
||||
".vue": "javascript",
|
||||
".svelte": "javascript",
|
||||
}
|
||||
|
||||
CODE_LANGUAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"python",
|
||||
"javascript",
|
||||
"typescript",
|
||||
"php",
|
||||
"go",
|
||||
"java",
|
||||
"c",
|
||||
"cpp",
|
||||
"csharp",
|
||||
"ruby",
|
||||
"rust",
|
||||
"shell",
|
||||
"lua",
|
||||
"sql",
|
||||
"html",
|
||||
"css",
|
||||
}
|
||||
)
|
||||
|
||||
DOCUMENT_LANGUAGES: frozenset[str] = frozenset({"markdown", "text"})
|
||||
|
||||
LINE_COMMENT_PREFIXES: dict[str, tuple[str, ...]] = {
|
||||
"python": ("#",),
|
||||
"shell": ("#",),
|
||||
"ruby": ("#",),
|
||||
"yaml": ("#",),
|
||||
"toml": ("#",),
|
||||
"javascript": ("//",),
|
||||
"typescript": ("//",),
|
||||
"php": ("//", "#"),
|
||||
"go": ("//",),
|
||||
"java": ("//",),
|
||||
"c": ("//",),
|
||||
"cpp": ("//",),
|
||||
"csharp": ("//",),
|
||||
"rust": ("//",),
|
||||
"sql": ("--",),
|
||||
"lua": ("--",),
|
||||
}
|
||||
|
||||
BLOCK_COMMENT_LANGUAGES: frozenset[str] = frozenset(
|
||||
{"javascript", "typescript", "php", "go", "java", "c", "cpp", "csharp", "rust", "css"}
|
||||
)
|
||||
|
||||
|
||||
def language_for(filename: str) -> str:
|
||||
lowered = filename.lower()
|
||||
for extension, language in LANGUAGE_BY_EXTENSION.items():
|
||||
if lowered.endswith(extension):
|
||||
return language
|
||||
return "unknown"
|
||||
|
||||
|
||||
def extract_comments(language: str, lines: list[str]) -> list[tuple[int, str]]:
|
||||
comments: list[tuple[int, str]] = []
|
||||
prefixes = LINE_COMMENT_PREFIXES.get(language, ())
|
||||
in_block = False
|
||||
block_open, block_close = ("/*", "*/") if language in BLOCK_COMMENT_LANGUAGES or language == "css" else ("", "")
|
||||
if language == "html":
|
||||
block_open, block_close = "<!--", "-->"
|
||||
for number, raw in enumerate(lines, start=1):
|
||||
stripped = raw.strip()
|
||||
if in_block:
|
||||
comments.append((number, stripped.replace(block_close, "").strip()))
|
||||
if block_close and block_close in stripped:
|
||||
in_block = False
|
||||
continue
|
||||
if block_open and block_open in stripped:
|
||||
fragment = stripped.split(block_open, 1)[1]
|
||||
comments.append((number, fragment.replace(block_close, "").strip()))
|
||||
if block_close not in fragment:
|
||||
in_block = True
|
||||
continue
|
||||
for prefix in prefixes:
|
||||
if stripped.startswith(prefix):
|
||||
comments.append((number, stripped[len(prefix):].strip()))
|
||||
break
|
||||
marker = f" {prefix}"
|
||||
if marker in raw and not _inside_string(raw, raw.index(marker)):
|
||||
comments.append((number, raw.split(marker, 1)[1].strip()))
|
||||
break
|
||||
return comments
|
||||
|
||||
|
||||
def _inside_string(line: str, position: int) -> bool:
|
||||
double = line.count('"', 0, position) % 2 == 1
|
||||
single = line.count("'", 0, position) % 2 == 1
|
||||
return double or single
|
||||
163
devplacepy/services/jobs/isslop/analysis/metrics.py
Normal file
163
devplacepy/services/jobs/isslop/analysis/metrics.py
Normal file
@ -0,0 +1,163 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.languages import extract_comments
|
||||
|
||||
COMPLEXITY_TOKENS = re.compile(
|
||||
r"\b(if|elif|else if|for|while|case|when|catch|except|and|or)\b|&&|\|\||\?\s"
|
||||
)
|
||||
FUNCTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||
"python": re.compile(r"^\s*(?:async\s+)?def\s+(\w+)", re.MULTILINE),
|
||||
"javascript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)|^\s*(\w+)\s*\([^)]*\)\s*{", re.MULTILINE),
|
||||
"typescript": re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)|^\s*(\w+)\s*\([^)]*\)\s*{", re.MULTILINE),
|
||||
"go": re.compile(r"^\s*func\s+(?:\([^)]+\)\s*)?(\w+)", re.MULTILINE),
|
||||
"java": re.compile(r"^\s*(?:public|private|protected|static|\s)+[\w<>\[\]]+\s+(\w+)\s*\(", re.MULTILINE),
|
||||
"php": re.compile(r"^\s*(?:public|private|protected|static|\s)*function\s+(\w+)", re.MULTILINE),
|
||||
"ruby": re.compile(r"^\s*def\s+(\w+)", re.MULTILINE),
|
||||
"rust": re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)", re.MULTILINE),
|
||||
}
|
||||
DOCSTRING_PATTERN = re.compile(r"def\s+\w+[^:]*:\s*\n\s*(?:\"\"\"|''')", re.MULTILINE)
|
||||
TOKEN_PATTERN = re.compile(r"[A-Za-z_]\w*|\d+|[^\sA-Za-z0-9_]")
|
||||
DUPLICATION_WINDOW: int = 6
|
||||
MAINTAINABILITY_CEILING: float = 171.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileMetrics:
|
||||
total_lines: int
|
||||
sloc: int
|
||||
blank_lines: int
|
||||
comment_lines: int
|
||||
blank_ratio: float
|
||||
comment_ratio: float
|
||||
avg_line_length: float
|
||||
max_line_length: int
|
||||
avg_leading_spaces: float
|
||||
avg_leading_tabs: float
|
||||
indent_variance: float
|
||||
function_count: int
|
||||
docstring_count: int
|
||||
max_function_length: int
|
||||
cyclomatic_total: int
|
||||
cyclomatic_max_estimate: int
|
||||
duplication_ratio: float
|
||||
maintainability_index: float
|
||||
import_count: int
|
||||
|
||||
|
||||
def shannon_entropy(value: str) -> float:
|
||||
if not value:
|
||||
return 0.0
|
||||
counts: dict[str, int] = {}
|
||||
for char in value:
|
||||
counts[char] = counts.get(char, 0) + 1
|
||||
total = len(value)
|
||||
return -sum((count / total) * math.log2(count / total) for count in counts.values())
|
||||
|
||||
|
||||
def _leading_whitespace(lines: list[str]) -> tuple[float, float, float]:
|
||||
spaces: list[int] = []
|
||||
tabs: list[int] = []
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
space_count = len(line) - len(line.lstrip(" "))
|
||||
tab_count = len(line) - len(line.lstrip("\t"))
|
||||
spaces.append(space_count)
|
||||
tabs.append(tab_count)
|
||||
if not spaces:
|
||||
return 0.0, 0.0, 0.0
|
||||
mean_spaces = sum(spaces) / len(spaces)
|
||||
mean_tabs = sum(tabs) / len(tabs)
|
||||
variance = sum((value - mean_spaces) ** 2 for value in spaces) / len(spaces)
|
||||
return mean_spaces, mean_tabs, math.sqrt(variance)
|
||||
|
||||
|
||||
def _duplication_ratio(lines: list[str]) -> float:
|
||||
normalized = [re.sub(r"\s+", " ", line.strip()) for line in lines if line.strip()]
|
||||
if len(normalized) < DUPLICATION_WINDOW * 2:
|
||||
return 0.0
|
||||
seen: dict[str, int] = {}
|
||||
duplicated = 0
|
||||
windows = 0
|
||||
for index in range(len(normalized) - DUPLICATION_WINDOW + 1):
|
||||
window = "\n".join(normalized[index:index + DUPLICATION_WINDOW])
|
||||
if len(window) < 60:
|
||||
continue
|
||||
digest = hashlib.sha1(window.encode("utf-8")).hexdigest()
|
||||
windows += 1
|
||||
if digest in seen:
|
||||
duplicated += 1
|
||||
seen[digest] = index
|
||||
return duplicated / windows if windows else 0.0
|
||||
|
||||
|
||||
def _function_lengths(language: str, lines: list[str]) -> tuple[int, int]:
|
||||
pattern = FUNCTION_PATTERNS.get(language)
|
||||
if pattern is None:
|
||||
return 0, 0
|
||||
starts: list[int] = []
|
||||
for number, line in enumerate(lines):
|
||||
if pattern.match(line):
|
||||
starts.append(number)
|
||||
if not starts:
|
||||
return 0, 0
|
||||
lengths: list[int] = []
|
||||
for index, start in enumerate(starts):
|
||||
end = starts[index + 1] if index + 1 < len(starts) else len(lines)
|
||||
lengths.append(end - start)
|
||||
return len(starts), max(lengths)
|
||||
|
||||
|
||||
def _maintainability(sloc: int, cyclomatic: int, tokens: list[str]) -> float:
|
||||
if sloc <= 0 or not tokens:
|
||||
return 100.0
|
||||
unique = len(set(tokens))
|
||||
volume = len(tokens) * math.log2(max(unique, 2))
|
||||
raw = MAINTAINABILITY_CEILING - 5.2 * math.log(max(volume, 1.0)) - 0.23 * cyclomatic - 16.2 * math.log(max(sloc, 1))
|
||||
return max(0.0, raw * 100.0 / MAINTAINABILITY_CEILING)
|
||||
|
||||
|
||||
def compute_metrics(language: str, text: str) -> FileMetrics:
|
||||
lines = text.splitlines()
|
||||
total = len(lines)
|
||||
blank = sum(1 for line in lines if not line.strip())
|
||||
comments = extract_comments(language, lines)
|
||||
comment_lines = len(comments)
|
||||
sloc = max(0, total - blank - comment_lines)
|
||||
lengths = [len(line) for line in lines if line.strip()]
|
||||
avg_length = sum(lengths) / len(lengths) if lengths else 0.0
|
||||
max_length = max(lengths) if lengths else 0
|
||||
mean_spaces, mean_tabs, indent_deviation = _leading_whitespace(lines)
|
||||
function_count, max_function_length = _function_lengths(language, lines)
|
||||
docstring_count = len(DOCSTRING_PATTERN.findall(text)) if language == "python" else 0
|
||||
cyclomatic = len(COMPLEXITY_TOKENS.findall(text))
|
||||
per_function = cyclomatic // function_count if function_count else cyclomatic
|
||||
tokens = TOKEN_PATTERN.findall(text)[:20000]
|
||||
import_count = len(re.findall(r"^\s*(?:import|from|require|use|#include)\b", text, re.MULTILINE))
|
||||
return FileMetrics(
|
||||
total_lines=total,
|
||||
sloc=sloc,
|
||||
blank_lines=blank,
|
||||
comment_lines=comment_lines,
|
||||
blank_ratio=blank / total if total else 0.0,
|
||||
comment_ratio=comment_lines / total if total else 0.0,
|
||||
avg_line_length=avg_length,
|
||||
max_line_length=max_length,
|
||||
avg_leading_spaces=mean_spaces,
|
||||
avg_leading_tabs=mean_tabs,
|
||||
indent_variance=indent_deviation,
|
||||
function_count=function_count,
|
||||
docstring_count=docstring_count,
|
||||
max_function_length=max_function_length,
|
||||
cyclomatic_total=cyclomatic,
|
||||
cyclomatic_max_estimate=per_function,
|
||||
duplication_ratio=_duplication_ratio(lines),
|
||||
maintainability_index=_maintainability(sloc, per_function, tokens),
|
||||
import_count=import_count,
|
||||
)
|
||||
302
devplacepy/services/jobs/isslop/analysis/scoring.py
Normal file
302
devplacepy/services/jobs/isslop/analysis/scoring.py
Normal file
@ -0,0 +1,302 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.domsignals.aggregate import DomEvidence
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_FACTORS,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
Signal,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.config import DOM_AI_WEIGHT
|
||||
|
||||
CATEGORY_SLOP: str = "ai-slop"
|
||||
CATEGORY_SOPHISTICATED: str = "sophisticated-ai"
|
||||
CATEGORY_HUMAN_CLEAN: str = "human-clean"
|
||||
CATEGORY_HUMAN_MESSY: str = "human-messy"
|
||||
CATEGORY_UNCERTAIN: str = "uncertain"
|
||||
|
||||
ORIGIN_HIGH_THRESHOLD: float = 55.0
|
||||
ORIGIN_LOW_THRESHOLD: float = 45.0
|
||||
QUALITY_SLOP_THRESHOLD: float = 50.0
|
||||
QUALITY_CLEAN_THRESHOLD: float = 30.0
|
||||
QUALITY_SATURATION: float = 14.0
|
||||
ORIGIN_SIGNAL_CAP: float = 45.0
|
||||
ORIGIN_SIGNAL_SCALE: float = 4.5
|
||||
|
||||
CRITICAL_PATH_PATTERN = re.compile(r"auth|login|password|payment|billing|crypto|security|token|session|admin", re.IGNORECASE)
|
||||
CRITICALITY_WEIGHT: float = 1.5
|
||||
|
||||
GRADE_BANDS: tuple[tuple[float, str], ...] = ((15.0, "A"), (30.0, "B"), (50.0, "C"), (70.0, "D"), (100.0, "F"))
|
||||
|
||||
AI_SCORE_WEIGHT: float = 0.68
|
||||
QUALITY_SCORE_WEIGHT: float = 0.32
|
||||
|
||||
AI_RAMP_LOW: float = 35.0
|
||||
AI_RAMP_HIGH: float = 65.0
|
||||
|
||||
IMAGE_TEXT_AI_WEIGHT: float = 0.75
|
||||
IMAGE_AI_WEIGHT: float = 0.25
|
||||
|
||||
|
||||
def compose_slop_score(ai_percent: float, quality_deficit: float) -> float:
|
||||
return round(AI_SCORE_WEIGHT * ai_percent + QUALITY_SCORE_WEIGHT * quality_deficit, 1)
|
||||
|
||||
|
||||
def ai_fraction(origin: float) -> float:
|
||||
if origin <= AI_RAMP_LOW:
|
||||
return 0.0
|
||||
if origin >= AI_RAMP_HIGH:
|
||||
return 1.0
|
||||
position = (origin - AI_RAMP_LOW) / (AI_RAMP_HIGH - AI_RAMP_LOW)
|
||||
return position * position * (3.0 - 2.0 * position)
|
||||
|
||||
HUMAN_MARKER_PATTERN = re.compile(r"\b(HACK|FIXME|XXX)\b.*[A-Z]{2,}-\d+|\bFIXME\(\w+\)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileScore:
|
||||
relative: str
|
||||
language: str
|
||||
sloc: int
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
category: str
|
||||
criticality: float
|
||||
signals: list[Signal] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoScores:
|
||||
origin_score: float
|
||||
quality_deficit: float
|
||||
slop_score: float
|
||||
grade: str
|
||||
category: str
|
||||
human_percent: float
|
||||
ai_percent: float
|
||||
confidence: str
|
||||
strong_signal_count: int
|
||||
medium_signal_count: int
|
||||
files_scored: int
|
||||
|
||||
|
||||
def quality_deficit_score(signals: list[Signal], metrics: FileMetrics) -> float:
|
||||
weighted = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis != AXIS_ORIGIN
|
||||
)
|
||||
if metrics.maintainability_index < 40.0:
|
||||
weighted += 2.0
|
||||
elif metrics.maintainability_index < 65.0:
|
||||
weighted += 1.0
|
||||
return round(100.0 * (1.0 - math.exp(-weighted / QUALITY_SATURATION)), 1)
|
||||
|
||||
|
||||
FINGERPRINT_ORIGIN_BASE: float = 50.0
|
||||
FINGERPRINT_ORIGIN_SCALE: float = 7.0
|
||||
|
||||
|
||||
def fingerprint_origin_score(signals: list[Signal]) -> float:
|
||||
origin_weight = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis == AXIS_ORIGIN
|
||||
)
|
||||
if origin_weight <= 0.0:
|
||||
return FINGERPRINT_ORIGIN_BASE
|
||||
score = FINGERPRINT_ORIGIN_BASE + origin_weight * FINGERPRINT_ORIGIN_SCALE
|
||||
if any(signal.code in ("AI_BUILDER_FINGERPRINT", "VIBE_STACK") for signal in signals):
|
||||
score += 12.0
|
||||
return round(min(100.0, score), 1)
|
||||
|
||||
|
||||
def origin_score(text: str, metrics: FileMetrics, signals: list[Signal]) -> float:
|
||||
score = 0.0
|
||||
if metrics.sloc >= 40:
|
||||
if metrics.indent_variance < 0.45:
|
||||
score += 16.0
|
||||
elif metrics.indent_variance < 1.2:
|
||||
score += 8.0
|
||||
elif metrics.indent_variance > 4.0:
|
||||
score -= 14.0
|
||||
if 0.10 <= metrics.blank_ratio <= 0.22 and metrics.total_lines >= 60:
|
||||
score += 6.0
|
||||
if metrics.comment_ratio >= 0.22 and metrics.sloc >= 40:
|
||||
score += 8.0
|
||||
if metrics.function_count >= 4 and metrics.docstring_count >= metrics.function_count:
|
||||
score += 10.0
|
||||
origin_weight = sum(
|
||||
signal.weight * SEVERITY_FACTORS.get(signal.severity, 0.3)
|
||||
for signal in signals
|
||||
if signal.axis == AXIS_ORIGIN
|
||||
)
|
||||
score += min(ORIGIN_SIGNAL_CAP, origin_weight * ORIGIN_SIGNAL_SCALE)
|
||||
if any(signal.code == "AI_SIGNATURE" for signal in signals):
|
||||
score += 22.0
|
||||
if any(signal.code == "NAMING_MIXED" for signal in signals):
|
||||
score -= 8.0
|
||||
if HUMAN_MARKER_PATTERN.search(text):
|
||||
score -= 10.0
|
||||
score += 24.0
|
||||
return round(min(100.0, max(0.0, score)), 1)
|
||||
|
||||
|
||||
def categorize(origin: float, quality: float) -> str:
|
||||
if origin >= ORIGIN_HIGH_THRESHOLD and quality >= QUALITY_SLOP_THRESHOLD:
|
||||
return CATEGORY_SLOP
|
||||
if origin >= ORIGIN_HIGH_THRESHOLD and quality < QUALITY_CLEAN_THRESHOLD:
|
||||
return CATEGORY_SOPHISTICATED
|
||||
if origin < ORIGIN_LOW_THRESHOLD:
|
||||
return CATEGORY_HUMAN_MESSY if quality >= QUALITY_SLOP_THRESHOLD else CATEGORY_HUMAN_CLEAN
|
||||
return CATEGORY_UNCERTAIN
|
||||
|
||||
|
||||
def criticality_for(relative: str) -> float:
|
||||
return CRITICALITY_WEIGHT if CRITICAL_PATH_PATTERN.search(relative) else 1.0
|
||||
|
||||
|
||||
def grade_for(slop: float) -> str:
|
||||
for ceiling, grade in GRADE_BANDS:
|
||||
if slop <= ceiling:
|
||||
return grade
|
||||
return "F"
|
||||
|
||||
|
||||
def _confidence(strong: int, medium: int, files: int) -> str:
|
||||
if files < 3:
|
||||
return "low"
|
||||
if strong >= 3:
|
||||
return "high"
|
||||
if strong >= 1 or medium >= 5:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def aggregate(files: list[FileScore]) -> RepoScores:
|
||||
scored = [entry for entry in files if entry.sloc > 0]
|
||||
if not scored:
|
||||
return RepoScores(0.0, 0.0, 0.0, "A", CATEGORY_UNCERTAIN, 50.0, 50.0, "low", 0, 0, 0)
|
||||
total_weight = sum(entry.sloc * entry.criticality for entry in scored)
|
||||
origin = sum(entry.origin_score * entry.sloc * entry.criticality for entry in scored) / total_weight
|
||||
quality = sum(entry.quality_deficit * entry.sloc * entry.criticality for entry in scored) / total_weight
|
||||
ai_mass = sum(entry.sloc * entry.criticality * ai_fraction(entry.origin_score) for entry in scored)
|
||||
ai_percent = round(100.0 * ai_mass / total_weight, 1)
|
||||
slop = compose_slop_score(ai_percent, quality)
|
||||
strong = sum(1 for entry in scored for signal in entry.signals if signal.severity == SEVERITY_STRONG)
|
||||
medium = sum(1 for entry in scored for signal in entry.signals if signal.severity == SEVERITY_MEDIUM)
|
||||
return RepoScores(
|
||||
origin_score=round(origin, 1),
|
||||
quality_deficit=round(quality, 1),
|
||||
slop_score=round(slop, 1),
|
||||
grade=grade_for(slop),
|
||||
category=categorize(origin, quality),
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
ai_percent=ai_percent,
|
||||
confidence=_confidence(strong, medium, len(scored)),
|
||||
strong_signal_count=strong,
|
||||
medium_signal_count=medium,
|
||||
files_scored=len(scored),
|
||||
)
|
||||
|
||||
|
||||
TEMPLATE_CONFIDENT_SCORE: float = 35.0
|
||||
TEMPLATE_STRONG_SCORE: float = 70.0
|
||||
TEMPLATE_INFLUENCE: float = 0.7
|
||||
TEMPLATE_STRONG_INFLUENCE: float = 0.85
|
||||
|
||||
DOM_BUILDER_CONFIDENT_THRESHOLD: float = 0.75
|
||||
DOM_BUILDER_FLOOR: float = 70.0
|
||||
|
||||
|
||||
def _force_slop_category(scores: RepoScores, floor: float) -> RepoScores:
|
||||
ai_percent = max(scores.ai_percent, floor)
|
||||
origin = max(scores.origin_score, floor)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
origin_score=origin,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
category=CATEGORY_SLOP,
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_template(scores: RepoScores, template_score: float) -> RepoScores:
|
||||
if template_score < TEMPLATE_CONFIDENT_SCORE:
|
||||
return scores
|
||||
if template_score >= TEMPLATE_STRONG_SCORE:
|
||||
floor = round(TEMPLATE_STRONG_INFLUENCE * template_score, 1)
|
||||
return _force_slop_category(scores, floor)
|
||||
floor = round(TEMPLATE_INFLUENCE * template_score, 1)
|
||||
ai_percent = max(scores.ai_percent, floor)
|
||||
origin = max(scores.origin_score, floor)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
category = categorize(origin, scores.quality_deficit)
|
||||
if category in (CATEGORY_HUMAN_CLEAN, CATEGORY_HUMAN_MESSY):
|
||||
category = CATEGORY_UNCERTAIN
|
||||
return replace(
|
||||
scores,
|
||||
origin_score=origin,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
category=category,
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_dom_signals(scores: RepoScores, dom: DomEvidence) -> RepoScores:
|
||||
if dom.detected_builder is not None and dom.builder_confidence >= DOM_BUILDER_CONFIDENT_THRESHOLD:
|
||||
return _force_slop_category(scores, DOM_BUILDER_FLOOR)
|
||||
if dom.score <= 0:
|
||||
return scores
|
||||
ai_percent = round((1.0 - DOM_AI_WEIGHT) * scores.ai_percent + DOM_AI_WEIGHT * dom.score, 1)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
)
|
||||
|
||||
|
||||
def adjust_for_images(scores: RepoScores, mean_image_ai: float) -> RepoScores:
|
||||
ai_percent = round(
|
||||
IMAGE_TEXT_AI_WEIGHT * scores.ai_percent + IMAGE_AI_WEIGHT * mean_image_ai, 1
|
||||
)
|
||||
slop = compose_slop_score(ai_percent, scores.quality_deficit)
|
||||
return replace(
|
||||
scores,
|
||||
ai_percent=ai_percent,
|
||||
human_percent=round(100.0 - ai_percent, 1),
|
||||
slop_score=slop,
|
||||
grade=grade_for(slop),
|
||||
)
|
||||
|
||||
|
||||
def repo_scores_to_dict(scores: RepoScores) -> dict[str, Any]:
|
||||
return {
|
||||
"origin_score": scores.origin_score,
|
||||
"quality_deficit": scores.quality_deficit,
|
||||
"slop_score": scores.slop_score,
|
||||
"grade": scores.grade,
|
||||
"category": scores.category,
|
||||
"human_percent": scores.human_percent,
|
||||
"ai_percent": scores.ai_percent,
|
||||
"confidence": scores.confidence,
|
||||
"strong_signal_count": scores.strong_signal_count,
|
||||
"medium_signal_count": scores.medium_signal_count,
|
||||
"files_scored": scores.files_scored,
|
||||
}
|
||||
77
devplacepy/services/jobs/isslop/analysis/signals/__init__.py
Normal file
77
devplacepy/services/jobs/isslop/analysis/signals/__init__.py
Normal file
@ -0,0 +1,77 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import FileMetrics
|
||||
|
||||
SEVERITY_STRONG: str = "strong"
|
||||
SEVERITY_MEDIUM: str = "medium"
|
||||
SEVERITY_WEAK: str = "weak"
|
||||
|
||||
AXIS_ORIGIN: str = "origin"
|
||||
AXIS_QUALITY: str = "quality"
|
||||
|
||||
SEVERITY_FACTORS: dict[str, float] = {
|
||||
SEVERITY_STRONG: 1.0,
|
||||
SEVERITY_MEDIUM: 0.6,
|
||||
SEVERITY_WEAK: 0.3,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Signal:
|
||||
code: str
|
||||
title: str
|
||||
severity: str
|
||||
axis: str
|
||||
weight: float
|
||||
line: int
|
||||
evidence: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"code": self.code,
|
||||
"title": self.title,
|
||||
"severity": self.severity,
|
||||
"axis": self.axis,
|
||||
"weight": self.weight,
|
||||
"line": self.line,
|
||||
"evidence": self.evidence[:200],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepoContext:
|
||||
root: Path
|
||||
python_dependencies: frozenset[str]
|
||||
javascript_dependencies: frozenset[str]
|
||||
local_python_modules: frozenset[str]
|
||||
has_python_manifest: bool
|
||||
has_javascript_manifest: bool
|
||||
javascript_alias_prefixes: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileContext:
|
||||
path: Path
|
||||
relative: str
|
||||
language: str
|
||||
text: str
|
||||
lines: list[str] = field(default_factory=list)
|
||||
comments: list[tuple[int, str]] = field(default_factory=list)
|
||||
metrics: FileMetrics | None = None
|
||||
repo: RepoContext | None = None
|
||||
fingerprint_only: bool = False
|
||||
|
||||
|
||||
Detector = Callable[[FileContext], list[Signal]]
|
||||
|
||||
|
||||
def run_detectors(context: FileContext, detectors: list[Detector]) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for detector in detectors:
|
||||
findings.extend(detector(context))
|
||||
return findings
|
||||
@ -0,0 +1,114 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
from devplacepy.services.jobs.isslop.analysis.signals.textual import EM_DASH_THRESHOLD, LLM_LEXICON_PATTERN, LLM_LEXICON_THRESHOLD
|
||||
|
||||
EMOJI_HEADER_PATTERN = re.compile(r"^#{1,4}\s*[\U0001F300-\U0001FAFF✨🚀🤝⭐✅❗]")
|
||||
BADGE_PATTERN = re.compile(r"!\[[^\]]*\]\([^)]*(?:badge|shields\.io)[^)]*\)")
|
||||
BOILERPLATE_PHRASES: tuple[str, ...] = (
|
||||
"getting started",
|
||||
"features",
|
||||
"installation",
|
||||
"prerequisites",
|
||||
"contributing",
|
||||
"built with",
|
||||
"tech stack",
|
||||
"acknowledgments",
|
||||
"roadmap",
|
||||
)
|
||||
MARKETING_PATTERN = re.compile(
|
||||
r"blazingly fast|seamlessly|robust and scalable|cutting[- ]edge|revolutioniz|effortlessly|supercharge|powerful and flexible",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
NOT_JUST_PATTERN = re.compile(r"it'?s not just \w+[,;.]? it'?s", re.IGNORECASE)
|
||||
|
||||
EMOJI_HEADER_THRESHOLD: int = 3
|
||||
BADGE_THRESHOLD: int = 6
|
||||
BOILERPLATE_THRESHOLD: int = 5
|
||||
MARKETING_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def detect_documentation(context: FileContext) -> list[Signal]:
|
||||
if context.language != "markdown":
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
emoji_headers = [number for number, line in enumerate(context.lines, start=1) if EMOJI_HEADER_PATTERN.match(line)]
|
||||
if len(emoji_headers) >= EMOJI_HEADER_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"README_EMOJI_STRUCTURE",
|
||||
f"{len(emoji_headers)} emoji-prefixed headers in documentation",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
emoji_headers[0],
|
||||
context.lines[emoji_headers[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
lowered = context.text.lower()
|
||||
phrase_hits = sum(1 for phrase in BOILERPLATE_PHRASES if phrase in lowered)
|
||||
if phrase_hits >= BOILERPLATE_THRESHOLD and emoji_headers:
|
||||
findings.append(
|
||||
Signal(
|
||||
"README_BOILERPLATE",
|
||||
f"Generic AI README structure ({phrase_hits} boilerplate sections with emoji headers)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
", ".join(phrase for phrase in BOILERPLATE_PHRASES if phrase in lowered)[:150],
|
||||
)
|
||||
)
|
||||
badges = BADGE_PATTERN.findall(context.text)
|
||||
if len(badges) >= BADGE_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("README_BADGE_STUFFING", f"{len(badges)} badges stacked in documentation", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, badges[0][:100])
|
||||
)
|
||||
marketing_hits = MARKETING_PATTERN.findall(context.text)
|
||||
if len(marketing_hits) >= MARKETING_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("MARKETING_TONE", f"Marketing tone in documentation ({len(marketing_hits)} phrases)", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, ", ".join(marketing_hits[:4]))
|
||||
)
|
||||
lexicon_hits = {str(hit).lower() for hit in LLM_LEXICON_PATTERN.findall(context.text)}
|
||||
if len(lexicon_hits) >= LLM_LEXICON_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_LEXICON",
|
||||
f"{len(lexicon_hits)} distinct LLM signature words in documentation",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(sorted(lexicon_hits)[:6]),
|
||||
)
|
||||
)
|
||||
em_dashes = context.text.count("\u2014")
|
||||
if em_dashes >= EM_DASH_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"EM_DASH_OVERUSE",
|
||||
f"{em_dashes} em dashes in documentation, a strong LLM prose habit",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
1,
|
||||
f"{em_dashes} em dashes",
|
||||
)
|
||||
)
|
||||
not_just = NOT_JUST_PATTERN.search(context.text)
|
||||
if not_just:
|
||||
line = context.text.count("\n", 0, not_just.start()) + 1
|
||||
findings.append(
|
||||
Signal("AI_PROSE_PATTERN", "Characteristic AI prose construction", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, line, not_just.group(0))
|
||||
)
|
||||
return findings
|
||||
138
devplacepy/services/jobs/isslop/analysis/signals/errors.py
Normal file
138
devplacepy/services/jobs/isslop/analysis/signals/errors.py
Normal file
@ -0,0 +1,138 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
BARE_EXCEPT_PATTERN = re.compile(r"^\s*except\s*:\s*(#.*)?$")
|
||||
BROAD_EXCEPT_PATTERN = re.compile(r"^\s*except\s+(?:Exception|BaseException)\s*(?:as\s+\w+)?\s*:\s*(#.*)?$")
|
||||
PASS_PATTERN = re.compile(r"^\s*pass\s*(#.*)?$")
|
||||
EMPTY_CATCH_PATTERN = re.compile(r"catch\s*(\([^)]*\))?\s*\{\s*\}")
|
||||
LOG_ONLY_CATCH_PATTERN = re.compile(r"catch\s*\(\s*(\w+)\s*\)\s*\{\s*console\.(log|error)\([^)]*\);?\s*\}")
|
||||
DEBUGGER_PATTERN = re.compile(r"^\s*debugger\s*;?\s*$")
|
||||
TODO_REMOVE_PATTERN = re.compile(r"TODO:? remove", re.IGNORECASE)
|
||||
CONSOLE_LOG_PATTERN = re.compile(r"\bconsole\.log\s*\(")
|
||||
PRINT_PATTERN = re.compile(r"^\s*print\s*\(")
|
||||
|
||||
CONSOLE_LOG_THRESHOLD: int = 6
|
||||
PRINT_DENSITY_THRESHOLD: float = 0.06
|
||||
PRINT_MIN_SLOC: int = 60
|
||||
|
||||
|
||||
def _python_swallowed(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for index, line in enumerate(context.lines):
|
||||
bare = BARE_EXCEPT_PATTERN.match(line)
|
||||
broad = BROAD_EXCEPT_PATTERN.match(line)
|
||||
if not bare and not broad:
|
||||
continue
|
||||
follow = ""
|
||||
for candidate in context.lines[index + 1: index + 3]:
|
||||
if candidate.strip():
|
||||
follow = candidate
|
||||
break
|
||||
swallows = bool(PASS_PATTERN.match(follow))
|
||||
if bare:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR" if swallows else "BARE_EXCEPT",
|
||||
"Bare except swallowing all errors" if swallows else "Bare except clause",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
index + 1,
|
||||
line.strip(),
|
||||
)
|
||||
)
|
||||
elif swallows:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR",
|
||||
"except Exception: pass swallows all errors",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
index + 1,
|
||||
f"{line.strip()} / pass",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_error_handling(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
if context.language == "python":
|
||||
findings.extend(_python_swallowed(context))
|
||||
metrics = context.metrics
|
||||
if metrics is not None and metrics.sloc >= PRINT_MIN_SLOC:
|
||||
prints = sum(1 for line in context.lines if PRINT_PATTERN.match(line))
|
||||
if prints / max(metrics.sloc, 1) > PRINT_DENSITY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_LEFTOVERS",
|
||||
f"{prints} print() calls scattered through module",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"{prints} print calls in {metrics.sloc} SLOC",
|
||||
)
|
||||
)
|
||||
if context.language in ("javascript", "typescript", "java", "php", "csharp"):
|
||||
for match in EMPTY_CATCH_PATTERN.finditer(context.text):
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"SWALLOWED_ERROR",
|
||||
"Empty catch block swallows all errors",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
line,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
for match in LOG_ONLY_CATCH_PATTERN.finditer(context.text):
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"LOG_ONLY_CATCH",
|
||||
"Catch block only logs to console",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
line,
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
console_hits = len(CONSOLE_LOG_PATTERN.findall(context.text))
|
||||
if console_hits >= CONSOLE_LOG_THRESHOLD and "test" not in context.relative.lower():
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_LEFTOVERS",
|
||||
f"{console_hits} console.log calls left in source",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"{console_hits} console.log calls",
|
||||
)
|
||||
)
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
if DEBUGGER_PATTERN.match(line):
|
||||
findings.append(
|
||||
Signal("DEBUG_LEFTOVERS", "debugger statement left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, line.strip())
|
||||
)
|
||||
for number, comment in context.comments:
|
||||
if TODO_REMOVE_PATTERN.search(comment):
|
||||
findings.append(
|
||||
Signal("DEBUG_LEFTOVERS", "TODO remove marker left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, comment)
|
||||
)
|
||||
return findings
|
||||
@ -0,0 +1,139 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PYTHON_IMPORT_PATTERN = re.compile(r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))", re.MULTILINE)
|
||||
JS_IMPORT_PATTERN = re.compile(r"""(?:import[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]([^'"]+)['"]""")
|
||||
|
||||
PYTHON_IMPORT_ALIASES: dict[str, str] = {
|
||||
"pil": "pillow",
|
||||
"cv2": "opencv-python",
|
||||
"yaml": "pyyaml",
|
||||
"bs4": "beautifulsoup4",
|
||||
"dotenv": "python-dotenv",
|
||||
"sklearn": "scikit-learn",
|
||||
"dateutil": "python-dateutil",
|
||||
"jose": "python-jose",
|
||||
"jwt": "pyjwt",
|
||||
"magic": "python-magic",
|
||||
"serial": "pyserial",
|
||||
"usb": "pyusb",
|
||||
"redis": "redis",
|
||||
"attr": "attrs",
|
||||
"google": "google",
|
||||
"pkg_resources": "setuptools",
|
||||
"setuptools": "setuptools",
|
||||
}
|
||||
|
||||
NODE_BUILTINS: frozenset[str] = frozenset(
|
||||
{
|
||||
"assert", "async_hooks", "buffer", "child_process", "cluster", "console", "constants",
|
||||
"crypto", "dgram", "diagnostics_channel", "dns", "domain", "events", "fs", "http", "http2",
|
||||
"https", "inspector", "module", "net", "os", "path", "perf_hooks", "process", "punycode",
|
||||
"querystring", "readline", "repl", "sea", "sqlite", "stream", "string_decoder", "test",
|
||||
"timers", "tls", "trace_events", "tty", "url", "util", "v8", "vm", "wasi",
|
||||
"worker_threads", "zlib",
|
||||
}
|
||||
)
|
||||
|
||||
NON_PACKAGE_PREFIXES: tuple[str, ...] = ("@/", "~", "#", "$")
|
||||
|
||||
MAX_FLAGS_PER_FILE: int = 5
|
||||
|
||||
|
||||
def _normalize_package(name: str) -> str:
|
||||
return name.lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
|
||||
def _python_findings(context: FileContext) -> list[Signal]:
|
||||
repo = context.repo
|
||||
if repo is None or not repo.has_python_manifest:
|
||||
return []
|
||||
known = {_normalize_package(dep) for dep in repo.python_dependencies}
|
||||
local = {_normalize_package(module) for module in repo.local_python_modules}
|
||||
findings: list[Signal] = []
|
||||
seen: set[str] = set()
|
||||
for match in PYTHON_IMPORT_PATTERN.finditer(context.text):
|
||||
module = (match.group(1) or match.group(2) or "").split(".")[0]
|
||||
if not module or module in seen:
|
||||
continue
|
||||
seen.add(module)
|
||||
normalized = _normalize_package(module)
|
||||
aliased = _normalize_package(PYTHON_IMPORT_ALIASES.get(normalized, normalized))
|
||||
if normalized in sys.stdlib_module_names or module in sys.stdlib_module_names:
|
||||
continue
|
||||
if normalized in known or aliased in known or normalized in local:
|
||||
continue
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEP_UNRESOLVED",
|
||||
f"Import '{module}' resolves to no manifest dependency, stdlib or local module",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
line,
|
||||
match.group(0).strip(),
|
||||
)
|
||||
)
|
||||
return findings[:MAX_FLAGS_PER_FILE]
|
||||
|
||||
|
||||
def _javascript_findings(context: FileContext) -> list[Signal]:
|
||||
repo = context.repo
|
||||
if repo is None or not repo.has_javascript_manifest:
|
||||
return []
|
||||
known = {dep.lower() for dep in repo.javascript_dependencies}
|
||||
findings: list[Signal] = []
|
||||
seen: set[str] = set()
|
||||
aliases = tuple(repo.javascript_alias_prefixes)
|
||||
for match in JS_IMPORT_PATTERN.finditer(context.text):
|
||||
target = match.group(1)
|
||||
if target.startswith((".", "/", "http:", "https:")):
|
||||
continue
|
||||
if target.startswith(NON_PACKAGE_PREFIXES):
|
||||
continue
|
||||
if aliases and target.startswith(aliases):
|
||||
continue
|
||||
stripped = target.removeprefix("node:")
|
||||
parts = stripped.split("/")
|
||||
package = "/".join(parts[:2]) if stripped.startswith("@") and len(parts) >= 2 else parts[0]
|
||||
if package in seen:
|
||||
continue
|
||||
seen.add(package)
|
||||
if ":" in package:
|
||||
continue
|
||||
if package.lower() in NODE_BUILTINS or target.startswith("node:"):
|
||||
continue
|
||||
if package.lower() in known:
|
||||
continue
|
||||
line = context.text.count("\n", 0, match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEP_UNRESOLVED",
|
||||
f"Import '{package}' is not declared in package.json",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
line,
|
||||
match.group(0).strip()[:120],
|
||||
)
|
||||
)
|
||||
return findings[:MAX_FLAGS_PER_FILE]
|
||||
|
||||
|
||||
def detect_hallucination(context: FileContext) -> list[Signal]:
|
||||
if context.language == "python":
|
||||
return _python_findings(context)
|
||||
if context.language in ("javascript", "typescript"):
|
||||
return _javascript_findings(context)
|
||||
return []
|
||||
@ -0,0 +1,96 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PATH_FRAMEWORK_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(re.compile(r"(^|/)_next/", re.IGNORECASE), "Next.js build output (_next/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)_nuxt/", re.IGNORECASE), "Nuxt build output (_nuxt/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)_astro/", re.IGNORECASE), "Astro build output (_astro/)", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)\.svelte-kit/", re.IGNORECASE), "SvelteKit build output", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)webpack-[0-9a-f]{8,}\.js$", re.IGNORECASE), "Webpack runtime chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)polyfills-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Bundler polyfills chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)main-app-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Next.js app-router bundle", SEVERITY_MEDIUM),
|
||||
(re.compile(r"(^|/)chunks/app/(page|layout|loading|error|not-found)-[0-9a-f]{6,}\.js$", re.IGNORECASE), "Next.js app-router page bundle", SEVERITY_MEDIUM),
|
||||
(re.compile(r"(^|/)framework-[0-9a-f]{8,}\.js$", re.IGNORECASE), "Next.js framework chunk", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)assets/index-[0-9A-Za-z_-]{8,}\.(js|css)$"), "Vite hashed asset bundle", SEVERITY_WEAK),
|
||||
(re.compile(r"(^|/)gptengineer\.js$|(^|/)cdn\.gpteng\.co", re.IGNORECASE), "GPT Engineer / Lovable runtime", SEVERITY_STRONG),
|
||||
(re.compile(r"(^|/)lovable-uploads/", re.IGNORECASE), "Lovable uploads directory", SEVERITY_STRONG),
|
||||
(re.compile(r"(^|/)gatsby-", re.IGNORECASE), "Gatsby build artifact", SEVERITY_WEAK),
|
||||
)
|
||||
|
||||
VITE_HASH_ASSET = re.compile(r"index-[0-9A-Za-z_-]{8,}\.(js|css)$")
|
||||
|
||||
CONTENT_FRAMEWORK_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"self\.__next_f|__NEXT_DATA__|next/dist/", re.IGNORECASE), "next.js runtime"),
|
||||
(re.compile(r"__nuxt|nuxt\.config", re.IGNORECASE), "nuxt runtime"),
|
||||
(re.compile(r"astro-island|astro:", re.IGNORECASE), "astro runtime"),
|
||||
(re.compile(r"data-radix|@radix-ui|radix-ui", re.IGNORECASE), "radix primitives"),
|
||||
(re.compile(r"lucide", re.IGNORECASE), "lucide icons"),
|
||||
(re.compile(r"shadcn|class-variance-authority|\bcn\(", re.IGNORECASE), "shadcn/ui"),
|
||||
(re.compile(r"tailwind|--tw-", re.IGNORECASE), "tailwind"),
|
||||
(re.compile(r"framer-motion", re.IGNORECASE), "framer motion"),
|
||||
(re.compile(r"createClient\([^)]*supabase|supabase\.co", re.IGNORECASE), "supabase backend"),
|
||||
(re.compile(r"clerk\.|@clerk/", re.IGNORECASE), "clerk auth"),
|
||||
)
|
||||
|
||||
CONTENT_STRONG_MARKERS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"gpteng|lovable|data-lov-id", re.IGNORECASE), "Lovable/GPT-Engineer marker"),
|
||||
(re.compile(r"generated by (v0|bolt|base44|lovable)", re.IGNORECASE), "AI builder attribution"),
|
||||
)
|
||||
|
||||
VIBE_COMBO_THRESHOLD: int = 3
|
||||
|
||||
|
||||
def detect_infrastructure(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
relative = context.relative
|
||||
|
||||
for pattern, title, severity in PATH_FRAMEWORK_PATTERNS:
|
||||
if pattern.search(relative):
|
||||
weight = 3.0 if severity == SEVERITY_STRONG else (1.0 if severity == SEVERITY_MEDIUM else 0.5)
|
||||
findings.append(
|
||||
Signal("BUILD_ARTIFACT_PATH", f"Framework build artifact path: {title}", severity, AXIS_ORIGIN, weight, 1, relative)
|
||||
)
|
||||
break
|
||||
|
||||
text = context.text
|
||||
for pattern, marker in CONTENT_STRONG_MARKERS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal(
|
||||
"AI_BUILDER_FINGERPRINT",
|
||||
f"AI builder marker in bundle: {marker}",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
4.0,
|
||||
text.count("\n", 0, match.start()) + 1,
|
||||
match.group(0)[:100],
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
stack_hits = [label for pattern, label in CONTENT_FRAMEWORK_PATTERNS if pattern.search(text)]
|
||||
if len(stack_hits) >= VIBE_COMBO_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"VIBE_STACK",
|
||||
f"Default AI frontend stack in bundle ({len(stack_hits)} libraries)",
|
||||
SEVERITY_STRONG if len(stack_hits) >= 5 else SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(stack_hits[:8]),
|
||||
)
|
||||
)
|
||||
return findings
|
||||
98
devplacepy/services/jobs/isslop/analysis/signals/language.py
Normal file
98
devplacepy/services/jobs/isslop/analysis/signals/language.py
Normal file
@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
VAR_PATTERN = re.compile(r"^\s*var\s+\w+")
|
||||
MODERN_JS_PATTERN = re.compile(r"\b(?:const|let|=>|class\s+\w+|import\s)")
|
||||
USE_EFFECT_FETCH_PATTERN = re.compile(r"useEffect\s*\(\s*(?:async\s*)?\(\s*\)\s*=>\s*\{[^}]{0,400}\bfetch\s*\(", re.DOTALL)
|
||||
ANY_TYPE_PATTERN = re.compile(r":\s*any\b")
|
||||
OPEN_WITHOUT_WITH_PATTERN = re.compile(r"^\s*\w+\s*=\s*open\s*\(")
|
||||
BROAD_EXCEPT_COUNT_PATTERN = re.compile(r"^\s*except\s+Exception\b", re.MULTILINE)
|
||||
INLINE_STYLE_PATTERN = re.compile(r"\sstyle\s*=\s*\"")
|
||||
|
||||
VAR_THRESHOLD: int = 3
|
||||
ANY_THRESHOLD: int = 5
|
||||
OPEN_THRESHOLD: int = 2
|
||||
BROAD_EXCEPT_THRESHOLD: int = 3
|
||||
INLINE_STYLE_THRESHOLD: int = 10
|
||||
|
||||
|
||||
def detect_language_tells(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
if context.language in ("javascript", "typescript"):
|
||||
var_lines = [number for number, line in enumerate(context.lines, start=1) if VAR_PATTERN.match(line)]
|
||||
if len(var_lines) >= VAR_THRESHOLD and MODERN_JS_PATTERN.search(context.text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"LEGACY_VAR",
|
||||
f"{len(var_lines)} var declarations mixed into modern code",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
var_lines[0],
|
||||
context.lines[var_lines[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
effect_match = USE_EFFECT_FETCH_PATTERN.search(context.text)
|
||||
if effect_match:
|
||||
line = context.text.count("\n", 0, effect_match.start()) + 1
|
||||
findings.append(
|
||||
Signal(
|
||||
"USEEFFECT_FETCH",
|
||||
"Data fetching inside useEffect without caching or retry",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
line,
|
||||
effect_match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
if context.language == "typescript":
|
||||
any_hits = ANY_TYPE_PATTERN.findall(context.text)
|
||||
if len(any_hits) >= ANY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("ANY_TYPES", f"{len(any_hits)} explicit any types", SEVERITY_MEDIUM, AXIS_QUALITY, 1.0, 1, f"{len(any_hits)} occurrences of : any")
|
||||
)
|
||||
if context.language == "python":
|
||||
open_lines = [number for number, line in enumerate(context.lines, start=1) if OPEN_WITHOUT_WITH_PATTERN.match(line)]
|
||||
if len(open_lines) >= OPEN_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"NO_CONTEXT_MANAGER",
|
||||
f"{len(open_lines)} open() calls without context manager",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
open_lines[0],
|
||||
context.lines[open_lines[0] - 1].strip(),
|
||||
)
|
||||
)
|
||||
broad = BROAD_EXCEPT_COUNT_PATTERN.findall(context.text)
|
||||
if len(broad) >= BROAD_EXCEPT_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_EXCEPTIONS",
|
||||
f"{len(broad)} undifferentiated except Exception handlers",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(broad)} broad handlers",
|
||||
)
|
||||
)
|
||||
if context.language == "html":
|
||||
inline_styles = INLINE_STYLE_PATTERN.findall(context.text)
|
||||
if len(inline_styles) >= INLINE_STYLE_THRESHOLD:
|
||||
findings.append(
|
||||
Signal("INLINE_STYLES", f"{len(inline_styles)} inline style attributes", SEVERITY_WEAK, AXIS_QUALITY, 1.0, 1, f"{len(inline_styles)} inline styles")
|
||||
)
|
||||
return findings
|
||||
191
devplacepy/services/jobs/isslop/analysis/signals/llmdefaults.py
Normal file
191
devplacepy/services/jobs/isslop/analysis/signals/llmdefaults.py
Normal file
@ -0,0 +1,191 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
CODE_LANGUAGES_FOR_DEFAULTS: frozenset[str] = frozenset(
|
||||
{"python", "javascript", "typescript", "php", "go", "java", "ruby", "csharp", "yaml", "json", "html"}
|
||||
)
|
||||
|
||||
DEFAULT_SECRET_PATTERN = re.compile(
|
||||
r"(secret|jwt[_-]?secret|secret[_-]?key|signing[_-]?key|session[_-]?secret)\w*\s*[:=]\s*"
|
||||
r"[\"'](your[_-]secret[_-]key(?:[_-]here)?|your[_-]super[_-]secret|supersecret\w*|super[_-]secret"
|
||||
r"|my[_-]?secret(?:[_-]?key)?|secret(?:123)?|secret[_-]key|change[_-]?(this|me)\w*|keyboard cat)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
WILDCARD_CORS_CREDENTIALS_PATTERN = re.compile(
|
||||
r"allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\][\s\S]{0,200}allow_credentials\s*=\s*True"
|
||||
r"|allow_credentials\s*=\s*True[\s\S]{0,200}allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\]"
|
||||
r"|credentials\s*:\s*true[\s\S]{0,120}origin\s*:\s*[\"']\*[\"']"
|
||||
r"|origin\s*:\s*[\"']\*[\"'][\s\S]{0,120}credentials\s*:\s*true",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
WILDCARD_CORS_PATTERN = re.compile(
|
||||
r"allow_origins\s*=\s*\[\s*[\"']\*[\"']\s*\]"
|
||||
r"|Access-Control-Allow-Origin[\"']?\s*[,:]\s*[\"']\*[\"']"
|
||||
r"|app\.use\(\s*cors\(\s*\)\s*\)"
|
||||
r"|origin\s*:\s*[\"']\*[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DEBUG_ENABLED_PATTERN = re.compile(
|
||||
r"\.run\([^)]*debug\s*=\s*True|^\s*DEBUG\s*=\s*True\s*$|debug\s*:\s*true\s*,?\s*//|APP_DEBUG=true",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
SEED_CREDENTIAL_PATTERN = re.compile(
|
||||
r"[\"'](admin123|password123|test123|admin@example\.com|test@test\.com|admin@admin\.com|letmein|qwerty123)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
DEFAULT_DATABASE_PATTERN = re.compile(
|
||||
r"mongodb(\+srv)?://localhost(:27017)?/(mydb|myapp|test|mydatabase|app)"
|
||||
r"|(postgres(ql)?|mysql)://(user|root|admin|postgres):(password|pass|root|admin|secret)@localhost",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
TUTORIAL_SERVER_PATTERN = re.compile(
|
||||
r"console\.log\(\s*[\"'`]Server (is )?(running|listening|started) (on|at) (port |http)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
APP_NAME_DEFAULT_PATTERN = re.compile(
|
||||
r"[\"'](my-?app|my-?project|test-?app|awesome-?project|my-?website)[\"']\s*[,:]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
MAX_FINDINGS: int = 10
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def detect_llm_defaults(context: FileContext) -> list[Signal]:
|
||||
if context.language not in CODE_LANGUAGES_FOR_DEFAULTS:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
|
||||
for match in DEFAULT_SECRET_PATTERN.finditer(text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_DEFAULT_SECRET",
|
||||
f"Stock LLM placeholder secret '{match.group(2)}' left in configuration",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
credentials_combo = WILDCARD_CORS_CREDENTIALS_PATTERN.search(text)
|
||||
if credentials_combo:
|
||||
findings.append(
|
||||
Signal(
|
||||
"WILDCARD_CORS_CREDENTIALS",
|
||||
"Wildcard CORS origin combined with credentials, the canonical LLM scaffold vulnerability",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
_line_of(text, credentials_combo.start()),
|
||||
credentials_combo.group(0)[:120],
|
||||
)
|
||||
)
|
||||
else:
|
||||
wildcard = WILDCARD_CORS_PATTERN.search(text)
|
||||
if wildcard:
|
||||
findings.append(
|
||||
Signal(
|
||||
"WILDCARD_CORS",
|
||||
"Wildcard CORS configuration, the default LLM scaffold",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, wildcard.start()),
|
||||
wildcard.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
debug = DEBUG_ENABLED_PATTERN.search(text)
|
||||
if debug:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEBUG_ENABLED",
|
||||
"Debug mode committed enabled, the tutorial default",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, debug.start()),
|
||||
debug.group(0).strip()[:120],
|
||||
)
|
||||
)
|
||||
|
||||
for match in SEED_CREDENTIAL_PATTERN.finditer(text):
|
||||
findings.append(
|
||||
Signal(
|
||||
"SEED_CREDENTIALS",
|
||||
f"Stock demo credential {match.group(1)} in source",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:120],
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
database = DEFAULT_DATABASE_PATTERN.search(text)
|
||||
if database:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEFAULT_DATABASE_URL",
|
||||
"Tutorial-default database connection string",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, database.start()),
|
||||
database.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
tutorial = TUTORIAL_SERVER_PATTERN.search(text)
|
||||
if tutorial:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TUTORIAL_SERVER_LOG",
|
||||
"Tutorial-style server startup console.log",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, tutorial.start()),
|
||||
tutorial.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
app_name = APP_NAME_DEFAULT_PATTERN.search(text)
|
||||
if app_name:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEFAULT_PROJECT_NAME",
|
||||
f"Default scaffold project name {app_name.group(1)}",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, app_name.start()),
|
||||
app_name.group(0)[:80],
|
||||
)
|
||||
)
|
||||
|
||||
return findings[:MAX_FINDINGS]
|
||||
115
devplacepy/services/jobs/isslop/analysis/signals/naming.py
Normal file
115
devplacepy/services/jobs/isslop/analysis/signals/naming.py
Normal file
@ -0,0 +1,115 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DECLARATION_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||
"python": (
|
||||
re.compile(r"^\s*(?:async\s+)?def\s+(\w+)"),
|
||||
re.compile(r"^\s*class\s+(\w+)"),
|
||||
re.compile(r"^\s*([a-zA-Z_]\w*)\s*(?::[^=]+)?=\s*[^=]"),
|
||||
),
|
||||
"javascript": (
|
||||
re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)"),
|
||||
re.compile(r"^\s*(?:export\s+)?class\s+(\w+)"),
|
||||
re.compile(r"^\s*(?:const|let|var)\s+(\w+)"),
|
||||
),
|
||||
}
|
||||
DECLARATION_PATTERNS["typescript"] = DECLARATION_PATTERNS["javascript"]
|
||||
|
||||
GENERIC_IDENTIFIERS: frozenset[str] = frozenset(
|
||||
{
|
||||
"data", "result", "res", "temp", "tmp", "item", "items", "value", "values", "val",
|
||||
"obj", "object", "thing", "stuff", "info", "foo", "bar", "baz", "output", "input",
|
||||
"myfunction", "myvar", "mydata", "example", "processdata", "dosomething", "handledata",
|
||||
"handleclick", "handlesubmit", "handlechange", "mycomponent", "untitled", "test1", "func",
|
||||
"arr", "lst", "dct", "retval", "resp", "ret",
|
||||
}
|
||||
)
|
||||
|
||||
VERBOSE_MIN_LENGTH: int = 30
|
||||
VERBOSE_MIN_WORDS: int = 5
|
||||
GENERIC_DENSITY_THRESHOLD: float = 0.30
|
||||
MIN_DECLARATIONS: int = 5
|
||||
|
||||
|
||||
def _split_words(identifier: str) -> list[str]:
|
||||
parts = re.split(r"_+", identifier)
|
||||
words: list[str] = []
|
||||
for part in parts:
|
||||
words.extend(re.findall(r"[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])", part))
|
||||
return [word for word in words if word]
|
||||
|
||||
|
||||
def detect_naming(context: FileContext) -> list[Signal]:
|
||||
patterns = DECLARATION_PATTERNS.get(context.language)
|
||||
if not patterns:
|
||||
return []
|
||||
declarations: list[tuple[int, str]] = []
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern in patterns:
|
||||
match = pattern.match(line)
|
||||
if match:
|
||||
name = next(group for group in match.groups() if group)
|
||||
declarations.append((number, name))
|
||||
break
|
||||
if len(declarations) < MIN_DECLARATIONS:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
generic = [(number, name) for number, name in declarations if name.lower().replace("_", "") in GENERIC_IDENTIFIERS]
|
||||
density = len(generic) / len(declarations)
|
||||
if density >= GENERIC_DENSITY_THRESHOLD:
|
||||
number, name = generic[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_NAMING",
|
||||
f"Generic identifiers in {density:.0%} of {len(declarations)} declarations",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
number,
|
||||
name,
|
||||
)
|
||||
)
|
||||
verbose = [
|
||||
(number, name)
|
||||
for number, name in declarations
|
||||
if len(name) >= VERBOSE_MIN_LENGTH and len(_split_words(name)) >= VERBOSE_MIN_WORDS
|
||||
]
|
||||
if verbose:
|
||||
number, name = verbose[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"TEXTBOOK_NAMING",
|
||||
"Over-verbose textbook identifier",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
number,
|
||||
name,
|
||||
)
|
||||
)
|
||||
snake = sum(1 for _, name in declarations if "_" in name and name.lower() == name)
|
||||
camel = sum(1 for _, name in declarations if "_" not in name and name.lower() != name and name[0].islower())
|
||||
total = len(declarations)
|
||||
if snake >= 3 and camel >= 3 and snake / total >= 0.25 and camel / total >= 0.25:
|
||||
findings.append(
|
||||
Signal(
|
||||
"NAMING_MIXED",
|
||||
f"Mixed snake_case ({snake}) and camelCase ({camel}) declarations",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
declarations[0][0],
|
||||
f"{snake} snake_case vs {camel} camelCase",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
149
devplacepy/services/jobs/isslop/analysis/signals/scaffolds.py
Normal file
149
devplacepy/services/jobs/isslop/analysis/signals/scaffolds.py
Normal file
@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
HTML_SCAFFOLD_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(re.compile(r"Generated by create next app", re.IGNORECASE), "Untouched create-next-app metadata", SEVERITY_STRONG),
|
||||
(re.compile(r"<title>\s*Create Next App\s*</title>", re.IGNORECASE), "Default Create Next App title", SEVERITY_STRONG),
|
||||
(re.compile(r"<title>\s*(Vite \+ React|Vite App|React App)\s*</title>", re.IGNORECASE), "Default Vite/CRA title", SEVERITY_STRONG),
|
||||
(re.compile(r"You need to enable JavaScript to run this app", re.IGNORECASE), "Create React App noscript boilerplate", SEVERITY_STRONG),
|
||||
(re.compile(r"/vite\.svg|src=[\"']/?vite\.svg", re.IGNORECASE), "Default Vite logo asset", SEVERITY_MEDIUM),
|
||||
(re.compile(r"<div\s+id=[\"']root[\"']\s*>\s*</div>", re.IGNORECASE), "Empty CRA/Vite root mount div", SEVERITY_MEDIUM),
|
||||
(re.compile(r"<div\s+id=[\"']__next[\"']", re.IGNORECASE), "Next.js hydration root", SEVERITY_WEAK),
|
||||
(re.compile(r"src=[\"']/src/main\.(jsx|tsx)[\"']", re.IGNORECASE), "Default Vite entry module", SEVERITY_MEDIUM),
|
||||
(re.compile(r"Edit <code>src/App", re.IGNORECASE), "Untouched CRA starter copy", SEVERITY_STRONG),
|
||||
(re.compile(r"Save to reload", re.IGNORECASE), "Untouched Vite/CRA starter copy", SEVERITY_STRONG),
|
||||
(re.compile(r"content=[\"']Web site created using create-react-app", re.IGNORECASE), "CRA description meta", SEVERITY_STRONG),
|
||||
(re.compile(r"%PUBLIC_URL%", re.IGNORECASE), "Unrendered CRA template token", SEVERITY_STRONG),
|
||||
(re.compile(r"__NEXT_DATA__|self\.__next_f", re.IGNORECASE), "Next.js runtime payload", SEVERITY_WEAK),
|
||||
(re.compile(r"<meta[^>]+name=[\"']generator[\"'][^>]+(gatsby|hugo|jekyll|astro|next\.js|nuxt|docusaurus)", re.IGNORECASE), "Static-site generator meta", SEVERITY_WEAK),
|
||||
)
|
||||
|
||||
PY_DOCSTRING_TRIVIAL_PATTERN = re.compile(
|
||||
r"def\s+\w+\([^)]*\)\s*(?:->[^\n:]+)?:\s*\n\s*(?:\"\"\"|''')[^\n]{5,80}(?:\"\"\"|''')\s*\n\s*return\b",
|
||||
re.MULTILINE,
|
||||
)
|
||||
PY_VERBOSE_NAME_PATTERN = re.compile(r"\b[a-z]+(?:_[a-z]+){4,}\b\s*=")
|
||||
PY_ERROR_PRINT_PATTERN = re.compile(
|
||||
r"except\s+\w*(?:Exception|Error)?\s*(?:as\s+(\w+))?\s*:\s*\n\s*print\(\s*f?[\"'](?:an?\s+)?error", re.IGNORECASE
|
||||
)
|
||||
PY_MAIN_GUARD_PATTERN = re.compile(r"^if\s+__name__\s*==\s*[\"']__main__[\"']\s*:", re.MULTILINE)
|
||||
PY_TYPE_HINT_PATTERN = re.compile(r"def\s+\w+\([^)]*:\s*\w+[^)]*\)\s*->\s*\w+")
|
||||
PY_FULL_DOCSTRING_PATTERN = re.compile(r"def\s+\w+[^:]*:\s*\n\s*(?:\"\"\"|''')")
|
||||
PY_LOGGING_FSTRING_PATTERN = re.compile(r"logging\.(info|debug|warning|error)\(\s*f[\"']")
|
||||
|
||||
VERBOSE_NAME_THRESHOLD: int = 2
|
||||
TRIVIAL_DOCSTRING_THRESHOLD: int = 2
|
||||
TEXTBOOK_MIN_FUNCTIONS: int = 5
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def _html_scaffold_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
for pattern, title, severity in HTML_SCAFFOLD_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
weight = 4.0 if severity == SEVERITY_STRONG else (2.0 if severity == SEVERITY_MEDIUM else 0.5)
|
||||
findings.append(
|
||||
Signal("FRAMEWORK_SCAFFOLD", f"Framework scaffold artifact: {title}", severity, AXIS_ORIGIN, weight, _line_of(text, match.start()), match.group(0)[:100])
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def _python_structure_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
metrics = context.metrics
|
||||
|
||||
trivial = PY_DOCSTRING_TRIVIAL_PATTERN.findall(text)
|
||||
if len(trivial) >= TRIVIAL_DOCSTRING_THRESHOLD:
|
||||
match = PY_DOCSTRING_TRIVIAL_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"TRIVIAL_DOCSTRINGS",
|
||||
f"{len(trivial)} trivial functions with textbook docstrings",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
"over-documented one-line functions",
|
||||
)
|
||||
)
|
||||
|
||||
verbose = PY_VERBOSE_NAME_PATTERN.findall(text)
|
||||
if len(verbose) >= VERBOSE_NAME_THRESHOLD:
|
||||
match = PY_VERBOSE_NAME_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"VERBOSE_TEXTBOOK_NAMES",
|
||||
f"{len(verbose)} extremely verbose textbook variable names",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
match.group(0).strip()[:80] if match else "",
|
||||
)
|
||||
)
|
||||
|
||||
error_print = PY_ERROR_PRINT_PATTERN.search(text)
|
||||
if error_print:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GENERIC_ERROR_PRINT",
|
||||
"Textbook 'except: print(f\"Error: {e}\")' handling",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
_line_of(text, error_print.start()),
|
||||
error_print.group(0).strip()[:80],
|
||||
)
|
||||
)
|
||||
|
||||
if metrics is not None and metrics.function_count >= TEXTBOOK_MIN_FUNCTIONS:
|
||||
typed = len(PY_TYPE_HINT_PATTERN.findall(text))
|
||||
documented = len(PY_FULL_DOCSTRING_PATTERN.findall(text))
|
||||
has_main_guard = bool(PY_MAIN_GUARD_PATTERN.search(text))
|
||||
logging_fstring = bool(PY_LOGGING_FSTRING_PATTERN.search(text))
|
||||
markers = sum(
|
||||
[
|
||||
typed >= max(3, metrics.function_count // 2),
|
||||
documented >= metrics.function_count,
|
||||
has_main_guard,
|
||||
logging_fstring,
|
||||
]
|
||||
)
|
||||
if markers >= 3:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TEXTBOOK_STRUCTURE",
|
||||
"Uniformly textbook structure: full type hints, docstring on every function, main guard",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
f"typed={typed}, docstrings={documented}, main_guard={has_main_guard}",
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_scaffolds(context: FileContext) -> list[Signal]:
|
||||
if context.language == "html":
|
||||
return _html_scaffold_signals(context)
|
||||
if context.language == "python":
|
||||
return _python_structure_signals(context)
|
||||
return []
|
||||
98
devplacepy/services/jobs/isslop/analysis/signals/security.py
Normal file
98
devplacepy/services/jobs/isslop/analysis/signals/security.py
Normal file
@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.metrics import shannon_entropy
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "AWS access key"),
|
||||
(re.compile(r"\bghp_[A-Za-z0-9]{36,}\b"), "GitHub personal token"),
|
||||
(re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}\b"), "GitHub fine-grained token"),
|
||||
(re.compile(r"\bsk_live_[A-Za-z0-9]{16,}\b"), "Stripe live key"),
|
||||
(re.compile(r"\bsk-[A-Za-z0-9]{40,}\b"), "API secret key"),
|
||||
(re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b"), "Google API key"),
|
||||
(re.compile(r"\bxox[baprs]-[A-Za-z0-9\-]{10,}\b"), "Slack token"),
|
||||
(re.compile(r"-----BEGIN[A-Z ]*PRIVATE KEY-----"), "Private key material"),
|
||||
(re.compile(r"\b(?:postgres|postgresql|mysql|mongodb(?:\+srv)?|redis|amqp)://\w+:[^@/\s]{4,}@"), "Credentials in connection string"),
|
||||
)
|
||||
|
||||
ENTROPY_ASSIGNMENT_PATTERN = re.compile(
|
||||
r"(secret|token|passwd|password|api[_-]?key|auth[_-]?key|private[_-]?key)\w*\s*[:=]\s*[\"']([^\"'\s]{16,})[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ENTROPY_THRESHOLD: float = 3.5
|
||||
|
||||
ALLOWLIST_MARKERS: tuple[str, ...] = (
|
||||
"example", "your_", "your-", "changeme", "change_me", "placeholder", "xxxx", "dummy",
|
||||
"sample", "<", "${", "{{", "todo", "insert", "test", "fake", "redacted", "akiaiosfodnn7example",
|
||||
)
|
||||
ALLOWLIST_PATH_MARKERS: tuple[str, ...] = (".example", "fixture", "mock", "sample", "/docs/", "/test", "_test", "spec.")
|
||||
|
||||
INJECTION_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
|
||||
(
|
||||
re.compile(r"[\"'](?:SELECT|INSERT|UPDATE|DELETE|DROP)\b[^\"']*[\"']\s*\+", re.IGNORECASE),
|
||||
"SQL built by string concatenation",
|
||||
SEVERITY_STRONG,
|
||||
),
|
||||
(
|
||||
re.compile(r"f[\"'](?:[^\"']*\b(?:SELECT|INSERT|UPDATE|DELETE)\b[^\"']*\{)", re.IGNORECASE),
|
||||
"SQL built with f-string interpolation",
|
||||
SEVERITY_STRONG,
|
||||
),
|
||||
(re.compile(r"\beval\s*\(\s*[a-zA-Z_$][\w$.]*\s*[,)]"), "eval() on a variable", SEVERITY_STRONG),
|
||||
(re.compile(r"dangerouslySetInnerHTML"), "dangerouslySetInnerHTML usage", SEVERITY_STRONG),
|
||||
(re.compile(r"\.innerHTML\s*=\s*(?![\"'`][^$])"), "innerHTML assigned from dynamic value", SEVERITY_MEDIUM),
|
||||
(re.compile(r"document\.write\s*\("), "document.write usage", SEVERITY_MEDIUM),
|
||||
(re.compile(r"child_process[.\w]*\.exec(?:Sync)?\s*\(\s*[`\"'][^`\"']*[$+]"), "Shell exec with interpolation", SEVERITY_STRONG),
|
||||
(re.compile(r"os\.system\s*\(\s*f?[\"'][^\"']*[{+]"), "os.system with interpolation", SEVERITY_STRONG),
|
||||
(re.compile(r"subprocess\.\w+\([^)]*shell\s*=\s*True[^)]*[+{]"), "subprocess shell=True with dynamic input", SEVERITY_STRONG),
|
||||
)
|
||||
|
||||
|
||||
def _is_allowlisted(line: str, relative_path: str) -> bool:
|
||||
lowered = line.lower()
|
||||
if any(marker in lowered for marker in ALLOWLIST_MARKERS):
|
||||
return True
|
||||
lowered_path = relative_path.lower()
|
||||
return any(marker in lowered_path for marker in ALLOWLIST_PATH_MARKERS)
|
||||
|
||||
|
||||
def detect_security(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern, title in SECRET_PATTERNS:
|
||||
if pattern.search(line) and not _is_allowlisted(line, context.relative):
|
||||
findings.append(
|
||||
Signal("HARDCODED_SECRET", f"Hardcoded secret: {title}", SEVERITY_STRONG, AXIS_QUALITY, 5.0, number, line.strip()[:120])
|
||||
)
|
||||
break
|
||||
entropy_match = ENTROPY_ASSIGNMENT_PATTERN.search(line)
|
||||
if entropy_match and not _is_allowlisted(line, context.relative):
|
||||
candidate = entropy_match.group(2)
|
||||
if shannon_entropy(candidate) >= ENTROPY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"HARDCODED_SECRET",
|
||||
f"High-entropy credential assigned to '{entropy_match.group(1)}'",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
number,
|
||||
line.strip()[:120],
|
||||
)
|
||||
)
|
||||
for pattern, title, severity in INJECTION_PATTERNS:
|
||||
if pattern.search(line):
|
||||
findings.append(
|
||||
Signal("INJECTION_RISK", title, severity, AXIS_QUALITY, 5.0, number, line.strip()[:120])
|
||||
)
|
||||
break
|
||||
return findings[:12]
|
||||
140
devplacepy/services/jobs/isslop/analysis/signals/structure.py
Normal file
140
devplacepy/services/jobs/isslop/analysis/signals/structure.py
Normal file
@ -0,0 +1,140 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
DEAD_CODE_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\bif\s*\(\s*false\s*\)"), "if (false) branch"),
|
||||
(re.compile(r"^\s*if\s+False\s*:"), "if False: branch"),
|
||||
(re.compile(r"^\s*if\s+0\s*:"), "if 0: branch"),
|
||||
(re.compile(r"\bwhile\s*\(\s*false\s*\)"), "while (false) loop"),
|
||||
)
|
||||
|
||||
ABSTRACTION_SUFFIX_PATTERN = re.compile(
|
||||
r"class\s+\w*(Factory|Manager|Provider|Strategy|Handler|Wrapper|Helper|Service|Orchestrator|Coordinator|Impl)\b"
|
||||
)
|
||||
CLASS_PATTERN = re.compile(r"^\s*(?:export\s+)?(?:abstract\s+)?class\s+(\w+)", re.MULTILINE)
|
||||
|
||||
UNIFORM_INDENT_DEVIATION: float = 0.45
|
||||
UNIFORM_MIN_SLOC: int = 50
|
||||
GOD_MODULE_SLOC: int = 600
|
||||
GOD_MODULE_IMPORTS: int = 20
|
||||
DUPLICATION_THRESHOLD: float = 0.08
|
||||
LONG_FUNCTION_LINES: int = 120
|
||||
COMPLEXITY_THRESHOLD: int = 15
|
||||
ABSTRACTION_DENSITY: float = 0.5
|
||||
ABSTRACTION_MIN_CLASSES: int = 3
|
||||
|
||||
|
||||
def detect_structure(context: FileContext) -> list[Signal]:
|
||||
metrics = context.metrics
|
||||
if metrics is None:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
if metrics.sloc >= UNIFORM_MIN_SLOC and metrics.indent_variance < UNIFORM_INDENT_DEVIATION:
|
||||
findings.append(
|
||||
Signal(
|
||||
"MACHINE_REGULARITY",
|
||||
f"Unnaturally uniform whitespace (indent deviation {metrics.indent_variance:.2f})",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"sloc={metrics.sloc}",
|
||||
)
|
||||
)
|
||||
if (
|
||||
context.language == "python"
|
||||
and metrics.function_count >= 4
|
||||
and metrics.docstring_count >= metrics.function_count
|
||||
):
|
||||
findings.append(
|
||||
Signal(
|
||||
"DOCSTRING_UNIFORMITY",
|
||||
f"Every one of {metrics.function_count} functions carries a docstring template",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"{metrics.docstring_count}/{metrics.function_count} docstrings",
|
||||
)
|
||||
)
|
||||
if metrics.sloc > GOD_MODULE_SLOC and metrics.import_count > GOD_MODULE_IMPORTS:
|
||||
findings.append(
|
||||
Signal(
|
||||
"GOD_MODULE",
|
||||
f"Module mixes many concerns ({metrics.sloc} SLOC, {metrics.import_count} imports)",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
f"sloc={metrics.sloc} imports={metrics.import_count}",
|
||||
)
|
||||
)
|
||||
if metrics.duplication_ratio > DUPLICATION_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DUPLICATED_LOGIC",
|
||||
f"Duplicated block ratio {metrics.duplication_ratio:.1%}",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
f"windows duplicated: {metrics.duplication_ratio:.1%}",
|
||||
)
|
||||
)
|
||||
if metrics.max_function_length > LONG_FUNCTION_LINES:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LONG_FUNCTION",
|
||||
f"Function of {metrics.max_function_length} lines",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"max function length {metrics.max_function_length}",
|
||||
)
|
||||
)
|
||||
if metrics.cyclomatic_max_estimate > COMPLEXITY_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"HIGH_COMPLEXITY",
|
||||
f"Estimated cyclomatic complexity {metrics.cyclomatic_max_estimate} per function",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
1,
|
||||
f"complexity estimate {metrics.cyclomatic_max_estimate}",
|
||||
)
|
||||
)
|
||||
for number, line in enumerate(context.lines, start=1):
|
||||
for pattern, title in DEAD_CODE_PATTERNS:
|
||||
if pattern.search(line):
|
||||
findings.append(
|
||||
Signal("DEAD_DEFENSIVE_CODE", f"Dead code: {title}", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, number, line.strip())
|
||||
)
|
||||
break
|
||||
classes = CLASS_PATTERN.findall(context.text)
|
||||
if len(classes) >= ABSTRACTION_MIN_CLASSES:
|
||||
pattern_classes = ABSTRACTION_SUFFIX_PATTERN.findall(context.text)
|
||||
if len(pattern_classes) / len(classes) >= ABSTRACTION_DENSITY:
|
||||
findings.append(
|
||||
Signal(
|
||||
"OVER_ABSTRACTION",
|
||||
f"{len(pattern_classes)} of {len(classes)} classes are pattern-suffixed abstractions",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(pattern_classes[:5]),
|
||||
)
|
||||
)
|
||||
return findings
|
||||
198
devplacepy/services/jobs/isslop/analysis/signals/textual.py
Normal file
198
devplacepy/services/jobs/isslop/analysis/signals/textual.py
Normal file
@ -0,0 +1,198 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
PLACEHOLDER_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"your (code|logic|implementation) here", re.IGNORECASE), "Placeholder body left in source"),
|
||||
(re.compile(r"\.\.\. ?rest of (the )?code", re.IGNORECASE), "Truncated generation marker"),
|
||||
(re.compile(r"\.\.\. ?(existing|other|previous) code", re.IGNORECASE), "Elided-code marker"),
|
||||
(re.compile(r"/\*\s*implementation\s*\*/", re.IGNORECASE), "Empty implementation stub"),
|
||||
(re.compile(r"TODO:? implement( this)?\b", re.IGNORECASE), "Unimplemented TODO stub"),
|
||||
(re.compile(r"TODO:? add (logic|implementation|code)", re.IGNORECASE), "Unimplemented TODO stub"),
|
||||
(re.compile(r"<!--\s*add content here\s*-->", re.IGNORECASE), "Placeholder HTML content"),
|
||||
(re.compile(r"rest omitted", re.IGNORECASE), "Truncated generation marker"),
|
||||
)
|
||||
|
||||
PROMPT_LEAK_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"as an ai (language )?model", re.IGNORECASE),
|
||||
re.compile(r"i (cannot|can't) (assist|help) with", re.IGNORECASE),
|
||||
re.compile(r"here('|’)?s the (updated|complete|revised|full|corrected) code", re.IGNORECASE),
|
||||
re.compile(r"\bcertainly!\s", re.IGNORECASE),
|
||||
re.compile(r"sure, here('s| is)", re.IGNORECASE),
|
||||
re.compile(r"i hope this helps", re.IGNORECASE),
|
||||
re.compile(r"let me know if (you|there)", re.IGNORECASE),
|
||||
re.compile(r"feel free to (adjust|modify|change|customize)", re.IGNORECASE),
|
||||
re.compile(r"note that this is a (simplified|basic) (example|implementation|version)", re.IGNORECASE),
|
||||
re.compile(r"in a (real|production)([- ]world)? (application|scenario|environment|setting)", re.IGNORECASE),
|
||||
re.compile(r"this is a placeholder", re.IGNORECASE),
|
||||
re.compile(r"replace .{1,40} with your (actual|own)", re.IGNORECASE),
|
||||
)
|
||||
|
||||
AI_SIGNATURE_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"generated (by|with) (chatgpt|copilot|claude|cursor|gemini|codeium|v0|bolt|ai)", re.IGNORECASE),
|
||||
re.compile(r"co-authored-by:\s*(claude|copilot)", re.IGNORECASE),
|
||||
re.compile(r"\U0001F916 generated with"),
|
||||
re.compile(r"created with the help of ai", re.IGNORECASE),
|
||||
re.compile(r"\bclaude code\b", re.IGNORECASE),
|
||||
re.compile(r"powered by (chatgpt|gpt-4|claude|gemini)", re.IGNORECASE),
|
||||
)
|
||||
|
||||
EMOJI_PATTERN = re.compile(
|
||||
"[\U0001F300-\U0001FAFF☀-➿⬀-⯿✅❌✨❗⭐]"
|
||||
)
|
||||
|
||||
ENTHUSIASM_PATTERN = re.compile(
|
||||
r"blazingly fast|seamlessly|robust and scalable|cutting[- ]edge|revolutioniz|effortlessly|supercharge|game[- ]chang",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
LLM_LEXICON_PATTERN = re.compile(
|
||||
r"\b(delve|delving|delves|showcas(?:e|es|ing)|pivotal|intricate|meticulous(?:ly)?|realm"
|
||||
r"|bolster(?:s|ing)?|garnered|underpins|seamless(?:ly)?|leverag(?:e|es|ing)|streamlin(?:e|es|ing)"
|
||||
r"|elevat(?:e|es|ing)|holistic|robust and|comprehensive suite|game[- ]chang(?:er|ing)"
|
||||
r"|it'?s (?:important|worth) (?:to note|noting)|keep in mind|as you can see|in this example)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
LLM_LEXICON_THRESHOLD: int = 3
|
||||
EM_DASH_THRESHOLD: int = 3
|
||||
|
||||
WORD_PATTERN = re.compile(r"[a-z][a-z0-9]+")
|
||||
NARRATION_MIN_OVERLAP: float = 0.55
|
||||
NARRATION_MIN_TOKENS: int = 2
|
||||
|
||||
|
||||
def _narration_signals(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
line_lookup = {number: text for number, text in context.comments}
|
||||
for number, comment in context.comments:
|
||||
if number + 1 in line_lookup or number >= len(context.lines):
|
||||
continue
|
||||
comment_tokens = set(WORD_PATTERN.findall(comment.lower()))
|
||||
if len(comment_tokens) < NARRATION_MIN_TOKENS:
|
||||
continue
|
||||
code_line = context.lines[number].lower()
|
||||
code_tokens = set(WORD_PATTERN.findall(code_line))
|
||||
if not code_tokens:
|
||||
continue
|
||||
overlap = len(comment_tokens & code_tokens) / len(comment_tokens)
|
||||
if overlap >= NARRATION_MIN_OVERLAP:
|
||||
findings.append(
|
||||
Signal(
|
||||
code="COMMENT_NARRATION",
|
||||
title="Comment narrates the next line of code",
|
||||
severity=SEVERITY_MEDIUM,
|
||||
axis=AXIS_ORIGIN,
|
||||
weight=2.0,
|
||||
line=number,
|
||||
evidence=comment,
|
||||
)
|
||||
)
|
||||
return findings[:8]
|
||||
|
||||
|
||||
def detect_textual(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for number, raw in enumerate(context.lines, start=1):
|
||||
for pattern, title in PLACEHOLDER_PATTERNS:
|
||||
if pattern.search(raw):
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_COMMENT", title, SEVERITY_STRONG, AXIS_QUALITY, 4.0, number, raw.strip())
|
||||
)
|
||||
break
|
||||
for pattern in AI_SIGNATURE_PATTERNS:
|
||||
if pattern.search(raw):
|
||||
findings.append(
|
||||
Signal(
|
||||
"AI_SIGNATURE",
|
||||
"AI tool self-attribution signature",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
4.0,
|
||||
number,
|
||||
raw.strip(),
|
||||
)
|
||||
)
|
||||
break
|
||||
for number, comment in context.comments:
|
||||
for pattern in PROMPT_LEAK_PATTERNS:
|
||||
if pattern.search(comment):
|
||||
findings.append(
|
||||
Signal(
|
||||
"PROMPT_LEAK",
|
||||
"Assistant conversation leakage in committed source",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
break
|
||||
emoji_hits = [(number, comment) for number, comment in context.comments if EMOJI_PATTERN.search(comment)]
|
||||
if len(emoji_hits) >= 2:
|
||||
number, comment = emoji_hits[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"EMOJI_COMMENTS",
|
||||
f"Emoji used in {len(emoji_hits)} comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
enthusiasm_hits = [(number, comment) for number, comment in context.comments if ENTHUSIASM_PATTERN.search(comment)]
|
||||
if enthusiasm_hits:
|
||||
number, comment = enthusiasm_hits[0]
|
||||
findings.append(
|
||||
Signal(
|
||||
"MARKETING_TONE",
|
||||
"Marketing-style enthusiasm in comments",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
number,
|
||||
comment,
|
||||
)
|
||||
)
|
||||
comment_text = " ".join(comment for _, comment in context.comments)
|
||||
lexicon_hits = {hit.lower() if isinstance(hit, str) else hit for hit in LLM_LEXICON_PATTERN.findall(comment_text)}
|
||||
if len(lexicon_hits) >= LLM_LEXICON_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LLM_LEXICON",
|
||||
f"{len(lexicon_hits)} distinct LLM signature words in comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
context.comments[0][0] if context.comments else 1,
|
||||
", ".join(sorted(str(hit) for hit in lexicon_hits)[:6]),
|
||||
)
|
||||
)
|
||||
em_dashes = comment_text.count("\u2014")
|
||||
if em_dashes >= EM_DASH_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"EM_DASH_OVERUSE",
|
||||
f"{em_dashes} em dashes in comments, a strong LLM prose habit",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
context.comments[0][0] if context.comments else 1,
|
||||
f"{em_dashes} em dashes",
|
||||
)
|
||||
)
|
||||
findings.extend(_narration_signals(context))
|
||||
return findings
|
||||
201
devplacepy/services/jobs/isslop/analysis/signals/vibeerrors.py
Normal file
201
devplacepy/services/jobs/isslop/analysis/signals/vibeerrors.py
Normal file
@ -0,0 +1,201 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
JS_LANGUAGES: frozenset[str] = frozenset({"javascript", "typescript"})
|
||||
|
||||
TOKEN_IN_STORAGE_PATTERN = re.compile(
|
||||
r"(?:local|session)Storage\.setItem\(\s*[\"'`][^\"'`]*(token|jwt|auth|access[_-]?token|refresh[_-]?token|api[_-]?key|secret|password|session)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
INSECURE_COOKIE_PATTERN = re.compile(r"(document\.cookie\s*=|res\.cookie\(|response\.set_cookie\(|setCookie\()", re.IGNORECASE)
|
||||
COOKIE_SECURITY_HINT = re.compile(r"httponly|secure|samesite", re.IGNORECASE)
|
||||
|
||||
USEEFFECT_BLOCK_PATTERN = re.compile(r"useEffect\s*\(\s*\(\s*\)\s*=>\s*\{", re.IGNORECASE)
|
||||
EFFECT_RESOURCE_PATTERN = re.compile(r"\b(setInterval|setTimeout|addEventListener|\.subscribe\(|\.on\()", re.IGNORECASE)
|
||||
EFFECT_CLEANUP_PATTERN = re.compile(r"return\s*(\(\s*\)\s*=>|function)")
|
||||
|
||||
CONDITIONAL_HOOK_PATTERN = re.compile(
|
||||
r"^\s*(if|for|while)\s*\([^)]*\)\s*\{[^}]{0,200}\buse(State|Effect|Ref|Memo|Callback|Context|Reducer)\s*\(",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
MIXED_CONTENT_PATTERN = re.compile(r"[\"'(]http://(?!localhost|127\.0\.0\.1|0\.0\.0\.0)[\w.-]+", re.IGNORECASE)
|
||||
JSON_PARSE_UNGUARDED_PATTERN = re.compile(r"JSON\.parse\(\s*(?:await\s+)?\w+(?:\.(?:text|body|data))?\s*\)")
|
||||
|
||||
ENV_ACCESS_PATTERN = re.compile(r"process\.env\.[A-Z0-9_]+")
|
||||
ENV_FALLBACK_HINT = re.compile(r"process\.env\.[A-Z0-9_]+\s*(\|\||\?\?|,|\))")
|
||||
PUBLIC_SECRET_PATTERN = re.compile(
|
||||
r"(NEXT_PUBLIC|VITE|REACT_APP|NUXT_PUBLIC)_[A-Z0-9_]*(SECRET|PRIVATE|SERVICE_ROLE|API_KEY|TOKEN|PASSWORD)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SUPABASE_SERVICE_ROLE_PATTERN = re.compile(r"service_role|SUPABASE_SERVICE_ROLE_KEY", re.IGNORECASE)
|
||||
|
||||
MISSING_ROUTE_GUARD_PATTERN = re.compile(r"(router\.(get|post|put|delete|patch)|app\.(get|post|put|delete|patch))\s*\(", re.IGNORECASE)
|
||||
|
||||
ENV_DENSITY_THRESHOLD: int = 4
|
||||
MAX_FINDINGS: int = 10
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def _effect_cleanup_findings(text: str) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
for match in USEEFFECT_BLOCK_PATTERN.finditer(text):
|
||||
start = match.end()
|
||||
depth = 1
|
||||
index = start
|
||||
while index < len(text) and depth > 0:
|
||||
char = text[index]
|
||||
if char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
index += 1
|
||||
body = text[start:index]
|
||||
if EFFECT_RESOURCE_PATTERN.search(body) and not EFFECT_CLEANUP_PATTERN.search(body):
|
||||
findings.append(
|
||||
Signal(
|
||||
"MISSING_EFFECT_CLEANUP",
|
||||
"useEffect registers a timer or listener without a cleanup return",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, match.start()),
|
||||
match.group(0)[:80],
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def detect_vibe_errors(context: FileContext) -> list[Signal]:
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
language = context.language
|
||||
|
||||
public_secret = PUBLIC_SECRET_PATTERN.search(text)
|
||||
if public_secret:
|
||||
findings.append(
|
||||
Signal(
|
||||
"PUBLIC_ENV_SECRET",
|
||||
"Secret exposed through a client-side public env variable",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, public_secret.start()),
|
||||
public_secret.group(0)[:100],
|
||||
)
|
||||
)
|
||||
|
||||
if language in JS_LANGUAGES or language == "html":
|
||||
token_storage = TOKEN_IN_STORAGE_PATTERN.search(text)
|
||||
if token_storage:
|
||||
findings.append(
|
||||
Signal(
|
||||
"TOKEN_IN_WEB_STORAGE",
|
||||
"Auth token stored in localStorage or sessionStorage, exposed to XSS",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
4.0,
|
||||
_line_of(text, token_storage.start()),
|
||||
token_storage.group(0)[:100],
|
||||
)
|
||||
)
|
||||
cookie_match = INSECURE_COOKIE_PATTERN.search(text)
|
||||
if cookie_match:
|
||||
window = text[cookie_match.start(): cookie_match.start() + 300]
|
||||
if not COOKIE_SECURITY_HINT.search(window):
|
||||
findings.append(
|
||||
Signal(
|
||||
"INSECURE_COOKIE",
|
||||
"Cookie set without HttpOnly, Secure or SameSite flags",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, cookie_match.start()),
|
||||
cookie_match.group(0)[:80],
|
||||
)
|
||||
)
|
||||
mixed = MIXED_CONTENT_PATTERN.search(text)
|
||||
if mixed:
|
||||
findings.append(
|
||||
Signal(
|
||||
"MIXED_CONTENT",
|
||||
"Insecure http:// resource referenced",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, mixed.start()),
|
||||
mixed.group(0)[:80],
|
||||
)
|
||||
)
|
||||
|
||||
if language in JS_LANGUAGES:
|
||||
conditional_hook = CONDITIONAL_HOOK_PATTERN.search(text)
|
||||
if conditional_hook:
|
||||
findings.append(
|
||||
Signal(
|
||||
"CONDITIONAL_HOOK",
|
||||
"React hook called conditionally, violates rules of hooks",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
2.0,
|
||||
_line_of(text, conditional_hook.start()),
|
||||
conditional_hook.group(0).strip()[:80],
|
||||
)
|
||||
)
|
||||
findings.extend(_effect_cleanup_findings(text))
|
||||
unguarded = JSON_PARSE_UNGUARDED_PATTERN.search(text)
|
||||
if unguarded and "try" not in text[max(0, unguarded.start() - 120): unguarded.start()]:
|
||||
findings.append(
|
||||
Signal(
|
||||
"UNGUARDED_JSON_PARSE",
|
||||
"JSON.parse on a response without a guard for non-JSON payloads",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
_line_of(text, unguarded.start()),
|
||||
unguarded.group(0)[:80],
|
||||
)
|
||||
)
|
||||
if SUPABASE_SERVICE_ROLE_PATTERN.search(text) and PUBLIC_SECRET_PATTERN.search(text) is None:
|
||||
env_exposed = re.search(r"(NEXT_PUBLIC|VITE|REACT_APP)[A-Z0-9_]*[\s\S]{0,60}service_role", text, re.IGNORECASE)
|
||||
if env_exposed:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SUPABASE_SERVICE_ROLE_EXPOSED",
|
||||
"Supabase service_role key reachable from client code",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_QUALITY,
|
||||
5.0,
|
||||
_line_of(text, env_exposed.start()),
|
||||
env_exposed.group(0)[:100],
|
||||
)
|
||||
)
|
||||
env_accesses = ENV_ACCESS_PATTERN.findall(text)
|
||||
env_fallbacks = ENV_FALLBACK_HINT.findall(text)
|
||||
if len(env_accesses) >= ENV_DENSITY_THRESHOLD and not env_fallbacks:
|
||||
findings.append(
|
||||
Signal(
|
||||
"ENV_NO_VALIDATION",
|
||||
f"{len(env_accesses)} process.env reads with no fallback or validation",
|
||||
SEVERITY_WEAK,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(env_accesses)} unvalidated env reads",
|
||||
)
|
||||
)
|
||||
|
||||
return findings[:MAX_FINDINGS]
|
||||
293
devplacepy/services/jobs/isslop/analysis/signals/webtells.py
Normal file
293
devplacepy/services/jobs/isslop/analysis/signals/webtells.py
Normal file
@ -0,0 +1,293 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from devplacepy.services.jobs.isslop.analysis.signals import (
|
||||
AXIS_ORIGIN,
|
||||
AXIS_QUALITY,
|
||||
SEVERITY_MEDIUM,
|
||||
SEVERITY_STRONG,
|
||||
SEVERITY_WEAK,
|
||||
FileContext,
|
||||
Signal,
|
||||
)
|
||||
|
||||
WEB_LANGUAGES: frozenset[str] = frozenset({"html", "css", "javascript", "typescript", "markdown"})
|
||||
|
||||
BUILDER_FINGERPRINTS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"cdn\.gpteng\.co|gptengineer\.js", re.IGNORECASE), "GPT Engineer runtime script"),
|
||||
(re.compile(r"lovable-uploads|data-lov-id|lovable\.app", re.IGNORECASE), "Lovable builder artifact"),
|
||||
(re.compile(r"bolt\.new", re.IGNORECASE), "Bolt.new builder reference"),
|
||||
(re.compile(r"(made|built|created|generated)\s+with\s+(lovable|bolt|v0|base44|windsurf|cursor|replit agent)", re.IGNORECASE), "AI builder attribution"),
|
||||
(re.compile(r"<meta[^>]+generator[^>]+(lovable|bolt|v0|base44|framer|durable|10web|hostinger ai)", re.IGNORECASE), "AI site generator meta tag"),
|
||||
(re.compile(r"v0\.dev|vercel\.ai/v0", re.IGNORECASE), "Vercel v0 builder reference"),
|
||||
)
|
||||
|
||||
SIGNATURE_GRADIENT_PATTERN = re.compile(r"#667eea.{0,80}#764ba2|#764ba2.{0,80}#667eea", re.IGNORECASE | re.DOTALL)
|
||||
GENERIC_PURPLE_GRADIENT_PATTERN = re.compile(
|
||||
r"linear-gradient\([^)]*(#6366f1|#8b5cf6|#a855f7|#7c3aed|#4f46e5|#818cf8)[^)]*\)", re.IGNORECASE
|
||||
)
|
||||
|
||||
SECTION_COMMENT_PATTERN = re.compile(
|
||||
r"<!--\s*([\w][\w /&'-]{0,40}\s+section|hero|navigation|navbar|footer|header)\s*-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
EDIT_NARRATION_PATTERN = re.compile(
|
||||
r"<!--\s*(updated|added|new|fixed|changed|modified|improved|enhanced|revised)\b[^>]{0,60}-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VIBE_STACK_INDICATORS: tuple[tuple[re.Pattern[str], int, str], ...] = (
|
||||
(re.compile(r"lucide", re.IGNORECASE), 5, "lucide icons"),
|
||||
(re.compile(r"data-radix|radix-ui|\bradix\b", re.IGNORECASE), 5, "radix primitives"),
|
||||
(re.compile(r"shadcn", re.IGNORECASE), 1, "shadcn/ui"),
|
||||
(re.compile(r"__next|/_next/", re.IGNORECASE), 2, "next.js runtime"),
|
||||
(re.compile(r"\bInter\b"), 3, "inter font"),
|
||||
(re.compile(r"framer-motion|data-framer", re.IGNORECASE), 2, "framer motion"),
|
||||
(re.compile(r"class=\"[^\"]*flex[^\"]*items-center[^\"]*justify-center[^\"]*\""), 6, "tailwind utility soup"),
|
||||
(re.compile(r"animate-(fade|pulse|bounce|spin|in)", re.IGNORECASE), 8, "animation utilities"),
|
||||
)
|
||||
INLINE_MONOLITH_STYLE_PATTERN = re.compile(r"<style[^>]*>([\s\S]*?)</style>", re.IGNORECASE)
|
||||
INLINE_MONOLITH_SCRIPT_PATTERN = re.compile(r"<script(?![^>]*src)[^>]*>([\s\S]*?)</script>", re.IGNORECASE)
|
||||
MONOLITH_STYLE_MIN_LINES: int = 80
|
||||
MONOLITH_SCRIPT_MIN_LINES: int = 40
|
||||
LANDING_SECTION_PATTERN = re.compile(
|
||||
r"(?:class|id)\s*=\s*[\"'][^\"']*\b(hero|features|testimonials|pricing|faq|cta)\b", re.IGNORECASE
|
||||
)
|
||||
|
||||
PLACEHOLDER_CONTACT_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\b(john|jane)(\.doe)?@example\.com\b", re.IGNORECASE), "Placeholder person email"),
|
||||
(re.compile(r"\b(info|contact|hello|support|your)@(example|yourcompany|yourdomain|company|yoursite)\.(com|org)\b", re.IGNORECASE), "Placeholder contact email"),
|
||||
(re.compile(r"\(555\)\s?\d{3}[- ]\d{4}|\+1\s?\(?555\)?[- ]\d{3,4}[- ]\d{4}|\b555-01\d{2}\b"), "Placeholder 555 phone number"),
|
||||
(re.compile(r"\b123 Main (Street|St)\b", re.IGNORECASE), "Placeholder street address"),
|
||||
(re.compile(r"\bYour Company( Name)?\b"), "Placeholder company name"),
|
||||
)
|
||||
|
||||
LOREM_PATTERN = re.compile(r"lorem ipsum dolor", re.IGNORECASE)
|
||||
DEAD_LINK_PATTERN = re.compile(r"href\s*=\s*[\"']#[\"']")
|
||||
TAILWIND_CDN_PATTERN = re.compile(r"cdn\.tailwindcss\.com", re.IGNORECASE)
|
||||
FONT_STACK_PATTERN = re.compile(r"fonts\.googleapis\.com[^\"']*(Inter|Poppins)", re.IGNORECASE)
|
||||
ICON_CDN_PATTERN = re.compile(r"(font-?awesome|cdnjs[^\"']*all\.min\.css|lucide)", re.IGNORECASE)
|
||||
|
||||
SMOOTH_SCROLL_PATTERN = re.compile(r"scroll-behavior\s*:\s*smooth|scrollIntoView\s*\(\s*\{\s*behavior\s*:\s*[\"']smooth")
|
||||
OBSERVER_REVEAL_PATTERN = re.compile(
|
||||
r"IntersectionObserver[\s\S]{0,400}classList\.(add|toggle)\s*\(\s*[\"'](visible|show|fade-in|animate|active|in-view)[\"']"
|
||||
)
|
||||
ANCHOR_SCRIPT_PATTERN = re.compile(r"querySelectorAll\s*\(\s*[\"']a\[href\^=[\"'\\]+#")
|
||||
|
||||
ROOT_PALETTE_PATTERN = re.compile(r"--primary(-color)?\s*:[\s\S]{0,300}--secondary(-color)?\s*:", re.IGNORECASE)
|
||||
UNIVERSAL_RESET_PATTERN = re.compile(
|
||||
r"\*\s*\{[^}]*margin\s*:\s*0[^}]*padding\s*:\s*0[^}]*box-sizing\s*:\s*border-box[^}]*\}", re.DOTALL
|
||||
)
|
||||
|
||||
HEADING_EMOJI_PATTERN = re.compile(r"<(h[1-3]|button)[^>]*>[^<]*[\U0001F300-\U0001FAFF✨🚀⭐✅]")
|
||||
STARTUP_COPY_PATTERN = re.compile(
|
||||
r"empower your (team|business)|built for (modern|the) (teams|businesses|ai era|ai age|modern web)"
|
||||
r"|unlock (the power|your potential|productivity)|transform your (business|workflow|ideas)"
|
||||
r"|elevate your|seamless integration|all rights reserved\.\s*(built|made) with",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
SECTION_COMMENT_THRESHOLD: int = 3
|
||||
LANDING_SECTION_THRESHOLD: int = 4
|
||||
DEAD_LINK_THRESHOLD: int = 5
|
||||
|
||||
|
||||
def _line_of(text: str, position: int) -> int:
|
||||
return text.count("\n", 0, position) + 1
|
||||
|
||||
|
||||
def detect_web_tells(context: FileContext) -> list[Signal]:
|
||||
if context.language not in WEB_LANGUAGES:
|
||||
return []
|
||||
findings: list[Signal] = []
|
||||
text = context.text
|
||||
|
||||
for pattern, title in BUILDER_FINGERPRINTS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal("AI_BUILDER_FINGERPRINT", title, SEVERITY_STRONG, AXIS_ORIGIN, 4.0, _line_of(text, match.start()), match.group(0)[:120])
|
||||
)
|
||||
|
||||
signature = SIGNATURE_GRADIENT_PATTERN.search(text)
|
||||
if signature:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SIGNATURE_GRADIENT",
|
||||
"Canonical LLM purple gradient #667eea to #764ba2",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
_line_of(text, signature.start()),
|
||||
signature.group(0)[:120],
|
||||
)
|
||||
)
|
||||
else:
|
||||
generic = GENERIC_PURPLE_GRADIENT_PATTERN.search(text)
|
||||
if generic:
|
||||
findings.append(
|
||||
Signal(
|
||||
"PURPLE_GRADIENT",
|
||||
"Indigo or violet gradient from default LLM palette",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, generic.start()),
|
||||
generic.group(0)[:120],
|
||||
)
|
||||
)
|
||||
|
||||
stack_hits: list[str] = []
|
||||
for pattern, minimum, label in VIBE_STACK_INDICATORS:
|
||||
if len(pattern.findall(text)) >= minimum:
|
||||
stack_hits.append(label)
|
||||
if len(stack_hits) >= 3:
|
||||
findings.append(
|
||||
Signal(
|
||||
"VIBE_STACK",
|
||||
f"Default AI frontend stack fingerprint ({len(stack_hits)} indicators)",
|
||||
SEVERITY_STRONG if len(stack_hits) >= 5 else SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
1,
|
||||
", ".join(stack_hits),
|
||||
)
|
||||
)
|
||||
|
||||
if context.language == "html":
|
||||
section_comments = SECTION_COMMENT_PATTERN.findall(text)
|
||||
if len(section_comments) >= SECTION_COMMENT_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SECTION_BANNER_COMMENTS",
|
||||
f"{len(section_comments)} template section banner comments",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(section_comments[:6]),
|
||||
)
|
||||
)
|
||||
edit_narrations = EDIT_NARRATION_PATTERN.findall(text)
|
||||
if edit_narrations:
|
||||
match = EDIT_NARRATION_PATTERN.search(text)
|
||||
findings.append(
|
||||
Signal(
|
||||
"EDIT_NARRATION_COMMENT",
|
||||
f"{len(edit_narrations)} chat-edit narration comments (Updated/Added/New ...)",
|
||||
SEVERITY_STRONG,
|
||||
AXIS_ORIGIN,
|
||||
3.0,
|
||||
_line_of(text, match.start()) if match else 1,
|
||||
match.group(0)[:100] if match else "",
|
||||
)
|
||||
)
|
||||
style_blocks = INLINE_MONOLITH_STYLE_PATTERN.findall(text)
|
||||
script_blocks = INLINE_MONOLITH_SCRIPT_PATTERN.findall(text)
|
||||
style_lines = sum(block.count("\n") for block in style_blocks)
|
||||
script_lines = sum(block.count("\n") for block in script_blocks)
|
||||
if style_lines >= MONOLITH_STYLE_MIN_LINES and script_lines >= MONOLITH_SCRIPT_MIN_LINES:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SINGLE_FILE_MONOLITH",
|
||||
f"Single-file page with {style_lines} CSS and {script_lines} JS lines inlined, the chat-output shape",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
f"inline style {style_lines} lines, inline script {script_lines} lines",
|
||||
)
|
||||
)
|
||||
landing_sections = {match.group(1).lower() for match in LANDING_SECTION_PATTERN.finditer(text)}
|
||||
if len(landing_sections) >= LANDING_SECTION_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"LANDING_TEMPLATE",
|
||||
"Canonical hero-features-testimonials-pricing landing structure",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
2.0,
|
||||
1,
|
||||
", ".join(sorted(landing_sections)),
|
||||
)
|
||||
)
|
||||
dead_links = DEAD_LINK_PATTERN.findall(text)
|
||||
if len(dead_links) >= DEAD_LINK_THRESHOLD:
|
||||
findings.append(
|
||||
Signal(
|
||||
"DEAD_ANCHOR_LINKS",
|
||||
f'{len(dead_links)} links pointing to href="#"',
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_QUALITY,
|
||||
1.0,
|
||||
1,
|
||||
f"{len(dead_links)} placeholder links",
|
||||
)
|
||||
)
|
||||
if TAILWIND_CDN_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("TAILWIND_CDN", "Tailwind loaded from CDN, the LLM prototype default", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, 1, "cdn.tailwindcss.com")
|
||||
)
|
||||
fonts = FONT_STACK_PATTERN.search(text)
|
||||
if fonts and ICON_CDN_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("DEFAULT_FONT_STACK", f"Default LLM font and icon stack ({fonts.group(1)} plus icon CDN)", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, _line_of(text, fonts.start()), fonts.group(0)[:100])
|
||||
)
|
||||
emoji_heading = HEADING_EMOJI_PATTERN.search(text)
|
||||
if emoji_heading:
|
||||
findings.append(
|
||||
Signal("EMOJI_HEADINGS", "Emoji inside headings or buttons", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, emoji_heading.start()), emoji_heading.group(0)[:100])
|
||||
)
|
||||
|
||||
for pattern, title in PLACEHOLDER_CONTACT_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_CONTENT", title, SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, _line_of(text, match.start()), match.group(0)[:100])
|
||||
)
|
||||
|
||||
lorem = LOREM_PATTERN.search(text)
|
||||
if lorem:
|
||||
findings.append(
|
||||
Signal("PLACEHOLDER_CONTENT", "Lorem ipsum filler text left in source", SEVERITY_MEDIUM, AXIS_QUALITY, 2.0, _line_of(text, lorem.start()), lorem.group(0))
|
||||
)
|
||||
|
||||
copy_match = STARTUP_COPY_PATTERN.search(text)
|
||||
if copy_match:
|
||||
findings.append(
|
||||
Signal("TEMPLATE_COPY", "Stock AI landing page copy phrase", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, copy_match.start()), copy_match.group(0)[:100])
|
||||
)
|
||||
|
||||
if context.language in ("javascript", "typescript", "html"):
|
||||
observer = OBSERVER_REVEAL_PATTERN.search(text)
|
||||
if observer:
|
||||
findings.append(
|
||||
Signal(
|
||||
"SCROLL_REVEAL_BOILERPLATE",
|
||||
"IntersectionObserver fade-in reveal boilerplate",
|
||||
SEVERITY_MEDIUM,
|
||||
AXIS_ORIGIN,
|
||||
1.0,
|
||||
_line_of(text, observer.start()),
|
||||
observer.group(0)[:100],
|
||||
)
|
||||
)
|
||||
if SMOOTH_SCROLL_PATTERN.search(text) and ANCHOR_SCRIPT_PATTERN.search(text):
|
||||
findings.append(
|
||||
Signal("SMOOTH_SCROLL_BOILERPLATE", "Smooth-scroll anchor polyfill boilerplate", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, 1, "smooth scroll + anchor querySelectorAll")
|
||||
)
|
||||
|
||||
if context.language in ("css", "html"):
|
||||
palette = ROOT_PALETTE_PATTERN.search(text)
|
||||
if palette:
|
||||
findings.append(
|
||||
Signal("DEFAULT_CSS_PALETTE", "Generic --primary/--secondary variable palette", SEVERITY_MEDIUM, AXIS_ORIGIN, 1.0, _line_of(text, palette.start()), palette.group(0)[:80])
|
||||
)
|
||||
reset = UNIVERSAL_RESET_PATTERN.search(text)
|
||||
if reset:
|
||||
findings.append(
|
||||
Signal("UNIVERSAL_RESET", "Universal margin/padding/box-sizing reset block", SEVERITY_WEAK, AXIS_ORIGIN, 1.0, _line_of(text, reset.start()), "* { margin:0; padding:0; box-sizing:border-box }")
|
||||
)
|
||||
|
||||
return findings[:14]
|
||||
170
devplacepy/services/jobs/isslop/analysis/templates.py
Normal file
170
devplacepy/services/jobs/isslop/analysis/templates.py
Normal file
@ -0,0 +1,170 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
README_SCAN_BYTES: int = 6144
|
||||
SATURATION: float = 16.0
|
||||
README_CAP: float = 18.0
|
||||
CONSTELLATION_FREEBIES: int = 3
|
||||
CONSTELLATION_STEP: float = 1.2
|
||||
CONSTELLATION_CAP: float = 12.0
|
||||
|
||||
KNOWN_TEMPLATE_SLUGS: tuple[tuple[str, float], ...] = (
|
||||
("next-js-boilerplate", 9.0),
|
||||
("nextjs-boilerplate", 9.0),
|
||||
("next-js-blog-boilerplate", 9.0),
|
||||
("vercel-ai-chatbot", 9.0),
|
||||
("ai-chatbot", 9.0),
|
||||
("create-t3-app", 9.0),
|
||||
("fullstack-nextjs-app-template", 9.0),
|
||||
("my-v0-project", 9.0),
|
||||
("nextjs-starter", 8.0),
|
||||
("saas-starter", 8.0),
|
||||
("astro-boilerplate", 8.0),
|
||||
("vite-project", 7.0),
|
||||
("my-t3-app", 9.0),
|
||||
("my-app", 5.0),
|
||||
("chatbot", 5.0),
|
||||
)
|
||||
|
||||
KNOWN_TEMPLATE_AUTHORS: tuple[str, ...] = (
|
||||
"ixartz",
|
||||
"uiux lab",
|
||||
"shoubhit dash",
|
||||
"vercel",
|
||||
"shadcn",
|
||||
"steven tey",
|
||||
)
|
||||
|
||||
GENERIC_IDENTITY_PATTERN = re.compile(r"\b(boilerplate|starter|template|scaffold)\b", re.IGNORECASE)
|
||||
|
||||
README_MARKERS: tuple[tuple[str, float, str], ...] = (
|
||||
(r"bootstrapped with .{0,40}create", 10.0, "bootstrapped-with"),
|
||||
(r"create[- ]t3[- ]app|\bt3 stack\b", 9.0, "t3-stack"),
|
||||
(r"(free,? )?open[- ]source (template|starter)|\b(app|chatbot|website) template\b|template built with", 9.0, "template-marketing"),
|
||||
(r"\bboilerplate\b", 8.0, "boilerplate-wording"),
|
||||
(r"\bstarter\b", 6.0, "starter-wording"),
|
||||
(r"getting started.{0,160}first,? run the development server", 7.0, "default-readme"),
|
||||
(r"deploy your own|one[- ]click deploy|deploy with vercel", 5.0, "deploy-your-own"),
|
||||
(r"https?://demo\.|/demo/", 4.0, "hosted-demo"),
|
||||
(r"\bsponsors?\b", 3.0, "sponsor-section"),
|
||||
)
|
||||
|
||||
SCAFFOLD_ARTIFACTS: tuple[str, ...] = (
|
||||
".storybook",
|
||||
".husky",
|
||||
".devcontainer",
|
||||
".github/FUNDING.yml",
|
||||
".github/dependabot.yml",
|
||||
".github/workflows",
|
||||
"commitlint.config.js",
|
||||
"commitlint.config.ts",
|
||||
"lint-staged.config.js",
|
||||
".lintstagedrc",
|
||||
".lintstagedrc.js",
|
||||
"renovate.json",
|
||||
"playwright.config.js",
|
||||
"playwright.config.ts",
|
||||
"vitest.config.js",
|
||||
"vitest.config.ts",
|
||||
"drizzle.config.ts",
|
||||
"components.json",
|
||||
"crowdin.yml",
|
||||
".releaserc",
|
||||
".releaserc.json",
|
||||
"docker-compose.yml",
|
||||
"Dockerfile",
|
||||
"sentry.client.config.ts",
|
||||
"sentry.server.config.ts",
|
||||
"CHANGELOG.md",
|
||||
"CODE_OF_CONDUCT.md",
|
||||
"oxlint.config.ts",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TemplateEvidence:
|
||||
score: float = 0.0
|
||||
markers: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _manifest_markers(root: Path) -> tuple[float, list[str]]:
|
||||
manifest = root / "package.json"
|
||||
if not manifest.exists():
|
||||
return 0.0, []
|
||||
try:
|
||||
body = json.loads(manifest.read_text(encoding="utf-8", errors="replace"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return 0.0, []
|
||||
if not isinstance(body, dict):
|
||||
return 0.0, []
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
name = str(body.get("name") or "").lower()
|
||||
matched_slug = next(
|
||||
((slug, slug_weight) for slug, slug_weight in KNOWN_TEMPLATE_SLUGS if slug in name), None
|
||||
)
|
||||
if matched_slug:
|
||||
weight += matched_slug[1]
|
||||
markers.append(f"package name matches known template '{matched_slug[0]}'")
|
||||
identity = f"{name} {str(body.get('description') or '')}"
|
||||
if not matched_slug and GENERIC_IDENTITY_PATTERN.search(identity):
|
||||
weight += 7.0
|
||||
markers.append("package identity describes itself as a boilerplate/starter/template")
|
||||
author = str(body.get("author") or "").lower()
|
||||
if any(known in author for known in KNOWN_TEMPLATE_AUTHORS):
|
||||
weight += 6.0
|
||||
markers.append("package author is a known template publisher")
|
||||
if "ct3aMetadata" in body:
|
||||
weight += 10.0
|
||||
markers.append("create-t3-app scaffold metadata present")
|
||||
return weight, markers
|
||||
|
||||
|
||||
def _readme_markers(root: Path) -> tuple[float, list[str]]:
|
||||
readme = next((root / name for name in ("README.md", "readme.md", "README") if (root / name).exists()), None)
|
||||
if readme is None:
|
||||
return 0.0, []
|
||||
try:
|
||||
text = readme.read_text(encoding="utf-8", errors="replace")[:README_SCAN_BYTES]
|
||||
except OSError:
|
||||
return 0.0, []
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
for pattern, marker_weight, label in README_MARKERS:
|
||||
if re.search(pattern, text, re.IGNORECASE | re.DOTALL):
|
||||
weight += marker_weight
|
||||
markers.append(f"README carries template marker: {label}")
|
||||
return min(weight, README_CAP), markers
|
||||
|
||||
|
||||
def _constellation_markers(root: Path) -> tuple[float, list[str]]:
|
||||
present = [artifact for artifact in SCAFFOLD_ARTIFACTS if (root / artifact).exists()]
|
||||
extra = max(0, len(present) - CONSTELLATION_FREEBIES)
|
||||
if extra <= 0:
|
||||
return 0.0, []
|
||||
weight = min(extra * CONSTELLATION_STEP, CONSTELLATION_CAP)
|
||||
return weight, [f"kitchen-sink scaffold constellation: {len(present)} standard scaffold artifacts"]
|
||||
|
||||
|
||||
def detect_template(root: Path) -> TemplateEvidence:
|
||||
weight = 0.0
|
||||
markers: list[str] = []
|
||||
for probe in (_manifest_markers, _readme_markers, _constellation_markers):
|
||||
try:
|
||||
probe_weight, probe_markers = probe(root)
|
||||
except OSError as error:
|
||||
logger.warning("Template probe failed in %s: %s", root, error)
|
||||
continue
|
||||
weight += probe_weight
|
||||
markers.extend(probe_markers)
|
||||
score = round(100.0 * (1.0 - math.exp(-weight / SATURATION)), 1) if weight > 0 else 0.0
|
||||
return TemplateEvidence(score=score, markers=markers)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user