retoorandClaude Sonnet 5.5
02ab747cd0
DevPlace CI / test (push) Failing after 44m45s
Continuous Delivery could never be turned off once a crop was planted. Add a one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant, autoreplant claim or Refactor. Autoreplant now also respects crop locks through the shared _crop_lock predicate. Plan and measurements in happyplanting.md. Also includes the pending community sync, node network and related work in the tree; the full suite passed (4999) with all of it. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
234 lines
13 KiB
Python
234 lines
13 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from .._shared import endpoint, field
|
|
|
|
SLUG = field("slug", "path", "string", True, "other-example-org", "Community slug, as listed by GET /admin/communities.")
|
|
ACCOUNT = [
|
|
field("login_username", "form", "string", False, "bot", "Username of an account on the community, for a login-protected community."),
|
|
field("login_email", "form", "string", False, "bot@example.com", "Email of that account, when the community logs in by email."),
|
|
field("login_password", "form", "string", False, "", "Password of that account. Stored for the collectors and never returned by any endpoint."),
|
|
field("login_totp_secret", "form", "string", False, "", "Authenticator (TOTP) secret of that account when it uses two-factor codes. Stored for the collectors and never returned by any endpoint."),
|
|
]
|
|
|
|
ITEM_SAMPLE = {
|
|
"uid": "COMMUNITY_UID",
|
|
"slug": "other-example-org",
|
|
"name": "Other Example",
|
|
"origin": "other.example.org",
|
|
"url": "https://other.example.org/",
|
|
"status": "ready",
|
|
"enabled": True,
|
|
"interval_seconds": 86400,
|
|
"run_count": 3,
|
|
"failure_count": 0,
|
|
"last_run_at": "2026-10-14T10:00:00+00:00",
|
|
"next_run_at": "2026-10-15T10:00:00+00:00",
|
|
"has_login": False,
|
|
}
|
|
|
|
GROUP = {
|
|
"slug": "community",
|
|
"title": "Community discovery",
|
|
"admin": True,
|
|
"intro": """
|
|
# Community discovery
|
|
|
|
Administrators add an external community by name or address. The system searches for it, detects
|
|
its feeds (RSS, Atom, JSON or HTML lists, with JavaScript rendering when a page needs it), learns
|
|
the structure once, records the community's description, rules and size, generates collectors and
|
|
then runs them on a schedule, pushing users, posts and comments through the
|
|
[community sync API](/docs/sync.html). Members never see these communities.
|
|
|
|
- **Statuses.** `pending`, `learning`, `ready`, `login_required`, `failed` and `failing` (five failed
|
|
runs in a row: the community is disabled and needs a human).
|
|
- **Login-protected communities.** Give `login_username` or `login_email` plus `login_password`; the
|
|
system finds the login form (or, for a page that needs JavaScript, drives it in the browser), logs
|
|
in before discovery and before every run. The password is never returned.
|
|
- **Dry run.** `dry_run=true` on a run validates and logs everything and writes nothing.
|
|
- **Sync log.** Every run records a verdict, exact counts per outcome and one row per record that needs an explanation (created, updated, skipped, rejected, failed, warnings, errors) with the reason in plain words. Read it with the sync log endpoint, per run or across the most recent runs.
|
|
- **Asynchronous.** Adding, relearning and running return at once; read the result from the detail
|
|
endpoint.
|
|
- **AI assistance.** For HTML pages whose structure the heuristics cannot read, the internal AI model
|
|
proposes selectors, and a proposal is only used when it matches the sampled pages.
|
|
|
|
Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html).
|
|
""",
|
|
"endpoints": [
|
|
endpoint(
|
|
id="community-list",
|
|
method="GET",
|
|
path="/admin/communities",
|
|
title="List communities",
|
|
summary="Every community with status, schedule and last run, plus the state of the sync switches.",
|
|
auth="admin",
|
|
interactive=True,
|
|
sample_response={"items": [ITEM_SAMPLE], "sync_enabled": True, "service_enabled": True, "service_dry_run": True},
|
|
),
|
|
endpoint(
|
|
id="community-add",
|
|
method="POST",
|
|
path="/admin/communities",
|
|
title="Add a community",
|
|
summary="Starts discovery in the background and redirects to the community.",
|
|
auth="admin",
|
|
encoding="form",
|
|
interactive=False,
|
|
params=[field("query", "form", "string", True, "https://forum.example.com", "Community name or address."), *ACCOUNT],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
|
),
|
|
endpoint(
|
|
id="community-detail",
|
|
method="GET",
|
|
path="/admin/communities/{slug}",
|
|
title="Community detail",
|
|
summary="Metadata, learned feeds, totals and recent runs with their logs.",
|
|
auth="admin",
|
|
interactive=True,
|
|
params=[SLUG],
|
|
sample_response={
|
|
"community": ITEM_SAMPLE,
|
|
"rules": "Be kind.",
|
|
"totals": {"posts_created": 120},
|
|
"feeds": [{"id": "feed1", "type": "json_api", "url": "https://other.example.org/api/posts", "comments": "inline", "pagination": "page"}],
|
|
"runs": [{"status": "ok", "dry_run": 0, "trigger": "service", "collected": {"users": 4, "posts": 20, "comments": 31}}],
|
|
"busy": False,
|
|
},
|
|
),
|
|
endpoint(
|
|
id="community-schema",
|
|
method="GET",
|
|
path="/admin/communities/{slug}/schema",
|
|
title="Learned schema",
|
|
summary="The complete learned schema of a community as JSON, editable by hand.",
|
|
auth="admin",
|
|
params=[SLUG],
|
|
sample_response={"community": ITEM_SAMPLE, "schema_text": "{...}", "has_previous": False, "errors": [], "notice": ""},
|
|
),
|
|
endpoint(
|
|
id="community-schema-save",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/schema",
|
|
title="Replace the learned schema",
|
|
summary="Validate a schema, check it against a small live sample, and only then store it. The previous schema is kept.",
|
|
auth="admin",
|
|
encoding="form",
|
|
destructive=True,
|
|
params=[SLUG, field("schema", "form", "string", True, "{...}", "The complete schema as a JSON object string.")],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema", "data": {"posts": 5, "comments": 12}},
|
|
notes=["`400` lists every structural problem; `422` means the live sample collected no post. Nothing is stored in either case. Audited as `admin.community.schema`."],
|
|
),
|
|
endpoint(
|
|
id="community-schema-restore",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/schema/restore",
|
|
title="Restore the previous schema",
|
|
summary="Swap back to the schema that was replaced by the last save.",
|
|
auth="admin",
|
|
encoding="form",
|
|
params=[SLUG],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema"},
|
|
notes=["`409` when there is no previous schema. Audited as `admin.community.schema_restore`."],
|
|
),
|
|
endpoint(
|
|
id="community-ledger",
|
|
method="GET",
|
|
path="/admin/communities/{slug}/ledger",
|
|
title="Sync log",
|
|
summary="Every record a run handled, with the result and the reason, problems first. Without run it spans the most recent runs.",
|
|
auth="admin",
|
|
interactive=True,
|
|
params=[
|
|
SLUG,
|
|
field("run", "query", "string", False, "RUN_UID", "A run uid from the community detail. Empty lists the most recent runs."),
|
|
field("kind", "query", "string", False, "posts", "users, posts, comments or feed."),
|
|
field("outcome", "query", "string", False, "rejected", "created, updated, unchanged, stale, skipped, warning, rejected, failed or error."),
|
|
field("reason", "query", "string", False, "content_refused", "A reason key from options.reason."),
|
|
field("q", "query", "string", False, "", "Search the title, name, origin id and detail."),
|
|
field("page", "query", "integer", False, "1", "Page number, 50 records per page, at most 200 pages."),
|
|
],
|
|
sample_response={
|
|
"community": ITEM_SAMPLE,
|
|
"scope": "run",
|
|
"run": {"uid": "RUN_UID", "status": "partial", "dry_run": False, "verdict": "attention", "headline": "Synced, with 1 rejected."},
|
|
"verdict": {"level": "attention", "title": "Look at this", "headline": "Synced, with 1 rejected."},
|
|
"tiles": [{"key": "created", "label": "Created", "tone": "good", "count": 99, "listed": 99}, {"key": "rejected", "label": "Rejected", "tone": "bad", "count": 1, "listed": 1}],
|
|
"items": [
|
|
{
|
|
"kind": "posts",
|
|
"outcome": "rejected",
|
|
"reason": "content_refused",
|
|
"reason_title": "Content filter",
|
|
"label": "A post the filter refused",
|
|
"origin_uid": "h:4215bd02760d2fa7415c3a7952b0966f",
|
|
"detail": "",
|
|
}
|
|
],
|
|
"pagination": {"page": 1, "per_page": 50, "total": 1, "total_pages": 1, "capped": False},
|
|
"filters": {"outcome": "rejected"},
|
|
"options": {"kind": ["posts"], "reason": [{"value": "content_refused", "title": "Content filter"}]},
|
|
},
|
|
),
|
|
endpoint(
|
|
id="community-run",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/run",
|
|
title="Run now",
|
|
summary="Collects and pushes in the background. A live run needs the sync_enabled setting.",
|
|
auth="admin",
|
|
encoding="form",
|
|
interactive=False,
|
|
params=[SLUG, field("dry_run", "form", "boolean", False, "false", "Validate and log, write nothing, keep the watermark.")],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
|
),
|
|
endpoint(
|
|
id="community-relearn",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/relearn",
|
|
title="Relearn",
|
|
summary="Learns the structure again with the stored account. A failed relearn keeps the working schema.",
|
|
auth="admin",
|
|
encoding="form",
|
|
interactive=False,
|
|
params=[SLUG],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
|
),
|
|
endpoint(
|
|
id="community-update",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/update",
|
|
title="Update schedule or account",
|
|
summary="Changes the interval, the enabled flag and the stored account. An empty password keeps the stored one.",
|
|
auth="admin",
|
|
encoding="form",
|
|
interactive=False,
|
|
params=[
|
|
SLUG,
|
|
field("interval_seconds", "form", "integer", True, "86400", "Seconds between runs, 900 to 2592000."),
|
|
field("enabled", "form", "boolean", False, "true", "Whether the scheduled service runs this community."),
|
|
*ACCOUNT,
|
|
field("clear_login", "form", "boolean", False, "false", "Remove the stored account."),
|
|
field("alias", "form", "boolean", False, "false", "Store authors under permanent pseudonyms and rewrite every mention. Refused once the origin has records under original names."),
|
|
field("translate_to", "form", "string", False, "en", "Language code to translate collected posts and comments into; empty turns it off."),
|
|
field("sample_percent", "form", "integer", False, "100", "Percentage of new posts accepted, 1 to 100, chosen deterministically per post."),
|
|
field("min_comments", "form", "integer", False, "0", "Minimum comments collected in the run for a new post to be accepted."),
|
|
field("topics", "form", "string", False, "devlog, rant", "Comma separated topic allow-list; empty accepts all."),
|
|
field("jitter_percent", "form", "integer", False, "0", "Randomise the next run time by up to this percentage of the interval, 0 to 50."),
|
|
field("collect_attachments", "form", "boolean", False, "false", "Download the images and PDFs linked from posts and comments (at most 4 per post, 40 and 25 MB per run) and store them as attachments."),
|
|
],
|
|
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
|
),
|
|
endpoint(
|
|
id="community-delete",
|
|
method="POST",
|
|
path="/admin/communities/{slug}/delete",
|
|
title="Delete a community",
|
|
summary="Moves the community to the trash and stops its collectors. Content already synchronised stays.",
|
|
auth="admin",
|
|
encoding="form",
|
|
interactive=False,
|
|
params=[SLUG],
|
|
sample_response={"ok": True, "redirect": "/admin/communities"},
|
|
),
|
|
],
|
|
}
|