@@ -0,0 +1,158 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
from devplacepy.cli._shared import _audit_cli
|
||||
|
||||
|
||||
def _prepare():
|
||||
from devplacepy.database import init_db
|
||||
|
||||
init_db()
|
||||
|
||||
|
||||
def _credentials(args) -> dict:
|
||||
return {
|
||||
"username": args.login_user or "",
|
||||
"email": args.login_email or "",
|
||||
"password": args.login_password or os.environ.get("COMMUNITY_PASSWORD", ""),
|
||||
}
|
||||
|
||||
|
||||
def _print_report(report):
|
||||
mode = "DRY RUN" if report.dry_run else "LIVE"
|
||||
print(f"\n[{mode}] status: {report.status}")
|
||||
print(f"collected: {report.collected}")
|
||||
for kind, counts in report.pushed.items():
|
||||
print(f" {kind}: {counts or 'nothing'}")
|
||||
for error in report.errors[:10]:
|
||||
print(f" problem: {error}")
|
||||
|
||||
|
||||
def cmd_community_discover(args):
|
||||
from devplacepy.services.community.logbook import LogBook
|
||||
from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
|
||||
from devplacepy.services.community.spec import STATUS_READY
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
from devplacepy.services.community.discoverer import slug_for_query
|
||||
|
||||
_prepare()
|
||||
runner = CommunityRunner()
|
||||
existing = store.get(slug_for_query(args.query))
|
||||
if existing and not args.relearn and not args.dry_run:
|
||||
print(f"Community '{existing['slug']}' already exists ({existing['status']}); pass --relearn to learn it again")
|
||||
if args.run and existing["status"] == STATUS_READY:
|
||||
with LogBook(echo=True):
|
||||
_print_report(asyncio.run(runner.run(existing, dry_run=False, trigger=TRIGGER_CLI)))
|
||||
return
|
||||
with LogBook(echo=True):
|
||||
result, row, _ = asyncio.run(
|
||||
runner.discover(
|
||||
args.query,
|
||||
"cli",
|
||||
dry_run=args.dry_run,
|
||||
relearn_slug=existing["slug"] if existing else None,
|
||||
credentials=_credentials(args),
|
||||
)
|
||||
)
|
||||
print(f"\n{result.slug or '-'}: {result.status}" + (f" ({result.error})" if result.error else ""))
|
||||
if result.status == STATUS_READY:
|
||||
for feed in result.schema["feeds"]:
|
||||
print(f" feed {feed['id']}: {feed['type']} {feed['url']} comments={feed['comments']['mode']} pagination={feed['pagination']['style']}")
|
||||
print(f" sample: { {kind: len(rows) for kind, rows in result.sample['records'].items()} }")
|
||||
if args.dry_run:
|
||||
print("dry run: nothing was saved")
|
||||
else:
|
||||
_audit_cli("cli.community.discover", f"CLI discovered community {result.slug}: {result.status}", target_label=result.slug, metadata={"status": result.status})
|
||||
if result.status != STATUS_READY and not args.dry_run and result.status != "login_required":
|
||||
sys.exit(1)
|
||||
if args.run and row and result.status == STATUS_READY and not args.dry_run:
|
||||
with LogBook(echo=True):
|
||||
_print_report(asyncio.run(runner.run(store.get(row["uid"]), dry_run=False, trigger=TRIGGER_CLI)))
|
||||
|
||||
|
||||
def cmd_community_run(args):
|
||||
from devplacepy.services.community.logbook import LogBook
|
||||
from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
row = store.get(args.slug)
|
||||
if row is None:
|
||||
print(f"Unknown community '{args.slug}'")
|
||||
sys.exit(1)
|
||||
with LogBook(echo=True):
|
||||
report = asyncio.run(CommunityRunner().run(row, dry_run=args.dry_run, trigger=TRIGGER_CLI))
|
||||
_print_report(report)
|
||||
if not args.dry_run:
|
||||
_audit_cli("cli.community.run", f"CLI ran community {row['slug']}: {report.status}", target_label=row["slug"], metadata={"status": report.status, "collected": report.collected})
|
||||
if report.status == "failed":
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_community_login(args):
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
row = store.get(args.slug)
|
||||
if row is None:
|
||||
print(f"Unknown community '{args.slug}'")
|
||||
sys.exit(1)
|
||||
if args.clear:
|
||||
store.update(row["uid"], login_username="", login_email="", login_password="")
|
||||
print(f"Removed the stored account of {row['slug']}")
|
||||
else:
|
||||
secrets = _credentials(args)
|
||||
if not secrets["password"] or not (secrets["username"] or secrets["email"]):
|
||||
print("Give --login-user or --login-email, and --login-password (or COMMUNITY_PASSWORD)")
|
||||
sys.exit(1)
|
||||
store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"])
|
||||
print(f"Stored the account for {row['slug']}; run 'community discover \"{row['query']}\" --relearn' to learn behind the login")
|
||||
_audit_cli("cli.community.login", f"CLI changed the stored account of community {row['slug']}", target_label=row["slug"])
|
||||
|
||||
|
||||
def cmd_community_list(args):
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
rows = store.list()
|
||||
if not rows:
|
||||
print("No communities")
|
||||
return
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['slug']:<28} {row['status']:<15} runs={row.get('run_count') or 0:<4} "
|
||||
f"last={row.get('last_run_at') or '-'} next={row.get('next_run_at') or '-'}"
|
||||
)
|
||||
|
||||
|
||||
def add_login_arguments(parser):
|
||||
parser.add_argument("--login-user", help="username of an account on the community")
|
||||
parser.add_argument("--login-email", help="email of an account on the community")
|
||||
parser.add_argument("--login-password", help="password (or set COMMUNITY_PASSWORD)")
|
||||
|
||||
|
||||
def register_community(subparsers):
|
||||
community = subparsers.add_parser("community", help="External community discovery and sync")
|
||||
sub = community.add_subparsers(title="action", dest="action")
|
||||
discover = sub.add_parser("discover", help="Discover, learn and generate collectors for a community")
|
||||
discover.add_argument("query", help="community name, address or search phrase")
|
||||
discover.add_argument("--run", action="store_true", help="also run the collectors once, live")
|
||||
discover.add_argument("--relearn", action="store_true", help="learn again even if the community exists")
|
||||
discover.add_argument("--dry-run", action="store_true", help="learn and report, save nothing")
|
||||
add_login_arguments(discover)
|
||||
discover.set_defaults(func=cmd_community_discover)
|
||||
run = sub.add_parser("run", help="Run the generated collectors of a community")
|
||||
run.add_argument("slug")
|
||||
run.add_argument("--dry-run", action="store_true", help="collect and validate, write nothing, keep the watermark")
|
||||
run.set_defaults(func=cmd_community_run)
|
||||
login = sub.add_parser("login", help="Store or remove the account used to log in to a community")
|
||||
login.add_argument("slug")
|
||||
login.add_argument("--clear", action="store_true", help="remove the stored account")
|
||||
add_login_arguments(login)
|
||||
login.set_defaults(func=cmd_community_login)
|
||||
listing = sub.add_parser("list", help="List communities")
|
||||
listing.set_defaults(func=cmd_community_list)
|
||||
Reference in New Issue
Block a user