devplacepy/devplacepy/cli/migrate.py

# retoor <retoor@molodetz.nl>

import sys
from devplacepy.cli._shared import _audit_cli


def _crc32(path):
    import zlib

    crc = 0
    with open(path, "rb") as handle:
        while True:
            chunk = handle.read(1024 * 1024)
            if not chunk:
                break
            crc = zlib.crc32(chunk, crc)
    return crc & 0xFFFFFFFF


def _migrate_file(source, dest, dry_run, report):
    import os
    import shutil

    if not source.exists():
        return
    if source.resolve() == dest.resolve():
        return
    size = source.stat().st_size
    if dest.exists():
        if dest.stat().st_size == size and _crc32(dest) == _crc32(source):
            report.append(("done", source, dest, size))
            if not dry_run:
                source.unlink()
            return
        report.append(("conflict", source, dest, size))
        return
    report.append(("move", source, dest, size))
    if dry_run:
        return
    dest.parent.mkdir(parents=True, exist_ok=True)
    tmp = dest.with_name(dest.name + ".migrating")
    shutil.copyfile(source, tmp)
    with open(tmp, "rb") as handle:
        os.fsync(handle.fileno())
    if tmp.stat().st_size != size or _crc32(tmp) != _crc32(source):
        tmp.unlink(missing_ok=True)
        raise RuntimeError(f"verification failed copying {source} -> {dest}")
    os.replace(tmp, dest)
    source.unlink()


def _prune_empty_dirs(root):
    if not root.exists():
        return
    for path in sorted(root.rglob("*"), reverse=True):
        if path.is_dir():
            try:
                path.rmdir()
            except OSError:
                pass
    try:
        root.rmdir()
    except OSError:
        pass


def _migrate_tree(source, dest, dry_run, report):
    if not source.exists():
        return
    if source.resolve() == dest.resolve():
        return
    for child in sorted(source.rglob("*")):
        if child.is_file():
            _migrate_file(child, dest / child.relative_to(source), dry_run, report)
    if not dry_run:
        _prune_empty_dirs(source)


def _db_is_locked(path):
    import sqlite3

    try:
        conn = sqlite3.connect(str(path), timeout=0.5)
        try:
            conn.execute("BEGIN IMMEDIATE")
            conn.rollback()
            return False
        finally:
            conn.close()
    except sqlite3.OperationalError:
        return True


def _checkpoint(path):
    import sqlite3

    conn = sqlite3.connect(str(path), timeout=5)
    try:
        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
        conn.commit()
    finally:
        conn.close()


def _migrate_db(source, dest, dry_run, report):
    if not source.exists():
        return
    if source.resolve() == dest.resolve():
        return
    if _db_is_locked(source):
        raise RuntimeError(
            f"{source} is locked - stop the app before running migrate-data"
        )
    if not dry_run:
        _checkpoint(source)
    _migrate_file(source, dest, dry_run, report)
    for suffix in ("-wal", "-shm"):
        _migrate_file(
            source.with_name(source.name + suffix),
            dest.with_name(dest.name + suffix),
            dry_run,
            report,
        )


def cmd_emoji_sync(args):
    from devplacepy.rendering import EMOJI_JS_PATH, write_emoji_module

    count = write_emoji_module()
    _audit_cli("cli.emoji.sync", f"CLI regenerated {count} emoji shortcodes", metadata={"count": count})
    print(f"Wrote {count} emoji shortcodes to {EMOJI_JS_PATH}")


def cmd_migrate_data(args):
    import os
    from pathlib import Path
    from collections import Counter
    from devplacepy import config

    base = config.BASE_DIR
    home = Path.home()
    dry = args.dry_run
    report = []

    config.ensure_data_dirs()

    db_items = []
    if config.DATABASE_URL == f"sqlite:///{config.DATA_DIR / 'devplace.db'}":
        db_items.append((base / "devplace.db", config.DATA_DIR / "devplace.db"))
    else:
        print("Skipping main DB: DEVPLACE_DATABASE_URL points outside the data dir.")
    if not os.environ.get("DEVII_TASKS_DB"):
        db_items.append((base / "devii_tasks.db", config.DEVII_TASKS_DB))
    if not os.environ.get("DEVII_LESSONS_DB"):
        db_items.append((base / "devii_lessons.db", config.DEVII_LESSONS_DB))

    file_items = [
        (base / name, config.KEYS_DIR / name)
        for name in (
            "notification-private.pem",
            "notification-private.pkcs8.pem",
            "notification-public.pem",
        )
    ]
    registry_dest = config.BOT_DIR / "article_registry.json"
    registry_sources = [
        path
        for path in (
            home / ".dpbot_article_registry.json",
            base / ".dpbot_article_registry.json",
        )
        if path.exists()
    ]
    registry_sources.sort(key=lambda path: path.stat().st_mtime, reverse=True)
    if registry_sources:
        file_items.append((registry_sources[0], registry_dest))
    for stale in registry_sources[1:]:
        print(f"Leaving older duplicate registry untouched: {stale}")

    legacy_var = base / "var"
    tree_items = [
        (base / "devplacepy" / "static" / "uploads", config.UPLOADS_DIR),
        (home / ".devplace_bots", config.BOT_DIR),
    ]
    for sub_name in ("container_workspaces", "zips", "zip_staging", "fork_staging"):
        tree_items.append((legacy_var / sub_name, config.DATA_PATHS[sub_name]))

    try:
        for source, dest in db_items:
            _migrate_db(source, dest, dry, report)
        for source, dest in file_items:
            _migrate_file(source, dest, dry, report)
        for source, dest in tree_items:
            _migrate_tree(source, dest, dry, report)
    except RuntimeError as exc:
        print(f"ERROR: {exc}")
        sys.exit(1)

    if not report:
        print("Nothing to migrate; the data directory is already consolidated.")
        return
    for status, source, dest, size in report:
        print(f"  [{status}] {source} -> {dest} ({size} bytes)")
    counts = Counter(status for status, *_ in report)
    print()
    print(
        ("Planned: " if dry else "Migrated: ")
        + ", ".join(f"{count} {status}" for status, count in sorted(counts.items()))
    )
    if any(status == "conflict" for status, *_ in report):
        print(
            "Conflicts left both source and destination untouched; resolve them by hand."
        )
    if dry:
        print("Dry run - nothing changed. Re-run without --dry-run to apply.")


def register_migrate(subparsers):
    subparsers.add_parser(
        "emoji-sync",
        help="Regenerate static/js/emoji-shortcodes.js from the emoji library",
    ).set_defaults(func=cmd_emoji_sync)

    migrate = subparsers.add_parser(
        "migrate-data",
        help="Relocate legacy runtime files into the consolidated data/ directory",
    )
    migrate.add_argument(
        "--dry-run",
        action="store_true",
        help="Print the source-to-destination plan without changing anything",
    )
    migrate.set_defaults(func=cmd_migrate_data)