devranta/examples/crawler/main.py

# main.py
import asyncio
import logging
import signal

from crawler import DevRantCrawler
from database import DatabaseManager

from devranta.api import Api

# --- Configuration ---
DB_FILE = "devrant.sqlite"
CONCURRENT_RANT_CONSUMERS = 10  # How many rants to process at once
CONCURRENT_USER_CONSUMERS = 5  # How many user profiles to fetch at once


async def main():
    """Initializes and runs the crawler."""
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s [%(levelname)s] - %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    api = Api()

    async with DatabaseManager(DB_FILE) as db:
        crawler = DevRantCrawler(
            api=api,
            db=db,
            rant_consumers=CONCURRENT_RANT_CONSUMERS,
            user_consumers=CONCURRENT_USER_CONSUMERS,
        )

        # Set up a signal handler for graceful shutdown on Ctrl+C
        loop = asyncio.get_running_loop()
        for sig in (signal.SIGINT, signal.SIGTERM):
            loop.add_signal_handler(
                sig, lambda s=sig: asyncio.create_task(crawler.shutdown())
            )

        await crawler.run()


if __name__ == "__main__":
    try:
        asyncio.run(main())
    except KeyboardInterrupt:
        logging.info("Main loop interrupted. Exiting.")