# main.py import asyncio import logging import signal from crawler import DevRantCrawler from database import DatabaseManager from devranta.api import Api # --- Configuration --- DB_FILE = "devrant.sqlite" CONCURRENT_RANT_CONSUMERS = 10 # How many rants to process at once CONCURRENT_USER_CONSUMERS = 5 # How many user profiles to fetch at once async def main(): """Initializes and runs the crawler.""" logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) api = Api() async with DatabaseManager(DB_FILE) as db: crawler = DevRantCrawler( api=api, db=db, rant_consumers=CONCURRENT_RANT_CONSUMERS, user_consumers=CONCURRENT_USER_CONSUMERS, ) # Set up a signal handler for graceful shutdown on Ctrl+C loop = asyncio.get_running_loop() for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler( sig, lambda s=sig: asyncio.create_task(crawler.shutdown()) ) await crawler.run() if __name__ == "__main__": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Main loop interrupted. Exiting.")