devranta/examples/crawler/main.py

# retoor <retoor@molodetz.nl>
import asyncio
import logging
import signal

from crawler import DevRantCrawler
from database import DatabaseManager
from devranta.api import Api


DB_FILE = "devrant.sqlite"
CONCURRENT_RANT_CONSUMERS = 10
CONCURRENT_USER_CONSUMERS = 5
BATCH_SIZE = 100
FLUSH_INTERVAL = 5.0


async def main():
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s [%(levelname)s] - %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    async with Api() as api:
        async with DatabaseManager(
            DB_FILE,
            batch_size=BATCH_SIZE,
            flush_interval=FLUSH_INTERVAL,
        ) as db:
            crawler = DevRantCrawler(
                api=api,
                db=db,
                rant_consumers=CONCURRENT_RANT_CONSUMERS,
                user_consumers=CONCURRENT_USER_CONSUMERS,
            )

            loop = asyncio.get_running_loop()
            for sig in (signal.SIGINT, signal.SIGTERM):
                loop.add_signal_handler(
                    sig, lambda s=sig: asyncio.create_task(crawler.shutdown())
                )

            await crawler.run()


if __name__ == "__main__":
    try:
        asyncio.run(main())
    except KeyboardInterrupt:
        logging.info("Main loop interrupted. Exiting.")