|
# main.py
|
|
import asyncio
|
|
import logging
|
|
import signal
|
|
|
|
from crawler import DevRantCrawler
|
|
from database import DatabaseManager
|
|
|
|
from devranta.api import Api
|
|
|
|
# --- Configuration ---
|
|
DB_FILE = "devrant.sqlite"
|
|
CONCURRENT_RANT_CONSUMERS = 10 # How many rants to process at once
|
|
CONCURRENT_USER_CONSUMERS = 5 # How many user profiles to fetch at once
|
|
|
|
|
|
async def main():
|
|
"""Initializes and runs the crawler."""
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] - %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
|
|
api = Api()
|
|
|
|
async with DatabaseManager(DB_FILE) as db:
|
|
crawler = DevRantCrawler(
|
|
api=api,
|
|
db=db,
|
|
rant_consumers=CONCURRENT_RANT_CONSUMERS,
|
|
user_consumers=CONCURRENT_USER_CONSUMERS,
|
|
)
|
|
|
|
# Set up a signal handler for graceful shutdown on Ctrl+C
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
loop.add_signal_handler(
|
|
sig, lambda s=sig: asyncio.create_task(crawler.shutdown())
|
|
)
|
|
|
|
await crawler.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
asyncio.run(main())
|
|
except KeyboardInterrupt:
|
|
logging.info("Main loop interrupted. Exiting.")
|