|
# retoor <retoor@molodetz.nl>
|
|
import asyncio
|
|
import logging
|
|
import signal
|
|
|
|
from crawler import DevRantCrawler
|
|
from database import DatabaseManager
|
|
from devranta.api import Api
|
|
|
|
|
|
DB_FILE = "devrant.sqlite"
|
|
CONCURRENT_RANT_CONSUMERS = 10
|
|
CONCURRENT_USER_CONSUMERS = 5
|
|
BATCH_SIZE = 100
|
|
FLUSH_INTERVAL = 5.0
|
|
|
|
|
|
async def main():
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] - %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
|
|
async with Api() as api:
|
|
async with DatabaseManager(
|
|
DB_FILE,
|
|
batch_size=BATCH_SIZE,
|
|
flush_interval=FLUSH_INTERVAL,
|
|
) as db:
|
|
crawler = DevRantCrawler(
|
|
api=api,
|
|
db=db,
|
|
rant_consumers=CONCURRENT_RANT_CONSUMERS,
|
|
user_consumers=CONCURRENT_USER_CONSUMERS,
|
|
)
|
|
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
loop.add_signal_handler(
|
|
sig, lambda s=sig: asyncio.create_task(crawler.shutdown())
|
|
)
|
|
|
|
await crawler.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
asyncio.run(main())
|
|
except KeyboardInterrupt:
|
|
logging.info("Main loop interrupted. Exiting.")
|