chore: remove stale pycache files and add asyncio timeout to sync loop

Remove all compiled Python cache files from the drstats package directory to prevent stale bytecode issues. Refactor the sync_rants function by extracting the rant processing logic into a private _sync_rants helper and wrap it with asyncio.wait_for using a 5-second timeout to handle rate-limited API responses gracefully, breaking the loop on timeout exceptions.
This commit is contained in:
retoor 2024-11-25 19:27:39 +00:00
parent 2657c83053
commit 5025647f04
8 changed files with 16 additions and 8 deletions

View File

@ -40,21 +40,29 @@ async def get_recent_rants(start_from=1, page_size=10):
start_from += page_size
async def _sync_rants(start_from, page_size,count):
async for rant in get_recent_rants(start_from, page_size):
start_from += page_size
count += 1
rant["tags"] = json.dumps(rant["tags"])
db["rants"].upsert(rant, ["id"])
print(f"Upserted {count} rant(s).")
return count
async def sync_rants():
count = 0
start_from = 0
page_size = 20
try:
async for rant in get_recent_rants(start_from, page_size):
while True:
try:
count += await asyncio.wait_for(_sync_rants(start_from, page_size,count),5)
start_from += page_size
count += 1
rant["tags"] = json.dumps(rant["tags"])
db["rants"].upsert(rant, ["id"])
print(f"Upserted {count} rant(s).")
except:
print("Rate limit of server exceeded. That's normal.s")
except Exception as ex:
print(ex)
print("If exception described above is an timeout related error, it's due ratelimiting and considered OK.")
break
async def sync_comments():