Catch closed RP, fixes #334.

Browse Source
This commit is contained in:
Friedrich Lindenberg 2020-06-28 15:02:20 +02:00
parent 0c46d9eead
commit 2637254bcf
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dataset/util.py
View File

@ -2,6 +2,7 @@ from hashlib import sha1
from urllib.parse import urlparse from urllib.parse import urlparse
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Iterable from collections.abc import Iterable
from sqlalchemy.exc import ResourceClosedError
QUERY_STEP = 1000 QUERY_STEP = 1000
row_type = OrderedDict row_type = OrderedDict
@ -19,15 +20,18 @@ def convert_row(row_type, row):
def iter_result_proxy(rp, step=None): def iter_result_proxy(rp, step=None):
"""Iterate over the ResultProxy.""" """Iterate over the ResultProxy."""
while True: try:
if step is None: while True:
chunk = rp.fetchall() if step is None:
else: chunk = rp.fetchall()
chunk = rp.fetchmany(size=step) else:
if not chunk: chunk = rp.fetchmany(size=step)
break if not chunk:
for row in chunk: break
yield row for row in chunk:
yield row
except ResourceClosedError:
return
class ResultIter(object): class ResultIter(object):