Add helper class for chunked inserts.

This commit is contained in:
Friedrich Lindenberg 2018-04-04 09:34:39 +02:00
parent 0d9b2877e2
commit 9d99ed3e29
2 changed files with 28 additions and 2 deletions

27
dataset/chunked.py Normal file
View File

@ -0,0 +1,27 @@
class ChunkedInsert(object):
"""Batch up insert operations
with ChunkedStorer(my_table) as storer:
table.insert(row)
Rows will be inserted in groups of 1000
"""
def __init__(self, table, chunksize=1000):
self.queue = []
self.table = table
self.chunksize = chunksize
def insert(self, item):
self.queue.append(item)
if len(self.queue) > self.chunksize:
self.table.insert_many(self.queue)
self.queue = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.table.insert_many(self.queue)
self.queue = []

View File

@ -3,9 +3,8 @@ from setuptools import setup, find_packages
setup( setup(
name='dataset', name='dataset',
version='1.0.6', version='1.0.7',
description="Toolkit for Python-based database access.", description="Toolkit for Python-based database access.",
long_description="",
classifiers=[ classifiers=[
"Development Status :: 3 - Alpha", "Development Status :: 3 - Alpha",
"Intended Audience :: Developers", "Intended Audience :: Developers",