Add helper class for chunked inserts.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
|
||||
|
||||
class ChunkedInsert(object):
|
||||
"""Batch up insert operations
|
||||
with ChunkedStorer(my_table) as storer:
|
||||
table.insert(row)
|
||||
|
||||
Rows will be inserted in groups of 1000
|
||||
"""
|
||||
|
||||
def __init__(self, table, chunksize=1000):
|
||||
self.queue = []
|
||||
self.table = table
|
||||
self.chunksize = chunksize
|
||||
|
||||
def insert(self, item):
|
||||
self.queue.append(item)
|
||||
if len(self.queue) > self.chunksize:
|
||||
self.table.insert_many(self.queue)
|
||||
self.queue = []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.table.insert_many(self.queue)
|
||||
self.queue = []
|
||||
@@ -3,9 +3,8 @@ from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name='dataset',
|
||||
version='1.0.6',
|
||||
version='1.0.7',
|
||||
description="Toolkit for Python-based database access.",
|
||||
long_description="",
|
||||
classifiers=[
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
|
||||
Reference in New Issue
Block a user