Add helper class for chunked inserts.
This commit is contained in:
parent
0d9b2877e2
commit
9d99ed3e29
27
dataset/chunked.py
Normal file
27
dataset/chunked.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
|
||||||
|
|
||||||
|
class ChunkedInsert(object):
|
||||||
|
"""Batch up insert operations
|
||||||
|
with ChunkedStorer(my_table) as storer:
|
||||||
|
table.insert(row)
|
||||||
|
|
||||||
|
Rows will be inserted in groups of 1000
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, table, chunksize=1000):
|
||||||
|
self.queue = []
|
||||||
|
self.table = table
|
||||||
|
self.chunksize = chunksize
|
||||||
|
|
||||||
|
def insert(self, item):
|
||||||
|
self.queue.append(item)
|
||||||
|
if len(self.queue) > self.chunksize:
|
||||||
|
self.table.insert_many(self.queue)
|
||||||
|
self.queue = []
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
self.table.insert_many(self.queue)
|
||||||
|
self.queue = []
|
||||||
3
setup.py
3
setup.py
@ -3,9 +3,8 @@ from setuptools import setup, find_packages
|
|||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='dataset',
|
name='dataset',
|
||||||
version='1.0.6',
|
version='1.0.7',
|
||||||
description="Toolkit for Python-based database access.",
|
description="Toolkit for Python-based database access.",
|
||||||
long_description="",
|
|
||||||
classifiers=[
|
classifiers=[
|
||||||
"Development Status :: 3 - Alpha",
|
"Development Status :: 3 - Alpha",
|
||||||
"Intended Audience :: Developers",
|
"Intended Audience :: Developers",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user