Add optional callback to ChunkedInsert
This commit adds an optional `callback` argument to the `ChunkedInsert`
object. This callback is a callable object which gets called before the
chunked insert happens. This is useful for clearing any local caches
that may be in place to deal with the eventual consistency resulting
from the delayed nature of the chunked inserts.
For example,
```
cache = set()
chunked_table = ChunkedInsert(table, callback=lambda queue: cache.clear())
while True:
data = get_data_id()
key = data['key']
if key in cache or table.find_one(key=key)
continue
cache.add(key)
chunked_table.insert(data)
```
This commit is contained in:
+20
-1
@@ -9,7 +9,7 @@ from datetime import datetime
|
||||
from sqlalchemy import FLOAT, TEXT, BIGINT
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError, ArgumentError
|
||||
|
||||
from dataset import connect
|
||||
from dataset import connect, chunked
|
||||
|
||||
from .sample_data import TEST_DATA, TEST_CITY_1
|
||||
|
||||
@@ -394,6 +394,25 @@ class TableTestCase(unittest.TestCase):
|
||||
self.tbl.insert_many(data, chunk_size=13)
|
||||
assert len(self.tbl) == len(data) + 6
|
||||
|
||||
def test_chunked_insert(self):
|
||||
data = TEST_DATA * 100
|
||||
with chunked.ChunkedInsert(self.tbl) as chunk_tbl:
|
||||
for item in data:
|
||||
chunk_tbl.insert(item)
|
||||
assert len(self.tbl) == len(data) + 6
|
||||
|
||||
def test_chunked_insert_callback(self):
|
||||
data = TEST_DATA * 100
|
||||
N = 0
|
||||
def callback(queue):
|
||||
nonlocal N
|
||||
N += len(queue)
|
||||
with chunked.ChunkedInsert(self.tbl, callback=callback) as chunk_tbl:
|
||||
for item in data:
|
||||
chunk_tbl.insert(item)
|
||||
assert len(data) == N
|
||||
assert len(self.tbl) == len(data) + 6
|
||||
|
||||
def test_update_many(self):
|
||||
tbl = self.db['update_many_test']
|
||||
tbl.insert_many([
|
||||
|
||||
Reference in New Issue
Block a user