Add optional callback to ChunkedInsert

This commit adds an optional `callback` argument to the `ChunkedInsert`
object. This callback is a callable object which gets called before the
chunked insert happens. This is useful for clearing any local caches
that may be in place to deal with the eventual consistency resulting
from the delayed nature of the chunked inserts.

For example,

```
cache = set()
chunked_table = ChunkedInsert(table, callback=lambda queue: cache.clear())
while True:
    data = get_data_id()
    key = data['key']
    if key in cache or table.find_one(key=key)
        continue
    cache.add(key)
    chunked_table.insert(data)
```
This commit is contained in:
Micha Gorelick
2020-03-13 14:29:03 +01:00
parent 73ff374513
commit a9a15966e2
2 changed files with 34 additions and 3 deletions
+20 -1
View File
@@ -9,7 +9,7 @@ from datetime import datetime
from sqlalchemy import FLOAT, TEXT, BIGINT
from sqlalchemy.exc import IntegrityError, SQLAlchemyError, ArgumentError
from dataset import connect
from dataset import connect, chunked
from .sample_data import TEST_DATA, TEST_CITY_1
@@ -394,6 +394,25 @@ class TableTestCase(unittest.TestCase):
self.tbl.insert_many(data, chunk_size=13)
assert len(self.tbl) == len(data) + 6
def test_chunked_insert(self):
data = TEST_DATA * 100
with chunked.ChunkedInsert(self.tbl) as chunk_tbl:
for item in data:
chunk_tbl.insert(item)
assert len(self.tbl) == len(data) + 6
def test_chunked_insert_callback(self):
data = TEST_DATA * 100
N = 0
def callback(queue):
nonlocal N
N += len(queue)
with chunked.ChunkedInsert(self.tbl, callback=callback) as chunk_tbl:
for item in data:
chunk_tbl.insert(item)
assert len(data) == N
assert len(self.tbl) == len(data) + 6
def test_update_many(self):
tbl = self.db['update_many_test']
tbl.insert_many([