From 66fd7ce17378a0dac68417ad296d34f9548d58e0 Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Wed, 4 Apr 2018 10:43:19 +0200 Subject: [PATCH] make sure all records have the same fields. --- dataset/chunked.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dataset/chunked.py b/dataset/chunked.py index 094abdb..b6820ce 100644 --- a/dataset/chunked.py +++ b/dataset/chunked.py @@ -10,18 +10,25 @@ class ChunkedInsert(object): def __init__(self, table, chunksize=1000): self.queue = [] + self.fields = set() self.table = table self.chunksize = chunksize + def flush(self): + for item in self.queue: + for field in self.fields: + item[field] = item.get(field) + self.table.insert_many(self.queue) + self.queue = [] + def insert(self, item): + self.fields.update(item.keys()) self.queue.append(item) - if len(self.queue) > self.chunksize: - self.table.insert_many(self.queue) - self.queue = [] + if len(self.queue) >= self.chunksize: + self.flush() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.table.insert_many(self.queue) - self.queue = [] + self.flush()