diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index c3565a9..f483b93 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -61,6 +61,34 @@ class Table(object): self._ensure_columns(row, types=types) self.database.engine.execute(self.table.insert(row)) + def insert_many(self, rows, ensure=True, types={}): + """ + Add many rows at a time, which is significantly faster than adding + them one by one. The rows are automatically processed in chunks of + 1000 per commit. + :: + + rows = [dict(name='Dolly')] * 10000 + table.insert_many(rows) + """ + def _process_chunk(chunk): + if ensure: + for row in chunk: + self._ensure_columns(row, types=types) + self.table.insert().execute(chunk) + chunk_size = 1000 + chunk = [] + i = 0 + for row in rows: + chunk.append(row) + i += 1 + if i == chunk_size: + _process_chunk(chunk) + chunk = [] + i = 0 + if i > 0: + _process_chunk(chunk) + def update(self, row, keys, ensure=True, types={}): """ Update a row in the table. The update is managed via diff --git a/docs/api.rst b/docs/api.rst index 5df5ce9..c5e6226 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -16,5 +16,5 @@ Table ----- .. autoclass:: dataset.Table - :members: columns, drop, insert, update, upsert, find, find_one, distinct, create_column, create_index, all + :members: columns, drop, insert, update, upsert, find, find_one, distinct, create_column, create_index, all, insert_many :special-members: __len__, __iter__ diff --git a/test/test_persistence.py b/test/test_persistence.py index 00cb053..f735369 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -96,6 +96,11 @@ class TableTestCase(unittest.TestCase): x = list(self.tbl.distinct('place', 'date')) assert len(x)==6, x + def test_insert_many(self): + data = TEST_DATA * 5000 + self.tbl.insert_many(data) + assert len(self.tbl) == len(data) + 6 + if __name__ == '__main__':