From c1e82bb9aeaebb4424849db2a87ebbed5ab811d0 Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Fri, 22 Apr 2016 11:35:14 -0400 Subject: [PATCH 1/7] Added insert_ignore and changed upsert. --- dataset/persistence/table.py | 74 +++++++++++++++++++++++++----------- docs/api.rst | 2 +- test/test_persistence.py | 25 ++++++++++++ 3 files changed, 77 insertions(+), 24 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 3eaf808..6ed57a5 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -79,6 +79,31 @@ class Table(object): if len(res.inserted_primary_key) > 0: return res.inserted_primary_key[0] + def insert_ignore(self, row, keys, ensure=None, types={}): + """ + Add a row (type: dict) into the table if the row does not exist. + + If rows with matching ``keys`` exist they will be added to the table. + + Setting ``ensure`` results in automatically creating missing columns, + i.e., keys of the row are not table columns. + + During column creation, ``types`` will be checked for a key + matching the name of a column to be created, and the given + SQLAlchemy column type will be used. Otherwise, the type is + guessed from the row value, defaulting to a simple unicode + field. + :: + + data = dict(id=10, title='I am a banana!') + table.insert_ignore(data, ['id']) + """ + res = self._upsert_pre_check(row, keys, ensure) + if res is None: + return self.insert(row, ensure=ensure, types=types) + else: + return False + def insert_many(self, rows, chunk_size=1000, ensure=None, types={}): """ Add many rows at a time. @@ -156,17 +181,7 @@ class Table(object): except KeyError: return 0 - def upsert(self, row, keys, ensure=None, types={}): - """ - An UPSERT is a smart combination of insert and update. - - If rows with matching ``keys`` exist they will be updated, otherwise a - new row is inserted in the table. - :: - - data = dict(id=10, title='I am a banana!') - table.upsert(data, ['id']) - """ + def _upsert_pre_check(self, row, keys, ensure): # check whether keys arg is a string and format as a list if not isinstance(keys, (list, tuple)): keys = [keys] @@ -181,19 +196,32 @@ class Table(object): filters[key] = row.get(key) res = self.find_one(**filters) - if res is not None: - row_count = self.update(row, keys, ensure=ensure, types=types) - if row_count == 0: - return False - elif row_count == 1: - try: - return res['id'] - except KeyError: - return True - else: - return True - else: + + return res + + def upsert(self, row, keys, ensure=None, types={}): + """ + An UPSERT is a smart combination of insert and update. + + If rows with matching ``keys`` exist they will be updated, otherwise a + new row is inserted in the table. + :: + + data = dict(id=10, title='I am a banana!') + table.upsert(data, ['id']) + """ + res = self._upsert_pre_check(row, keys, ensure) + if res is None: return self.insert(row, ensure=ensure, types=types) + else: + row_count = self.update(row, keys, ensure=ensure, types=types) + result = lambda row_count : row_count > 0 + if row_count == 1: + try: + result = res['id'] + except KeyError: + pass + return result def delete(self, *_clauses, **_filter): """ diff --git a/docs/api.rst b/docs/api.rst index 89d4416..5770e59 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -16,7 +16,7 @@ Table ----- .. autoclass:: dataset.Table - :members: columns, find, find_one, all, count, distinct, insert, insert_many, update, upsert, delete, create_column, drop_column, create_index, drop + :members: columns, find, find_one, all, count, distinct, insert, insert_ignore, insert_many, update, upsert, delete, create_column, drop_column, create_index, drop :special-members: __len__, __iter__ diff --git a/test/test_persistence.py b/test/test_persistence.py index db2836c..9dac6be 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -170,6 +170,31 @@ class TableTestCase(unittest.TestCase): assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) assert self.tbl.find_one(id=last_id)['place'] == 'Berlin' + def test_insert_ignore(self): + self.tbl.insert_ignore({ + 'date': datetime(2011, 1, 2), + 'temperature': -10, + 'place': 'Berlin'}, + ['place'] + ) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) + self.tbl.insert_ignore({ + 'date': datetime(2011, 1, 2), + 'temperature': -10, + 'place': 'Berlin'}, + ['place'] + ) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) + + def test_insert_ignore_all_key(self): + for i in range(0, 2): + self.tbl.insert_ignore({ + 'date': datetime(2011, 1, 2), + 'temperature': -10, + 'place': 'Berlin'}, + ['date', 'temperature', 'place'] + ) + def test_upsert(self): self.tbl.upsert({ 'date': datetime(2011, 1, 2), From ff9bed01f1fc83d7b5654533310fd44ddaf57e1f Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Fri, 22 Apr 2016 12:57:35 -0400 Subject: [PATCH 2/7] Fixed Build Jobs Build #181 errors --- dataset/persistence/table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 6ed57a5..d81c663 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -100,7 +100,7 @@ class Table(object): """ res = self._upsert_pre_check(row, keys, ensure) if res is None: - return self.insert(row, ensure=ensure, types=types) + return self.insert(row, ensure=ensure, types=types) else: return False @@ -215,7 +215,7 @@ class Table(object): return self.insert(row, ensure=ensure, types=types) else: row_count = self.update(row, keys, ensure=ensure, types=types) - result = lambda row_count : row_count > 0 + result = (True, False)[row_count > 0] if row_count == 1: try: result = res['id'] From 66547aafb5be229e857918fb01a254dbe82c41f3 Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Fri, 22 Apr 2016 13:04:17 -0400 Subject: [PATCH 3/7] Fixed Build Jobs Build #182 errors --- dataset/persistence/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index d81c663..b3bafc2 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -218,7 +218,7 @@ class Table(object): result = (True, False)[row_count > 0] if row_count == 1: try: - result = res['id'] + result = res['id'] except KeyError: pass return result From 9c162a056c9802b821792a2011501b3f949ded4e Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Fri, 22 Apr 2016 14:13:52 -0400 Subject: [PATCH 4/7] Changed upsert code style. The logic should be the same. --- dataset/persistence/table.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index b3bafc2..6c41357 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -215,12 +215,11 @@ class Table(object): return self.insert(row, ensure=ensure, types=types) else: row_count = self.update(row, keys, ensure=ensure, types=types) - result = (True, False)[row_count > 0] - if row_count == 1: - try: - result = res['id'] - except KeyError: - pass + try: + result = (res['id'], row_count > 0)[row_count == 1] + except KeyError: + result = row_count > 0 + return result def delete(self, *_clauses, **_filter): From 7fd1a71506b26c71df84d6c3b531edbd8cb42dbe Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Sat, 23 Apr 2016 07:59:25 -0400 Subject: [PATCH 5/7] Corrected upsert logical error. --- dataset/persistence/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 6c41357..1dd2e8c 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -216,7 +216,7 @@ class Table(object): else: row_count = self.update(row, keys, ensure=ensure, types=types) try: - result = (res['id'], row_count > 0)[row_count == 1] + result = (row_count > 0, res['id'])[row_count == 1] except KeyError: result = row_count > 0 From e7f45b2a688e7eb6e731e6d8b5d14d2dad7bf1f4 Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Wed, 27 Apr 2016 16:39:50 -0400 Subject: [PATCH 6/7] Added error check to _upsert_pre_check(), res = None when error --- dataset/persistence/table.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 1dd2e8c..a5c7ee3 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -183,20 +183,23 @@ class Table(object): def _upsert_pre_check(self, row, keys, ensure): # check whether keys arg is a string and format as a list - if not isinstance(keys, (list, tuple)): - keys = [keys] - self._check_dropped() + try: + if not isinstance(keys, (list, tuple)): + keys = [keys] + self._check_dropped() - ensure = self.database.ensure_schema if ensure is None else ensure - if ensure: - self.create_index(keys) + ensure = self.database.ensure_schema if ensure is None else ensure + if ensure: + self.create_index(keys) - filters = {} - for key in keys: - filters[key] = row.get(key) - - res = self.find_one(**filters) + filters = {} + for key in keys: + filters[key] = row.get(key) + res = self.find_one(**filters) + except: + res = None + return res def upsert(self, row, keys, ensure=None, types={}): From 72b01df60dd25e9d3c06eb36a7b615dbc67f411c Mon Sep 17 00:00:00 2001 From: Saun Shewanown Date: Wed, 27 Apr 2016 16:48:45 -0400 Subject: [PATCH 7/7] Fixed Build Jobs Build #186 blank space errors --- dataset/persistence/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index a5c7ee3..a43f0e2 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -199,7 +199,7 @@ class Table(object): res = self.find_one(**filters) except: res = None - + return res def upsert(self, row, keys, ensure=None, types={}):