From 07c290f6913ee045a321bd4dc22197b635feb452 Mon Sep 17 00:00:00 2001 From: 3onyc <3onyc@x3tech.com> Date: Thu, 21 Nov 2013 08:47:43 +0100 Subject: [PATCH 1/9] Implement SQLite datetime fix --- dataset/__init__.py | 5 +++++ dataset/persistence/util.py | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/dataset/__init__.py b/dataset/__init__.py index d9d4f96..3785251 100644 --- a/dataset/__init__.py +++ b/dataset/__init__.py @@ -4,6 +4,7 @@ import warnings warnings.filterwarnings( 'ignore', 'Unicode type received non-unicode bind param value.') +from dataset.persistence.util import sqlite_datetime_fix from dataset.persistence.database import Database from dataset.persistence.table import Table from dataset.freeze.app import freeze @@ -27,4 +28,8 @@ def connect(url=None, schema=None, reflectMetadata=True): """ if url is None: url = os.environ.get('DATABASE_URL', url) + + if url.startswith("sqlite://"): + sqlite_datetime_fix() + return Database(url, schema=schema, reflectMetadata=reflectMetadata) diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index 019f620..d22faf1 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -1,7 +1,7 @@ -from datetime import datetime +from datetime import datetime, timedelta from inspect import isgenerator -from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean +from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event def guess_type(sample): @@ -50,3 +50,20 @@ class ResultIter(object): def __iter__(self): return self + + +def sqlite_datetime_fix(): + class SQLiteDateTimeType(types.TypeDecorator): + impl = types.Integer + epoch = datetime(1970, 1, 1, 0, 0, 0) + + def process_bind_param(self, value, dialect): + return (value / 1000 - self.epoch).total_seconds() + + def process_result_value(self, value, dialect): + return self.epoch + timedelta(seconds=value / 1000) + + @event.listens_for(Table, "column_reflect") + def setup_epoch(inspector, table, column_info): + if isinstance(column_info['type'], types.DateTime): + column_info['type'] = SQLiteDateTimeType() From a75c1968b498beb672928638a452c7e884567630 Mon Sep 17 00:00:00 2001 From: 3onyc <3onyc@x3tech.com> Date: Tue, 10 Dec 2013 14:06:06 +0100 Subject: [PATCH 2/9] Only apply fix when the current engine has an SQLite dialect --- dataset/persistence/util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index d22faf1..a378f17 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -63,7 +63,13 @@ def sqlite_datetime_fix(): def process_result_value(self, value, dialect): return self.epoch + timedelta(seconds=value / 1000) + def is_sqlite(inspector): + return inspector.engine.dialect.name == "sqlite" + + def is_datetime(column_info): + return isinstance(column_info['type'], types.DateTime) + @event.listens_for(Table, "column_reflect") def setup_epoch(inspector, table, column_info): - if isinstance(column_info['type'], types.DateTime): + if is_sqlite(inspector) and is_datetime(column_info): column_info['type'] = SQLiteDateTimeType() From 03cca31b5bf0a61c024da3b6cc97d19795d77a01 Mon Sep 17 00:00:00 2001 From: Gregor Aisch Date: Thu, 19 Dec 2013 23:46:10 +0100 Subject: [PATCH 3/9] added test for ordered keys --- test/test_persistence.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_persistence.py b/test/test_persistence.py index 015f1fc..b8dc7e1 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -244,5 +244,11 @@ class TableTestCase(unittest.TestCase): assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type assert 'foo' in tbl.columns, tbl.columns + def test_key_order(self): + res = self.db.query('SELECT temperature, place FROM weather LIMIT 1') + keys = res.next().keys() + assert keys[0] == 'temperature' + assert keys[1] == 'place' + if __name__ == '__main__': unittest.main() From 2566581442eff9b17ee439296e7c4e9c232cee26 Mon Sep 17 00:00:00 2001 From: Gregor Aisch Date: Fri, 20 Dec 2013 00:01:26 +0100 Subject: [PATCH 4/9] using OrderedDict to preserve key order --- dataset/persistence/table.py | 6 +++++- dataset/persistence/util.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index c76b1fd..2296d59 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -1,5 +1,9 @@ import logging from itertools import count +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy.sql import and_, expression from sqlalchemy.schema import Column, Index @@ -280,7 +284,7 @@ class Table(object): rp = self.database.executable.execute(query) data = rp.fetchone() if data is not None: - return dict(zip(rp.keys(), data)) + return OrderedDict(zip(rp.keys(), data)) def _args_to_order_by(self, order_by): if order_by[0] == '-': diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index 019f620..e0f358f 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -1,5 +1,9 @@ from datetime import datetime from inspect import isgenerator +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean @@ -46,7 +50,7 @@ class ResultIter(object): else: # stop here raise StopIteration - return dict(zip(self.keys, row)) + return OrderedDict(zip(self.keys, row)) def __iter__(self): return self From 508e0256893d838e6eac7144b36277980de67056 Mon Sep 17 00:00:00 2001 From: Abhinav Upadhyay Date: Sat, 4 Jan 2014 11:53:15 +0530 Subject: [PATCH 5/9] Fix insert for tables that do not have any primary key. The insert method returns the value of the primary key for the inserted row. But for tables that do not have any primary key, it raises an IndexError, which is clearly wrong. At best it should return None in such cases and avoid raising any exception. --- dataset/persistence/table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index c76b1fd..6737d5b 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -68,7 +68,8 @@ class Table(object): if ensure: self._ensure_columns(row, types=types) res = self.database.executable.execute(self.table.insert(row)) - return res.inserted_primary_key[0] + if len(res.inserted_primary_key) > 0: + return res.inserted_primary_key[0] def insert_many(self, rows, chunk_size=1000, ensure=True, types={}): """ From 1fe1b163f6987d33d51b010e758df967b011b090 Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Sun, 5 Jan 2014 15:55:37 +0100 Subject: [PATCH 6/9] Release 0.3.15. --- CHANGELOG.md | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfecd19..fda69ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ *The changelog has only been started with version 0.3.12, previous changes must be reconstructed from revision history.* +* 0.3.15: Fixes to update and insertion of data, thanks to @cli248 + and @abhinav-upadhyay. * 0.3.14: dataset went viral somehow. Thanks to @gtsafas for refactorings, @alasdairnicol for fixing the Freezfile example in the documentation. @diegoguimaraes fixed the behaviour of insert to diff --git a/setup.py b/setup.py index 490a922..d55098e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='dataset', - version='0.3.14', + version='0.3.15', description="Toolkit for Python-based data processing.", long_description="", classifiers=[ From b43129218833633d45cdf6638b90cf1a7bf32762 Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Sun, 5 Jan 2014 18:36:11 +0100 Subject: [PATCH 7/9] fix test for ordereddicts --- test/test_persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_persistence.py b/test/test_persistence.py index 30f8c79..03a8b1c 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -4,7 +4,7 @@ from datetime import datetime from dataset import connect from dataset.util import DatasetException -from sample_data import TEST_DATA, TEST_CITY_1 +from .sample_data import TEST_DATA, TEST_CITY_1 from sqlalchemy.exc import IntegrityError @@ -244,7 +244,7 @@ class TableTestCase(unittest.TestCase): def test_key_order(self): res = self.db.query('SELECT temperature, place FROM weather LIMIT 1') - keys = res.next().keys() + keys = list(res.next().keys()) assert keys[0] == 'temperature' assert keys[1] == 'place' From 06046aa16f7446768d62c81f554f5643d8c09b5a Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Sun, 5 Jan 2014 18:42:54 +0100 Subject: [PATCH 8/9] Update to version 0.4.0 --- CHANGELOG.md | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fda69ae..47befe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ *The changelog has only been started with version 0.3.12, previous changes must be reconstructed from revision history.* +* 0.4: Python 3 support and switch to alembic for migrations. * 0.3.15: Fixes to update and insertion of data, thanks to @cli248 and @abhinav-upadhyay. * 0.3.14: dataset went viral somehow. Thanks to @gtsafas for diff --git a/setup.py b/setup.py index 3f0d700..95ef190 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ if sys.version_info <= (2, 6): setup( name='dataset', - version='0.3.15', + version='0.4.0', description="Toolkit for Python-based data processing.", long_description="", classifiers=[ From 6d5e1c755fd86658a05dd784b8311784d58453aa Mon Sep 17 00:00:00 2001 From: Aditya Upadhyay Date: Sun, 12 Jan 2014 23:56:14 +0530 Subject: [PATCH 9/9] fixed bug with offsets and limits. --- dataset/persistence/table.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 8fb39ea..9b69d76 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -7,7 +7,7 @@ except ImportError: from sqlalchemy.sql import and_, expression from sqlalchemy.schema import Column, Index - +from sqlalchemy import alias from dataset.persistence.util import guess_type from dataset.persistence.util import ResultIter from dataset.util import DatasetException @@ -333,7 +333,7 @@ class Table(object): args = self._args_to_clause(_filter) # query total number of rows first - count_query = self.table.count(whereclause=args, limit=_limit, offset=_offset) + count_query = alias(self.table.select(whereclause=args, limit=_limit, offset=_offset), name='count_query_alias').count() rp = self.database.executable.execute(count_query) total_row_count = rp.fetchone()[0] @@ -353,8 +353,6 @@ class Table(object): qlimit = min(_limit - (_step * i), _step) if qlimit <= 0: break - if qoffset > total_row_count: - break queries.append(self.table.select(whereclause=args, limit=qlimit, offset=qoffset, order_by=order_by)) return ResultIter((self.database.executable.execute(q) for q in queries))