diff --git a/CHANGELOG.md b/CHANGELOG.md index dfecd19..47befe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ *The changelog has only been started with version 0.3.12, previous changes must be reconstructed from revision history.* +* 0.4: Python 3 support and switch to alembic for migrations. +* 0.3.15: Fixes to update and insertion of data, thanks to @cli248 + and @abhinav-upadhyay. * 0.3.14: dataset went viral somehow. Thanks to @gtsafas for refactorings, @alasdairnicol for fixing the Freezfile example in the documentation. @diegoguimaraes fixed the behaviour of insert to diff --git a/dataset/__init__.py b/dataset/__init__.py index d9d4f96..3785251 100644 --- a/dataset/__init__.py +++ b/dataset/__init__.py @@ -4,6 +4,7 @@ import warnings warnings.filterwarnings( 'ignore', 'Unicode type received non-unicode bind param value.') +from dataset.persistence.util import sqlite_datetime_fix from dataset.persistence.database import Database from dataset.persistence.table import Table from dataset.freeze.app import freeze @@ -27,4 +28,8 @@ def connect(url=None, schema=None, reflectMetadata=True): """ if url is None: url = os.environ.get('DATABASE_URL', url) + + if url.startswith("sqlite://"): + sqlite_datetime_fix() + return Database(url, schema=schema, reflectMetadata=reflectMetadata) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 055d055..9b69d76 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -1,9 +1,13 @@ import logging from itertools import count +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy.sql import and_, expression from sqlalchemy.schema import Column, Index - +from sqlalchemy import alias from dataset.persistence.util import guess_type from dataset.persistence.util import ResultIter from dataset.util import DatasetException @@ -68,7 +72,8 @@ class Table(object): if ensure: self._ensure_columns(row, types=types) res = self.database.executable.execute(self.table.insert(row)) - return res.inserted_primary_key[0] + if len(res.inserted_primary_key) > 0: + return res.inserted_primary_key[0] def insert_many(self, rows, chunk_size=1000, ensure=True, types={}): """ @@ -283,7 +288,7 @@ class Table(object): rp = self.database.executable.execute(query) data = rp.fetchone() if data is not None: - return dict(zip(rp.keys(), data)) + return OrderedDict(zip(rp.keys(), data)) def _args_to_order_by(self, order_by): if order_by[0] == '-': @@ -328,7 +333,7 @@ class Table(object): args = self._args_to_clause(_filter) # query total number of rows first - count_query = self.table.count(whereclause=args, limit=_limit, offset=_offset) + count_query = alias(self.table.select(whereclause=args, limit=_limit, offset=_offset), name='count_query_alias').count() rp = self.database.executable.execute(count_query) total_row_count = rp.fetchone()[0] @@ -348,8 +353,6 @@ class Table(object): qlimit = min(_limit - (_step * i), _step) if qlimit <= 0: break - if qoffset > total_row_count: - break queries.append(self.table.select(whereclause=args, limit=qlimit, offset=qoffset, order_by=order_by)) return ResultIter((self.database.executable.execute(q) for q in queries)) diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index f41ca0b..2395797 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -1,7 +1,11 @@ -from datetime import datetime +from datetime import datetime, timedelta from inspect import isgenerator +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict # Python < 2.7 drop-in -from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean +from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event def guess_type(sample): @@ -46,9 +50,32 @@ class ResultIter(object): else: # stop here raise StopIteration - return dict(zip(self.keys, row)) + return OrderedDict(zip(self.keys, row)) next = __next__ def __iter__(self): return self + + +def sqlite_datetime_fix(): + class SQLiteDateTimeType(types.TypeDecorator): + impl = types.Integer + epoch = datetime(1970, 1, 1, 0, 0, 0) + + def process_bind_param(self, value, dialect): + return (value / 1000 - self.epoch).total_seconds() + + def process_result_value(self, value, dialect): + return self.epoch + timedelta(seconds=value / 1000) + + def is_sqlite(inspector): + return inspector.engine.dialect.name == "sqlite" + + def is_datetime(column_info): + return isinstance(column_info['type'], types.DateTime) + + @event.listens_for(Table, "column_reflect") + def setup_epoch(inspector, table, column_info): + if is_sqlite(inspector) and is_datetime(column_info): + column_info['type'] = SQLiteDateTimeType() diff --git a/setup.py b/setup.py index 73b4a63..29a0f4c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ if sys.version_info <= (2, 6): setup( name='dataset', - version='0.3.14', + version='0.4.0', description="Toolkit for Python-based data processing.", long_description="", classifiers=[ diff --git a/test/test_persistence.py b/test/test_persistence.py index 8f8a4ec..1207fb2 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -8,6 +8,7 @@ from dataset import connect from dataset.util import DatasetException from .sample_data import TEST_DATA, TEST_CITY_1 +from sqlalchemy.exc import IntegrityError class DatabaseTestCase(unittest.TestCase): @@ -247,3 +248,9 @@ class TableTestCase(unittest.TestCase): assert 'foo' in tbl.table.c, tbl.table.c assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type assert 'foo' in tbl.columns, tbl.columns + + def test_key_order(self): + res = self.db.query('SELECT temperature, place FROM weather LIMIT 1') + keys = list(res.next().keys()) + assert keys[0] == 'temperature' + assert keys[1] == 'place'