Merge branch 'master' into fix-table-cache

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.egg-info
 dist/*
+build/*
 .DS_Store
 .watchr
 

.travis.yml

@@ -0,0 +1,11 @@
+---
+language: python
+python:
+- '3.3'
+- '2.7'
+- '2.6'
+install:
+- pip install flake8
+script:
+- flake8 --ignore=E501,E123,E124,E126,E127,E128 dataset test
+- python setup.py test

dataset/__init__.py

@@ -8,7 +8,6 @@ from dataset.persistence.util import sqlite_datetime_fix
 from dataset.persistence.database import Database
 from dataset.persistence.table import Table
 from dataset.freeze.app import freeze
-from sqlalchemy import Integer, Text
 
 __all__ = ['Database', 'Table', 'freeze', 'connect']
 

dataset/freeze/config.py

@@ -86,4 +86,3 @@ class Export(object):
     @property
     def name(self):
         return self.get('name', self.get('query'))
-

dataset/freeze/format/common.py

@@ -1,5 +1,4 @@
 import os
-import logging
 import re
 import locale
 
@@ -69,8 +68,7 @@ class Serializer(object):
 
     @property
     def wrap(self):
-        return self.export.get_bool('wrap',
+        return self.export.get_bool('wrap', default=self.mode == 'list')
-                default=self.mode=='list')
 
     def serialize(self):
         self.init()

dataset/freeze/format/ftabson.py

@@ -21,4 +21,3 @@ class TabsonSerializer(JSONSerializer):
         if meta is not None:
             result['meta'] = meta
         return result
-

dataset/persistence/database.py

@@ -11,9 +11,9 @@ except ImportError:
 
 from sqlalchemy import create_engine
 from sqlalchemy.pool import NullPool
-from sqlalchemy.schema import MetaData, Column, Index
+from sqlalchemy.schema import MetaData, Column
 from sqlalchemy.schema import Table as SQLATable
-from sqlalchemy import Integer, Text, String
+from sqlalchemy import Integer, String
 
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
@@ -107,10 +107,8 @@ class Database(object):
 
     @property
     def tables(self):
-        """ Get a listing of all tables that exist in the database.
+        """
-
+        Get a listing of all tables that exist in the database.
-        >>> print db.tables
-        set([u'user', u'action'])
         """
         return list(
             set(self.metadata.tables.keys()) | set(self._tables.keys())

dataset/persistence/table.py

@@ -1,9 +1,5 @@
 import logging
 from itertools import count
-try:
-    from collections import OrderedDict
-except ImportError:
-    from ordereddict import OrderedDict  # Python < 2.7 drop-in
 
 from sqlalchemy.sql import and_, expression
 from sqlalchemy.schema import Column, Index
@@ -28,9 +24,6 @@ class Table(object):
     def columns(self):
         """
         Get a listing of all columns that exist in the table.
-
-        >>> print 'age' in table.columns
-        True
         """
         return set(self.table.columns.keys())
 
@@ -105,7 +98,6 @@ class Table(object):
         if chunk:
             _process_chunk(chunk)
 
-
     def update(self, row, keys, ensure=True, types={}):
         """
         Update a row in the table. The update is managed via
@@ -126,7 +118,7 @@ class Table(object):
         if not isinstance(keys, (list, tuple)):
             keys = [keys]
         self._check_dropped()
-        if not keys or len(keys)==len(row):
+        if not keys or len(keys) == len(row):
             return False
         clause = [(u, row.get(u)) for u in keys]
 
@@ -288,7 +280,7 @@ class Table(object):
         rp = self.database.executable.execute(query)
         data = rp.fetchone()
         if data is not None:
-            return OrderedDict(zip(rp.keys(), data))
+            return data
 
     def _args_to_order_by(self, order_by):
         if order_by[0] == '-':
@@ -337,6 +329,9 @@ class Table(object):
         rp = self.database.executable.execute(count_query)
         total_row_count = rp.fetchone()[0]
 
+        if _limit is None:
+            _limit = total_row_count
+
         if _step is None or _step is False or _step == 0:
             _step = total_row_count
 
@@ -348,9 +343,7 @@ class Table(object):
 
         for i in count():
             qoffset = _offset + (_step * i)
-            qlimit = _step
+            qlimit = min(_limit - (_step * i), _step)
-            if _limit is not None:
-                qlimit = min(_limit - (_step * i), _step)
             if qlimit <= 0:
                 break
             queries.append(self.table.select(whereclause=args, limit=qlimit,

dataset/persistence/util.py

@@ -1,9 +1,5 @@
 from datetime import datetime, timedelta
 from inspect import isgenerator
-try:
-    from collections import OrderedDict
-except ImportError:
-    from ordereddict import OrderedDict  # Python < 2.7 drop-in
 
 from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event
 
@@ -50,7 +46,7 @@ class ResultIter(object):
             else:
                 # stop here
                 raise StopIteration
-        return OrderedDict(zip(self.keys, row))
+        return row
 
     next = __next__
 
@@ -64,6 +60,8 @@ def sqlite_datetime_fix():
         epoch = datetime(1970, 1, 1, 0, 0, 0)
 
         def process_bind_param(self, value, dialect):
+            if isinstance(value, datetime):
+                return value
             return (value / 1000 - self.epoch).total_seconds()
 
         def process_result_value(self, value, dialect):

dataset/util.py

@@ -1,12 +1,12 @@
 #coding: utf-8
 import re
-from unicodedata import normalize as ucnorm, category
 
 SLUG_REMOVE = re.compile(r'[,\s\.\(\)/\\;:]*')
 
+
 class DatasetException(Exception):
     pass
 
+
 class FreezeException(DatasetException):
     pass
-

setup.py

@@ -25,17 +25,18 @@ setup(
     author_email='info@okfn.org',
     url='http://github.com/pudo/dataset',
     license='MIT',
-    packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+    packages=find_packages(exclude=['ez_setup', 'examples', 'test']),
     namespace_packages=[],
     include_package_data=False,
     zip_safe=False,
     install_requires=[
-        'sqlalchemy >= 0.8.1',
+        'sqlalchemy >= 0.9.1',
-        'alembic >= 0.6.1',
+        'alembic >= 0.6.2',
         'python-slugify >= 0.0.6',
         "PyYAML >= 3.10"
     ] + py26_dependency,
     tests_require=[],
+    test_suite='test',
     entry_points={
         'console_scripts': [
             'datafreeze = dataset.freeze.app:main',

test/test_persistence.py

@@ -2,15 +2,18 @@ import os
 import unittest
 from datetime import datetime
 
+from sqlalchemy.exc import IntegrityError
+
 from dataset import connect
 from dataset.util import DatasetException
+
 from .sample_data import TEST_DATA, TEST_CITY_1
-from sqlalchemy.exc import IntegrityError
 
 
 class DatabaseTestCase(unittest.TestCase):
 
     def setUp(self):
+        self.old_db_url = os.environ.get('DATABASE_URL')
         os.environ['DATABASE_URL'] = 'sqlite:///:memory:'
         self.db = connect('sqlite:///:memory:')
         self.tbl = self.db['weather']
@@ -19,7 +22,10 @@ class DatabaseTestCase(unittest.TestCase):
 
     def tearDown(self):
         # ensure env variable was unset
-        del os.environ['DATABASE_URL']
+        if self.old_db_url is None:
+            del os.environ['DATABASE_URL']
+        else:
+            os.environ['DATABASE_URL'] = self.old_db_url
 
     def test_valid_database_url(self):
         assert self.db.url, os.environ['DATABASE_URL']
@@ -46,7 +52,7 @@ class DatabaseTestCase(unittest.TestCase):
 
         table.insert({
             'string_id': 'foobar'})
-        assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar'
+        assert table.find_one(string_id='foobar')['string_id'] == 'foobar'
 
     def test_create_table_custom_id2(self):
         pid = "string_id"
@@ -57,19 +63,19 @@ class DatabaseTestCase(unittest.TestCase):
 
         table.insert({
             'string_id': 'foobar'})
-        assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar'
+        assert table.find_one(string_id='foobar')['string_id'] == 'foobar'
 
     def test_create_table_custom_id3(self):
         pid = "int_id"
-        table = self.db.create_table("foo4", primary_id = pid)
+        table = self.db.create_table("foo4", primary_id=pid)
         assert table.table.exists()
         assert len(table.table.columns) == 1, table.table.columns
         assert pid in table.table.c, table.table.c
 
         table.insert({'int_id': 123})
         table.insert({'int_id': 124})
-        assert table.find_one(int_id = 123)['int_id'] == 123
+        assert table.find_one(int_id=123)['int_id'] == 123
-        assert table.find_one(int_id = 124)['int_id'] == 124
+        assert table.find_one(int_id=124)['int_id'] == 124
         self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123}))
 
     def test_create_table_shorthand1(self):
@@ -81,8 +87,8 @@ class DatabaseTestCase(unittest.TestCase):
 
         table.insert({'int_id': 123})
         table.insert({'int_id': 124})
-        assert table.find_one(int_id = 123)['int_id'] == 123
+        assert table.find_one(int_id=123)['int_id'] == 123
-        assert table.find_one(int_id = 124)['int_id'] == 124
+        assert table.find_one(int_id=124)['int_id'] == 124
         self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123}))
 
     def test_create_table_shorthand2(self):
@@ -94,7 +100,7 @@ class DatabaseTestCase(unittest.TestCase):
 
         table.insert({
             'string_id': 'foobar'})
-        assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar'
+        assert table.find_one(string_id='foobar')['string_id'] == 'foobar'
 
     def test_create_table_shorthand3(self):
         pid = "string_id"
@@ -105,7 +111,7 @@ class DatabaseTestCase(unittest.TestCase):
 
         table.insert({
             'string_id': 'foobar'})
-        assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar'
+        assert table.find_one(string_id='foobar')['string_id'] == 'foobar'
 
     def test_load_table(self):
         tbl = self.db.load_table('weather')
@@ -131,7 +137,7 @@ class TableTestCase(unittest.TestCase):
             'temperature': -10,
             'place': 'Berlin'}
         )
-        assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl)
+        assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl)
         assert self.tbl.find_one(id=last_id)['place'] == 'Berlin'
 
     def test_upsert(self):
@@ -141,17 +147,17 @@ class TableTestCase(unittest.TestCase):
             'place': 'Berlin'},
             ['place']
         )
-        assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl)
+        assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl)
         self.tbl.upsert({
             'date': datetime(2011, 1, 2),
             'temperature': -10,
             'place': 'Berlin'},
             ['place']
         )
-        assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl)
+        assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl)
 
     def test_upsert_all_key(self):
-        for i in range(0,2):
+        for i in range(0, 2):
             self.tbl.upsert({
                 'date': datetime(2011, 1, 2),
                 'temperature': -10,
@@ -165,7 +171,7 @@ class TableTestCase(unittest.TestCase):
             'temperature': -10,
             'place': 'Berlin'}
         )
-        assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl)
+        assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl)
         self.tbl.delete(place='Berlin')
         assert len(self.tbl) == len(TEST_DATA), len(self.tbl)
         self.tbl.delete()
@@ -187,6 +193,10 @@ class TableTestCase(unittest.TestCase):
         assert len(ds) == 3, ds
         ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2))
         assert len(ds) == 2, ds
+        ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2, _step=1))
+        assert len(ds) == 2, ds
+        ds = list(self.tbl.find(place=TEST_CITY_1, _limit=1, _step=2))
+        assert len(ds) == 1, ds
 
     def test_distinct(self):
         x = list(self.tbl.distinct('place'))
@@ -195,8 +205,8 @@ class TableTestCase(unittest.TestCase):
         assert len(x) == 6, x
 
     def test_insert_many(self):
-        data = TEST_DATA * 5000
+        data = TEST_DATA * 100
-        self.tbl.insert_many(data)
+        self.tbl.insert_many(data, chunk_size=13)
         assert len(self.tbl) == len(data) + 6
 
     def test_drop_warning(self):
@@ -239,7 +249,7 @@ class TableTestCase(unittest.TestCase):
         tbl = self.tbl
         tbl.create_column('foo', FLOAT)
         assert 'foo' in tbl.table.c, tbl.table.c
-        assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type
+        assert isinstance(tbl.table.c['foo'].type, FLOAT), tbl.table.c['foo'].type
         assert 'foo' in tbl.columns, tbl.columns
 
     def test_key_order(self):
@@ -247,6 +257,3 @@ class TableTestCase(unittest.TestCase):
         keys = list(res.next().keys())
         assert keys[0] == 'temperature'
         assert keys[1] == 'place'
-
-if __name__ == '__main__':
-    unittest.main()