From 2291953e5951ac6275f07c22224c972350ccf7c8 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Wed, 18 Dec 2013 04:20:13 +0100 Subject: [PATCH 01/13] Do not install test package into global namespace --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d59bea6..ad8c7da 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( author_email='info@okfn.org', url='http://github.com/pudo/dataset', license='MIT', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + packages=find_packages(exclude=['ez_setup', 'examples', 'test']), namespace_packages=[], include_package_data=False, zip_safe=False, From 425528dc17405d6da5df03e40944e1478defb6fa Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Wed, 18 Dec 2013 04:21:10 +0100 Subject: [PATCH 02/13] Set up test suite running through setup.py Run with python setup.py test --- setup.py | 1 + test/test_persistence.py | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index ad8c7da..73b4a63 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( "PyYAML >= 3.10" ] + py26_dependency, tests_require=[], + test_suite='test', entry_points={ 'console_scripts': [ 'datafreeze = dataset.freeze.app:main', diff --git a/test/test_persistence.py b/test/test_persistence.py index 1ae95fc..998b2e4 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -2,10 +2,12 @@ import os import unittest from datetime import datetime +from sqlalchemy.exc import IntegrityError + from dataset import connect from dataset.util import DatasetException -from sample_data import TEST_DATA, TEST_CITY_1 -from sqlalchemy.exc import IntegrityError + +from .sample_data import TEST_DATA, TEST_CITY_1 class DatabaseTestCase(unittest.TestCase): @@ -241,6 +243,3 @@ class TableTestCase(unittest.TestCase): assert 'foo' in tbl.table.c, tbl.table.c assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type assert 'foo' in tbl.columns, tbl.columns - -if __name__ == '__main__': - unittest.main() From 95717d4584e26027fb6c182ff82cb1321aee878d Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Wed, 18 Dec 2013 04:21:34 +0100 Subject: [PATCH 03/13] Add Travis CI tests --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..08d6c48 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +--- +language: python +python: +- '3.3' +- '2.7' +- '2.6' +script: +- python setup.py test From a6095a0506962ebd68f164bbef61220e01bc047e Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Wed, 18 Dec 2013 11:42:19 +0100 Subject: [PATCH 04/13] Reset DATABASE_URL OS environ value properly Let's be nice, who knows when this will save the day. --- test/test_persistence.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_persistence.py b/test/test_persistence.py index 998b2e4..3feee9d 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -13,6 +13,7 @@ from .sample_data import TEST_DATA, TEST_CITY_1 class DatabaseTestCase(unittest.TestCase): def setUp(self): + self.old_db_url = os.environ.get('DATABASE_URL') os.environ['DATABASE_URL'] = 'sqlite:///:memory:' self.db = connect('sqlite:///:memory:') self.tbl = self.db['weather'] @@ -21,7 +22,10 @@ class DatabaseTestCase(unittest.TestCase): def tearDown(self): # ensure env variable was unset - del os.environ['DATABASE_URL'] + if self.old_db_url is None: + del os.environ['DATABASE_URL'] + else: + os.environ['DATABASE_URL'] = self.old_db_url def test_valid_database_url(self): assert self.db.url, os.environ['DATABASE_URL'] From ea28ddec01c4fb73bf4a2ad027a2fd0ebd2136c5 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Wed, 18 Dec 2013 11:50:34 +0100 Subject: [PATCH 05/13] Make tests more than twice as fast --- test/test_persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_persistence.py b/test/test_persistence.py index 3feee9d..8f8a4ec 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -201,8 +201,8 @@ class TableTestCase(unittest.TestCase): assert len(x) == 6, x def test_insert_many(self): - data = TEST_DATA * 5000 - self.tbl.insert_many(data) + data = TEST_DATA * 100 + self.tbl.insert_many(data, chunk_size=13) assert len(self.tbl) == len(data) + 6 def test_drop_warning(self): From 5a7cafd4e24b1b822ddaccf27bfe192787c52383 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 21:06:38 +0100 Subject: [PATCH 06/13] Don't convert to datetime, if it is already datetime --- dataset/persistence/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index 2395797..3439c3f 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -64,6 +64,8 @@ def sqlite_datetime_fix(): epoch = datetime(1970, 1, 1, 0, 0, 0) def process_bind_param(self, value, dialect): + if isinstance(value, datetime): + return value return (value / 1000 - self.epoch).total_seconds() def process_result_value(self, value, dialect): From eff6925af91c03653d64ed7e75fdfa29c7b5e182 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 21:20:18 +0100 Subject: [PATCH 07/13] Fix infinite loop in find method If a _limit is not given, the query limit should be given by eit --- dataset/persistence/table.py | 7 ++++--- test/test_persistence.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 9b69d76..83cd4ee 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -337,6 +337,9 @@ class Table(object): rp = self.database.executable.execute(count_query) total_row_count = rp.fetchone()[0] + if _limit is None: + _limit = total_row_count + if _step is None or _step is False or _step == 0: _step = total_row_count @@ -348,9 +351,7 @@ class Table(object): for i in count(): qoffset = _offset + (_step * i) - qlimit = _step - if _limit is not None: - qlimit = min(_limit - (_step * i), _step) + qlimit = min(_limit - (_step * i), _step) if qlimit <= 0: break queries.append(self.table.select(whereclause=args, limit=qlimit, diff --git a/test/test_persistence.py b/test/test_persistence.py index 1207fb2..c8922a3 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -194,6 +194,10 @@ class TableTestCase(unittest.TestCase): assert len(ds) == 3, ds ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2)) assert len(ds) == 2, ds + ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2, _step=1)) + assert len(ds) == 2, ds + ds = list(self.tbl.find(place=TEST_CITY_1, _limit=1, _step=2)) + assert len(ds) == 1, ds def test_distinct(self): x = list(self.tbl.distinct('place')) From 0726dd9cf3ed917d2a142a2100ff39983c30fc5a Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 21:22:11 +0100 Subject: [PATCH 08/13] Remove OrderedDict implementation as RowProxy is returned http://docs.sqlalchemy.org/en/rel_0_9/core/connections.html?highlight=rowproxy#sqlalchemy.engine.RowProxy --- dataset/persistence/table.py | 6 +----- dataset/persistence/util.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 83cd4ee..29c45d1 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -1,9 +1,5 @@ import logging from itertools import count -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy.sql import and_, expression from sqlalchemy.schema import Column, Index @@ -288,7 +284,7 @@ class Table(object): rp = self.database.executable.execute(query) data = rp.fetchone() if data is not None: - return OrderedDict(zip(rp.keys(), data)) + return data def _args_to_order_by(self, order_by): if order_by[0] == '-': diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index 3439c3f..5197aaf 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -1,9 +1,5 @@ from datetime import datetime, timedelta from inspect import isgenerator -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event @@ -50,7 +46,7 @@ class ResultIter(object): else: # stop here raise StopIteration - return OrderedDict(zip(self.keys, row)) + return row next = __next__ From 6292bda8aaea57beeae2b4ccc420dc2f74752390 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 21:22:19 +0100 Subject: [PATCH 09/13] Upgrade dependencies --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 29a0f4c..49c9b9d 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ setup( include_package_data=False, zip_safe=False, install_requires=[ - 'sqlalchemy >= 0.8.1', - 'alembic >= 0.6.1', + 'sqlalchemy >= 0.9.1', + 'alembic >= 0.6.2', 'python-slugify >= 0.0.6', "PyYAML >= 3.10" ] + py26_dependency, From 92817d5f4de8d3839e3573ba2e23c7767809e7c8 Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 21:45:30 +0100 Subject: [PATCH 10/13] Make flake8 part of test suite --- .travis.yml | 1 + dataset/__init__.py | 1 - dataset/freeze/app.py | 4 ++-- dataset/freeze/config.py | 3 +-- dataset/freeze/format/common.py | 4 +--- dataset/freeze/format/ftabson.py | 1 - dataset/persistence/database.py | 10 ++++------ dataset/persistence/table.py | 8 ++------ dataset/util.py | 4 ++-- setup.py | 2 +- test/test_persistence.py | 29 ++++++++++++++--------------- 11 files changed, 28 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08d6c48..f6b58c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,3 +6,4 @@ python: - '2.6' script: - python setup.py test +- flake8 --ignore=E501,E123,E124,E126,E127,E128 dataset test diff --git a/dataset/__init__.py b/dataset/__init__.py index 3785251..a0713ac 100644 --- a/dataset/__init__.py +++ b/dataset/__init__.py @@ -8,7 +8,6 @@ from dataset.persistence.util import sqlite_datetime_fix from dataset.persistence.database import Database from dataset.persistence.table import Table from dataset.freeze.app import freeze -from sqlalchemy import Integer, Text __all__ = ['Database', 'Table', 'freeze', 'connect'] diff --git a/dataset/freeze/app.py b/dataset/freeze/app.py index 7498b4b..686ee29 100644 --- a/dataset/freeze/app.py +++ b/dataset/freeze/app.py @@ -33,7 +33,7 @@ def freeze(result, format='csv', filename='freeze.csv', result = db['person'].all() dataset.freeze(result, format='json', filename='all-persons.json') - + If ``result`` is a table (rather than a result set), all records in the table are exported (as if ``result.all()`` had been called). @@ -59,7 +59,7 @@ def freeze(result, format='csv', filename='freeze.csv', *json* A JSON file containing a list of dictionaries for each row - in the table. If a ``callback`` is given, JSON with padding + in the table. If a ``callback`` is given, JSON with padding (JSONP) will be generated. *tabson* diff --git a/dataset/freeze/config.py b/dataset/freeze/config.py index 46f52db..d4c711a 100644 --- a/dataset/freeze/config.py +++ b/dataset/freeze/config.py @@ -82,8 +82,7 @@ class Export(object): @property def skip(self): return self.get_bool('skip') - + @property def name(self): return self.get('name', self.get('query')) - diff --git a/dataset/freeze/format/common.py b/dataset/freeze/format/common.py index 51ea9a2..c112d41 100644 --- a/dataset/freeze/format/common.py +++ b/dataset/freeze/format/common.py @@ -1,5 +1,4 @@ import os -import logging import re import locale @@ -69,8 +68,7 @@ class Serializer(object): @property def wrap(self): - return self.export.get_bool('wrap', - default=self.mode=='list') + return self.export.get_bool('wrap', default=self.mode == 'list') def serialize(self): self.init() diff --git a/dataset/freeze/format/ftabson.py b/dataset/freeze/format/ftabson.py index f902604..620bc44 100644 --- a/dataset/freeze/format/ftabson.py +++ b/dataset/freeze/format/ftabson.py @@ -21,4 +21,3 @@ class TabsonSerializer(JSONSerializer): if meta is not None: result['meta'] = meta return result - diff --git a/dataset/persistence/database.py b/dataset/persistence/database.py index b1c07bf..cfefb99 100644 --- a/dataset/persistence/database.py +++ b/dataset/persistence/database.py @@ -11,9 +11,9 @@ except ImportError: from sqlalchemy import create_engine from sqlalchemy.pool import NullPool -from sqlalchemy.schema import MetaData, Column, Index +from sqlalchemy.schema import MetaData, Column from sqlalchemy.schema import Table as SQLATable -from sqlalchemy import Integer, Text, String +from sqlalchemy import Integer, String from alembic.migration import MigrationContext from alembic.operations import Operations @@ -107,10 +107,8 @@ class Database(object): @property def tables(self): - """ Get a listing of all tables that exist in the database. - - >>> print db.tables - set([u'user', u'action']) + """ + Get a listing of all tables that exist in the database. """ return list( set(self.metadata.tables.keys()) | set(self._tables.keys()) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 29c45d1..55c294c 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -24,9 +24,6 @@ class Table(object): def columns(self): """ Get a listing of all columns that exist in the table. - - >>> print 'age' in table.columns - True """ return set(self.table.columns.keys()) @@ -90,7 +87,7 @@ class Table(object): self._ensure_columns(row, types=types) self.table.insert().execute(chunk) self._check_dropped() - + chunk = [] for i, row in enumerate(rows, start=1): chunk.append(row) @@ -100,7 +97,6 @@ class Table(object): if chunk: _process_chunk(chunk) - def update(self, row, keys, ensure=True, types={}): """ @@ -122,7 +118,7 @@ class Table(object): if not isinstance(keys, (list, tuple)): keys = [keys] self._check_dropped() - if not keys or len(keys)==len(row): + if not keys or len(keys) == len(row): return False clause = [(u, row.get(u)) for u in keys] diff --git a/dataset/util.py b/dataset/util.py index 1f8083c..9da9936 100644 --- a/dataset/util.py +++ b/dataset/util.py @@ -1,12 +1,12 @@ #coding: utf-8 import re -from unicodedata import normalize as ucnorm, category SLUG_REMOVE = re.compile(r'[,\s\.\(\)/\\;:]*') + class DatasetException(Exception): pass + class FreezeException(DatasetException): pass - diff --git a/setup.py b/setup.py index 49c9b9d..b823dc4 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( 'python-slugify >= 0.0.6', "PyYAML >= 3.10" ] + py26_dependency, - tests_require=[], + tests_require=['flake8'], test_suite='test', entry_points={ 'console_scripts': [ diff --git a/test/test_persistence.py b/test/test_persistence.py index c8922a3..d9a7bab 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -8,7 +8,6 @@ from dataset import connect from dataset.util import DatasetException from .sample_data import TEST_DATA, TEST_CITY_1 -from sqlalchemy.exc import IntegrityError class DatabaseTestCase(unittest.TestCase): @@ -53,7 +52,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_custom_id2(self): pid = "string_id" @@ -64,19 +63,19 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_custom_id3(self): pid = "int_id" - table = self.db.create_table("foo4", primary_id = pid) + table = self.db.create_table("foo4", primary_id=pid) assert table.table.exists() assert len(table.table.columns) == 1, table.table.columns assert pid in table.table.c, table.table.c table.insert({'int_id': 123}) table.insert({'int_id': 124}) - assert table.find_one(int_id = 123)['int_id'] == 123 - assert table.find_one(int_id = 124)['int_id'] == 124 + assert table.find_one(int_id=123)['int_id'] == 123 + assert table.find_one(int_id=124)['int_id'] == 124 self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123})) def test_create_table_shorthand1(self): @@ -88,8 +87,8 @@ class DatabaseTestCase(unittest.TestCase): table.insert({'int_id': 123}) table.insert({'int_id': 124}) - assert table.find_one(int_id = 123)['int_id'] == 123 - assert table.find_one(int_id = 124)['int_id'] == 124 + assert table.find_one(int_id=123)['int_id'] == 123 + assert table.find_one(int_id=124)['int_id'] == 124 self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123})) def test_create_table_shorthand2(self): @@ -101,7 +100,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_shorthand3(self): pid = "string_id" @@ -112,7 +111,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_load_table(self): tbl = self.db.load_table('weather') @@ -138,7 +137,7 @@ class TableTestCase(unittest.TestCase): 'temperature': -10, 'place': 'Berlin'} ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) assert self.tbl.find_one(id=last_id)['place'] == 'Berlin' def test_upsert(self): @@ -148,17 +147,17 @@ class TableTestCase(unittest.TestCase): 'place': 'Berlin'}, ['place'] ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) self.tbl.upsert({ 'date': datetime(2011, 1, 2), 'temperature': -10, 'place': 'Berlin'}, ['place'] ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) def test_upsert_all_key(self): - for i in range(0,2): + for i in range(0, 2): self.tbl.upsert({ 'date': datetime(2011, 1, 2), 'temperature': -10, @@ -172,7 +171,7 @@ class TableTestCase(unittest.TestCase): 'temperature': -10, 'place': 'Berlin'} ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) self.tbl.delete(place='Berlin') assert len(self.tbl) == len(TEST_DATA), len(self.tbl) self.tbl.delete() From 697cfcf16551bd9a213b6ebd597dadc73780126a Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 22:13:52 +0100 Subject: [PATCH 11/13] Explicitly install flake8 and run it --- .travis.yml | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6b58c4..49c047f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - '3.3' - '2.7' - '2.6' +install: +- pip install flake8 script: -- python setup.py test - flake8 --ignore=E501,E123,E124,E126,E127,E128 dataset test +- python setup.py test diff --git a/setup.py b/setup.py index b823dc4..49c9b9d 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( 'python-slugify >= 0.0.6', "PyYAML >= 3.10" ] + py26_dependency, - tests_require=['flake8'], + tests_require=[], test_suite='test', entry_points={ 'console_scripts': [ From 47456b35a7065b1363d95603b145bb847e59f84e Mon Sep 17 00:00:00 2001 From: Stefan Wehrmeyer Date: Sat, 25 Jan 2014 22:19:03 +0100 Subject: [PATCH 12/13] Fix type comparison to isinstance check --- test/test_persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_persistence.py b/test/test_persistence.py index d9a7bab..696f746 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -249,7 +249,7 @@ class TableTestCase(unittest.TestCase): tbl = self.tbl tbl.create_column('foo', FLOAT) assert 'foo' in tbl.table.c, tbl.table.c - assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type + assert isinstance(tbl.table.c['foo'].type, FLOAT), tbl.table.c['foo'].type assert 'foo' in tbl.columns, tbl.columns def test_key_order(self): From ec4c91c20ede286622cb2681d8ccafe2fdeeff64 Mon Sep 17 00:00:00 2001 From: Chen Liu Date: Sat, 25 Jan 2014 17:57:29 -0500 Subject: [PATCH 13/13] add 'build' to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a9f95c..3de0350 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *.egg-info dist/* +build/* .DS_Store .watchr