diff --git a/.gitignore b/.gitignore index 8a9f95c..3de0350 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *.egg-info dist/* +build/* .DS_Store .watchr diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..49c047f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +--- +language: python +python: +- '3.3' +- '2.7' +- '2.6' +install: +- pip install flake8 +script: +- flake8 --ignore=E501,E123,E124,E126,E127,E128 dataset test +- python setup.py test diff --git a/dataset/__init__.py b/dataset/__init__.py index 3785251..a0713ac 100644 --- a/dataset/__init__.py +++ b/dataset/__init__.py @@ -8,7 +8,6 @@ from dataset.persistence.util import sqlite_datetime_fix from dataset.persistence.database import Database from dataset.persistence.table import Table from dataset.freeze.app import freeze -from sqlalchemy import Integer, Text __all__ = ['Database', 'Table', 'freeze', 'connect'] diff --git a/dataset/freeze/app.py b/dataset/freeze/app.py index 7498b4b..686ee29 100644 --- a/dataset/freeze/app.py +++ b/dataset/freeze/app.py @@ -33,7 +33,7 @@ def freeze(result, format='csv', filename='freeze.csv', result = db['person'].all() dataset.freeze(result, format='json', filename='all-persons.json') - + If ``result`` is a table (rather than a result set), all records in the table are exported (as if ``result.all()`` had been called). @@ -59,7 +59,7 @@ def freeze(result, format='csv', filename='freeze.csv', *json* A JSON file containing a list of dictionaries for each row - in the table. If a ``callback`` is given, JSON with padding + in the table. If a ``callback`` is given, JSON with padding (JSONP) will be generated. *tabson* diff --git a/dataset/freeze/config.py b/dataset/freeze/config.py index 46f52db..d4c711a 100644 --- a/dataset/freeze/config.py +++ b/dataset/freeze/config.py @@ -82,8 +82,7 @@ class Export(object): @property def skip(self): return self.get_bool('skip') - + @property def name(self): return self.get('name', self.get('query')) - diff --git a/dataset/freeze/format/common.py b/dataset/freeze/format/common.py index 51ea9a2..c112d41 100644 --- a/dataset/freeze/format/common.py +++ b/dataset/freeze/format/common.py @@ -1,5 +1,4 @@ import os -import logging import re import locale @@ -69,8 +68,7 @@ class Serializer(object): @property def wrap(self): - return self.export.get_bool('wrap', - default=self.mode=='list') + return self.export.get_bool('wrap', default=self.mode == 'list') def serialize(self): self.init() diff --git a/dataset/freeze/format/ftabson.py b/dataset/freeze/format/ftabson.py index f902604..620bc44 100644 --- a/dataset/freeze/format/ftabson.py +++ b/dataset/freeze/format/ftabson.py @@ -21,4 +21,3 @@ class TabsonSerializer(JSONSerializer): if meta is not None: result['meta'] = meta return result - diff --git a/dataset/persistence/database.py b/dataset/persistence/database.py index c536e50..417517e 100644 --- a/dataset/persistence/database.py +++ b/dataset/persistence/database.py @@ -11,9 +11,9 @@ except ImportError: from sqlalchemy import create_engine from sqlalchemy.pool import NullPool -from sqlalchemy.schema import MetaData, Column, Index +from sqlalchemy.schema import MetaData, Column from sqlalchemy.schema import Table as SQLATable -from sqlalchemy import Integer, Text, String +from sqlalchemy import Integer, String from alembic.migration import MigrationContext from alembic.operations import Operations @@ -107,10 +107,8 @@ class Database(object): @property def tables(self): - """ Get a listing of all tables that exist in the database. - - >>> print db.tables - set([u'user', u'action']) + """ + Get a listing of all tables that exist in the database. """ return list( set(self.metadata.tables.keys()) | set(self._tables.keys()) diff --git a/dataset/persistence/table.py b/dataset/persistence/table.py index 9b69d76..55c294c 100644 --- a/dataset/persistence/table.py +++ b/dataset/persistence/table.py @@ -1,9 +1,5 @@ import logging from itertools import count -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy.sql import and_, expression from sqlalchemy.schema import Column, Index @@ -28,9 +24,6 @@ class Table(object): def columns(self): """ Get a listing of all columns that exist in the table. - - >>> print 'age' in table.columns - True """ return set(self.table.columns.keys()) @@ -94,7 +87,7 @@ class Table(object): self._ensure_columns(row, types=types) self.table.insert().execute(chunk) self._check_dropped() - + chunk = [] for i, row in enumerate(rows, start=1): chunk.append(row) @@ -104,7 +97,6 @@ class Table(object): if chunk: _process_chunk(chunk) - def update(self, row, keys, ensure=True, types={}): """ @@ -126,7 +118,7 @@ class Table(object): if not isinstance(keys, (list, tuple)): keys = [keys] self._check_dropped() - if not keys or len(keys)==len(row): + if not keys or len(keys) == len(row): return False clause = [(u, row.get(u)) for u in keys] @@ -288,7 +280,7 @@ class Table(object): rp = self.database.executable.execute(query) data = rp.fetchone() if data is not None: - return OrderedDict(zip(rp.keys(), data)) + return data def _args_to_order_by(self, order_by): if order_by[0] == '-': @@ -337,6 +329,9 @@ class Table(object): rp = self.database.executable.execute(count_query) total_row_count = rp.fetchone()[0] + if _limit is None: + _limit = total_row_count + if _step is None or _step is False or _step == 0: _step = total_row_count @@ -348,9 +343,7 @@ class Table(object): for i in count(): qoffset = _offset + (_step * i) - qlimit = _step - if _limit is not None: - qlimit = min(_limit - (_step * i), _step) + qlimit = min(_limit - (_step * i), _step) if qlimit <= 0: break queries.append(self.table.select(whereclause=args, limit=qlimit, diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py index 2395797..5197aaf 100644 --- a/dataset/persistence/util.py +++ b/dataset/persistence/util.py @@ -1,9 +1,5 @@ from datetime import datetime, timedelta from inspect import isgenerator -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict # Python < 2.7 drop-in from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event @@ -50,7 +46,7 @@ class ResultIter(object): else: # stop here raise StopIteration - return OrderedDict(zip(self.keys, row)) + return row next = __next__ @@ -64,6 +60,8 @@ def sqlite_datetime_fix(): epoch = datetime(1970, 1, 1, 0, 0, 0) def process_bind_param(self, value, dialect): + if isinstance(value, datetime): + return value return (value / 1000 - self.epoch).total_seconds() def process_result_value(self, value, dialect): diff --git a/dataset/util.py b/dataset/util.py index 1f8083c..9da9936 100644 --- a/dataset/util.py +++ b/dataset/util.py @@ -1,12 +1,12 @@ #coding: utf-8 import re -from unicodedata import normalize as ucnorm, category SLUG_REMOVE = re.compile(r'[,\s\.\(\)/\\;:]*') + class DatasetException(Exception): pass + class FreezeException(DatasetException): pass - diff --git a/setup.py b/setup.py index 95ef190..49c9b9d 100644 --- a/setup.py +++ b/setup.py @@ -25,17 +25,18 @@ setup( author_email='info@okfn.org', url='http://github.com/pudo/dataset', license='MIT', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + packages=find_packages(exclude=['ez_setup', 'examples', 'test']), namespace_packages=[], include_package_data=False, zip_safe=False, install_requires=[ - 'sqlalchemy >= 0.8.1', - 'alembic >= 0.6.1', + 'sqlalchemy >= 0.9.1', + 'alembic >= 0.6.2', 'python-slugify >= 0.0.6', "PyYAML >= 3.10" ] + py26_dependency, tests_require=[], + test_suite='test', entry_points={ 'console_scripts': [ 'datafreeze = dataset.freeze.app:main', diff --git a/test/test_persistence.py b/test/test_persistence.py index 03a8b1c..696f746 100644 --- a/test/test_persistence.py +++ b/test/test_persistence.py @@ -2,15 +2,18 @@ import os import unittest from datetime import datetime +from sqlalchemy.exc import IntegrityError + from dataset import connect from dataset.util import DatasetException + from .sample_data import TEST_DATA, TEST_CITY_1 -from sqlalchemy.exc import IntegrityError class DatabaseTestCase(unittest.TestCase): def setUp(self): + self.old_db_url = os.environ.get('DATABASE_URL') os.environ['DATABASE_URL'] = 'sqlite:///:memory:' self.db = connect('sqlite:///:memory:') self.tbl = self.db['weather'] @@ -19,7 +22,10 @@ class DatabaseTestCase(unittest.TestCase): def tearDown(self): # ensure env variable was unset - del os.environ['DATABASE_URL'] + if self.old_db_url is None: + del os.environ['DATABASE_URL'] + else: + os.environ['DATABASE_URL'] = self.old_db_url def test_valid_database_url(self): assert self.db.url, os.environ['DATABASE_URL'] @@ -46,7 +52,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_custom_id2(self): pid = "string_id" @@ -57,19 +63,19 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_custom_id3(self): pid = "int_id" - table = self.db.create_table("foo4", primary_id = pid) + table = self.db.create_table("foo4", primary_id=pid) assert table.table.exists() assert len(table.table.columns) == 1, table.table.columns assert pid in table.table.c, table.table.c table.insert({'int_id': 123}) table.insert({'int_id': 124}) - assert table.find_one(int_id = 123)['int_id'] == 123 - assert table.find_one(int_id = 124)['int_id'] == 124 + assert table.find_one(int_id=123)['int_id'] == 123 + assert table.find_one(int_id=124)['int_id'] == 124 self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123})) def test_create_table_shorthand1(self): @@ -81,8 +87,8 @@ class DatabaseTestCase(unittest.TestCase): table.insert({'int_id': 123}) table.insert({'int_id': 124}) - assert table.find_one(int_id = 123)['int_id'] == 123 - assert table.find_one(int_id = 124)['int_id'] == 124 + assert table.find_one(int_id=123)['int_id'] == 123 + assert table.find_one(int_id=124)['int_id'] == 124 self.assertRaises(IntegrityError, lambda: table.insert({'int_id': 123})) def test_create_table_shorthand2(self): @@ -94,7 +100,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_create_table_shorthand3(self): pid = "string_id" @@ -105,7 +111,7 @@ class DatabaseTestCase(unittest.TestCase): table.insert({ 'string_id': 'foobar'}) - assert table.find_one(string_id = 'foobar')['string_id'] == 'foobar' + assert table.find_one(string_id='foobar')['string_id'] == 'foobar' def test_load_table(self): tbl = self.db.load_table('weather') @@ -131,7 +137,7 @@ class TableTestCase(unittest.TestCase): 'temperature': -10, 'place': 'Berlin'} ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) assert self.tbl.find_one(id=last_id)['place'] == 'Berlin' def test_upsert(self): @@ -141,17 +147,17 @@ class TableTestCase(unittest.TestCase): 'place': 'Berlin'}, ['place'] ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) self.tbl.upsert({ 'date': datetime(2011, 1, 2), 'temperature': -10, 'place': 'Berlin'}, ['place'] ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) def test_upsert_all_key(self): - for i in range(0,2): + for i in range(0, 2): self.tbl.upsert({ 'date': datetime(2011, 1, 2), 'temperature': -10, @@ -165,7 +171,7 @@ class TableTestCase(unittest.TestCase): 'temperature': -10, 'place': 'Berlin'} ) - assert len(self.tbl) == len(TEST_DATA)+1, len(self.tbl) + assert len(self.tbl) == len(TEST_DATA) + 1, len(self.tbl) self.tbl.delete(place='Berlin') assert len(self.tbl) == len(TEST_DATA), len(self.tbl) self.tbl.delete() @@ -187,6 +193,10 @@ class TableTestCase(unittest.TestCase): assert len(ds) == 3, ds ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2)) assert len(ds) == 2, ds + ds = list(self.tbl.find(place=TEST_CITY_1, _limit=2, _step=1)) + assert len(ds) == 2, ds + ds = list(self.tbl.find(place=TEST_CITY_1, _limit=1, _step=2)) + assert len(ds) == 1, ds def test_distinct(self): x = list(self.tbl.distinct('place')) @@ -195,8 +205,8 @@ class TableTestCase(unittest.TestCase): assert len(x) == 6, x def test_insert_many(self): - data = TEST_DATA * 5000 - self.tbl.insert_many(data) + data = TEST_DATA * 100 + self.tbl.insert_many(data, chunk_size=13) assert len(self.tbl) == len(data) + 6 def test_drop_warning(self): @@ -239,7 +249,7 @@ class TableTestCase(unittest.TestCase): tbl = self.tbl tbl.create_column('foo', FLOAT) assert 'foo' in tbl.table.c, tbl.table.c - assert FLOAT == type(tbl.table.c['foo'].type), tbl.table.c['foo'].type + assert isinstance(tbl.table.c['foo'].type, FLOAT), tbl.table.c['foo'].type assert 'foo' in tbl.columns, tbl.columns def test_key_order(self): @@ -247,6 +257,3 @@ class TableTestCase(unittest.TestCase): keys = list(res.next().keys()) assert keys[0] == 'temperature' assert keys[1] == 'place' - -if __name__ == '__main__': - unittest.main()