Merge pull request #110 from xrotwang/master

extended test coverage for the freeze package, thereby fixing some bugs.

dataset/freeze/app.py

@@ -1,7 +1,7 @@
 import logging
 import argparse
 
-from sqlalchemy.exc import ProgrammingError
+from sqlalchemy.exc import ProgrammingError, OperationalError
 from dataset.util import FreezeException
 from dataset.persistence.table import Table
 from dataset.persistence.database import Database
@@ -11,13 +11,16 @@ from dataset.freeze.format import get_serializer
 
 log = logging.getLogger(__name__)
 
-parser = argparse.ArgumentParser(
-    description='Generate static JSON and CSV extracts from a SQL database.',
-    epilog='For further information, please check the documentation.')
-parser.add_argument('config', metavar='CONFIG', type=str,
-                    help='freeze file cofiguration')
-parser.add_argument('--db', default=None,
-                    help='Override the freezefile database URI')
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description='Generate static JSON and CSV extracts from a SQL database.',
+        epilog='For further information, please check the documentation.')
+    parser.add_argument('config', metavar='CONFIG', type=str,
+                        help='freeze file cofiguration')
+    parser.add_argument('--db', default=None,
+                        help='Override the freezefile database URI')
+    return parser
 
 
 def freeze(result, format='csv', filename='freeze.csv', fileobj=None,
@@ -102,28 +105,32 @@ def freeze_export(export, result=None):
         serializer_cls = get_serializer(export)
         serializer = serializer_cls(export, query)
         serializer.serialize()
-    except ProgrammingError as pe:
-        raise FreezeException("Invalid query: %s" % pe)
+    except (OperationalError, ProgrammingError) as e:
+        raise FreezeException("Invalid query: %s" % e)
 
 
-def main():
+def freeze_with_config(config, db=None):
+    for export in config.exports:
+        if db is not None:
+            export.data['database'] = db
+        if export.skip:
+            log.info("Skipping: %s", export.name)
+            continue
+        log.info("Running: %s", export.name)
+        freeze_export(export)
+
+
+def main():  # pragma: no cover
     # Set up default logger.
     logging.basicConfig(level=logging.INFO)
 
     try:
+        parser = create_parser()
         args = parser.parse_args()
-        config = Configuration(args.config)
-        for export in config.exports:
-            if args.db is not None:
-                export.data['database'] = args.db
-            if export.skip:
-                log.info("Skipping: %s", export.name)
-                continue
-            log.info("Running: %s", export.name)
-            freeze_export(export)
+        freeze_with_config(Configuration(args.config), args.db)
     except FreezeException as fe:
         log.error(fe)
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     logging.basicConfig(level=logging.DEBUG)
     main()

dataset/freeze/format/common.py

@@ -28,7 +28,7 @@ class Serializer(object):
         self._get_basepath()
 
         if export.get('filename') == '-':
-            export['fileobj'] = sys.stdout
+            export.data['fileobj'] = sys.stdout
         self.fileobj = export.get('fileobj')
 
     def _get_basepath(self):

dataset/freeze/format/fcsv.py

@@ -34,7 +34,7 @@ class CSVSerializer(Serializer):
 
             # handle fileobj that has been passed in:
             if path is not None:
-                if PY3:
+                if PY3:  # pragma: no cover
                     fh = open(path, 'wt', encoding='utf8', newline='')
                 else:
                     fh = open(path, 'wb')
@@ -42,7 +42,7 @@ class CSVSerializer(Serializer):
                 fh = self.fileobj
 
             writer = csv.writer(fh)
-            if PY3:
+            if PY3:  # pragma: no cover
                 writer.writerow(keys)
             else:
                 writer.writerow([k.encode('utf-8') for k in keys])

dataset/freeze/format/fjson.py

@@ -1,14 +1,16 @@
 import json
-from datetime import datetime
+from datetime import datetime, date
 from collections import defaultdict
 
+from six import PY3
+
 from dataset.freeze.format.common import Serializer
 
 
 class JSONEncoder(json.JSONEncoder):
 
     def default(self, obj):
-        if isinstance(obj, datetime):
+        if isinstance(obj, (datetime, date)):
             return obj.isoformat()
 
 
@@ -37,7 +39,13 @@ class JSONSerializer(Serializer):
         for path, result in self.buckets.items():
             result = self.wrap(result)
 
-            fh = open(path, 'wb') if self.fileobj is None else self.fileobj
+            if self.fileobj is None:
+                if PY3:  # pragma: no cover
+                    fh = open(path, 'w', encoding='utf8')
+                else:
+                    fh = open(path, 'wb')
+            else:
+                fh = self.fileobj
 
             data = json.dumps(result,
                               cls=JSONEncoder,

dataset/persistence/database.py

@@ -3,12 +3,7 @@ import threading
 import re
 from sqlalchemy.util import safe_reraise
 
-try:
-    from urllib.parse import urlencode
-    from urllib.parse import parse_qs
-except ImportError:
-    from urllib import urlencode
-    from urlparse import parse_qs
+from six.moves.urllib.parse import urlencode, parse_qs
 
 from sqlalchemy import create_engine
 from sqlalchemy.pool import NullPool

dataset/persistence/util.py

@@ -3,7 +3,7 @@ from inspect import isgenerator
 
 try:
     from collections import OrderedDict
-except ImportError:
+except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict
 
 from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event

setup.cfg

@@ -1,2 +1,5 @@
 [metadata]
 description-file = README.md
+
+[flake8]
+ignore = E501,E123,E124,E126,E127,E128 

test/test_freeze.py

@@ -60,3 +60,18 @@ class FreezeTestCase(unittest.TestCase):
                     else:
                         v1 = '%s' % v1
                 self.assertEqual(v2, v1)
+
+
+class SerializerTestCase(unittest.TestCase):
+    def test_Serializer(self):
+        from dataset.freeze.format.common import Serializer
+        from dataset.freeze.config import Export
+        from dataset.util import FreezeException
+
+        self.assertRaises(FreezeException, Serializer, {}, {})
+        s = Serializer(Export({'filename': 'f'}, {'mode': 'nomode'}), '')
+        self.assertRaises(FreezeException, getattr, s, 'wrap')
+        s = Serializer(Export({'filename': 'f'}, {}), '')
+        s.wrap
+        s = Serializer(Export({'filename': '-'}, {}), '')
+        self.assert_(s.fileobj)

test/test_freeze_app.py

@@ -0,0 +1,62 @@
+"""
+Test CLI following the recipe at http://dustinrcollins.com/testing-python-command-line-apps
+"""
+import os
+from unittest import TestCase
+from tempfile import mkdtemp
+from shutil import rmtree
+from copy import copy
+
+from six import StringIO
+
+from dataset import connect
+from dataset.util import FreezeException
+from dataset.freeze.config import Configuration, Export
+from dataset.freeze.app import create_parser, freeze_with_config, freeze_export
+from .sample_data import TEST_DATA
+
+
+class FreezeTestCase(TestCase):
+    """
+    Base TestCase class, sets up a CLI parser
+    """
+    def setUp(self):
+        parser = create_parser()
+        self.parser = parser
+        self.d = mkdtemp()
+        self.db_path = os.path.abspath(os.path.join(self.d, 'db.sqlite'))
+        self.db = 'sqlite:///' + self.db_path
+        _db = connect(self.db)
+        tbl = _db['weather']
+        for i, row in enumerate(TEST_DATA):
+            _row = copy(row)
+            _row['count'] = i
+            _row['bool'] = True
+            _row['none'] = None
+            tbl.insert(_row)
+
+    def tearDown(self):
+        rmtree(self.d, ignore_errors=True)
+
+    def test_with_empty_args(self):
+        """
+        User passes no args, should fail with SystemExit
+        """
+        self.assertRaises(SystemExit, self.parser.parse_args, [])
+
+    def test_with_config(self):
+        """
+        """
+        cfg = Configuration(os.path.join(os.path.dirname(__file__), 'Freezefile.yaml'))
+        cfg.data['common']['database'] = self.db
+        cfg.data['common']['prefix'] = self.d
+        cfg.data['common']['query'] = 'SELECT * FROM weather'
+        cfg.data['exports'] = [
+            {'filename': '{{identity:count}}.json', 'mode': 'item', 'transform': {'bool': 'identity'}},
+            {'filename': 'weather.json', 'format': 'tabson'},
+            {'filename': 'weather.csv', 'fileobj': StringIO(), 'format': 'csv'},
+            {'filename': 'weather.json', 'fileobj': StringIO(), 'format': 'tabson'},
+            {'filename': 'weather.json', 'format': 'tabson', 'callback': 'read'},
+            {'skip': True}]
+        freeze_with_config(cfg, db=self.db)
+        self.assertRaises(FreezeException, freeze_export, Export(cfg.data['common'], {'query': 'SELECT * FROM notable'}))

test/test_persistence.py

@@ -4,7 +4,7 @@ from datetime import datetime
 
 try:
     from collections import OrderedDict
-except ImportError:
+except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict  # Python < 2.7 drop-in
 
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError