From d58e3ec691391b6958f320e7ed103495575870a7 Mon Sep 17 00:00:00 2001
From: xrotwang <xrotwang@googlemail.com>
Date: Thu, 6 Nov 2014 14:09:52 +0100
Subject: [PATCH 1/2] extended test coverage for the freeze package, thereby
 fixing some bugs.

---
 dataset/freeze/app.py           | 49 ++++++++++++++-----------
 dataset/freeze/format/common.py |  2 +-
 dataset/freeze/format/fcsv.py   |  4 +--
 dataset/freeze/format/fjson.py  | 14 ++++++--
 dataset/persistence/database.py |  7 +---
 dataset/persistence/util.py     |  2 +-
 setup.cfg                       |  3 ++
 test/test_freeze.py             | 15 ++++++++
 test/test_freeze_app.py         | 63 +++++++++++++++++++++++++++++++++
 test/test_persistence.py        |  2 +-
 10 files changed, 126 insertions(+), 35 deletions(-)
 create mode 100644 test/test_freeze_app.py

diff --git a/dataset/freeze/app.py b/dataset/freeze/app.py
index 541dd80..2b9304b 100644
--- a/dataset/freeze/app.py
+++ b/dataset/freeze/app.py
@@ -1,7 +1,7 @@
 import logging
 import argparse
 
-from sqlalchemy.exc import ProgrammingError
+from sqlalchemy.exc import ProgrammingError, OperationalError
 from dataset.util import FreezeException
 from dataset.persistence.table import Table
 from dataset.persistence.database import Database
@@ -11,13 +11,16 @@ from dataset.freeze.format import get_serializer
 
 log = logging.getLogger(__name__)
 
-parser = argparse.ArgumentParser(
-    description='Generate static JSON and CSV extracts from a SQL database.',
-    epilog='For further information, please check the documentation.')
-parser.add_argument('config', metavar='CONFIG', type=str,
-                    help='freeze file cofiguration')
-parser.add_argument('--db', default=None,
-                    help='Override the freezefile database URI')
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description='Generate static JSON and CSV extracts from a SQL database.',
+        epilog='For further information, please check the documentation.')
+    parser.add_argument('config', metavar='CONFIG', type=str,
+                        help='freeze file cofiguration')
+    parser.add_argument('--db', default=None,
+                        help='Override the freezefile database URI')
+    return parser
 
 
 def freeze(result, format='csv', filename='freeze.csv', fileobj=None,
@@ -102,28 +105,32 @@ def freeze_export(export, result=None):
         serializer_cls = get_serializer(export)
         serializer = serializer_cls(export, query)
         serializer.serialize()
-    except ProgrammingError as pe:
-        raise FreezeException("Invalid query: %s" % pe)
+    except (OperationalError, ProgrammingError) as e:
+        raise FreezeException("Invalid query: %s" % e)
 
 
-def main():
+def freeze_with_config(config, db=None):
+    for export in config.exports:
+        if db is not None:
+            export.data['database'] = db
+        if export.skip:
+            log.info("Skipping: %s", export.name)
+            continue
+        log.info("Running: %s", export.name)
+        freeze_export(export)
+
+
+def main():  # pragma: no cover
     # Set up default logger.
     logging.basicConfig(level=logging.INFO)
 
     try:
+        parser = create_parser()
         args = parser.parse_args()
-        config = Configuration(args.config)
-        for export in config.exports:
-            if args.db is not None:
-                export.data['database'] = args.db
-            if export.skip:
-                log.info("Skipping: %s", export.name)
-                continue
-            log.info("Running: %s", export.name)
-            freeze_export(export)
+        freeze_with_config(Configuration(args.config), args.db)
     except FreezeException as fe:
         log.error(fe)
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     logging.basicConfig(level=logging.DEBUG)
     main()
diff --git a/dataset/freeze/format/common.py b/dataset/freeze/format/common.py
index ee00369..e95ab30 100644
--- a/dataset/freeze/format/common.py
+++ b/dataset/freeze/format/common.py
@@ -28,7 +28,7 @@ class Serializer(object):
         self._get_basepath()
 
         if export.get('filename') == '-':
-            export['fileobj'] = sys.stdout
+            export.data['fileobj'] = sys.stdout
         self.fileobj = export.get('fileobj')
 
     def _get_basepath(self):
diff --git a/dataset/freeze/format/fcsv.py b/dataset/freeze/format/fcsv.py
index 961c3fd..d0835ef 100644
--- a/dataset/freeze/format/fcsv.py
+++ b/dataset/freeze/format/fcsv.py
@@ -34,7 +34,7 @@ class CSVSerializer(Serializer):
 
             # handle fileobj that has been passed in:
             if path is not None:
-                if PY3:
+                if PY3:  # pragma: no cover
                     fh = open(path, 'wt', encoding='utf8', newline='')
                 else:
                     fh = open(path, 'wb')
@@ -42,7 +42,7 @@ class CSVSerializer(Serializer):
                 fh = self.fileobj
 
             writer = csv.writer(fh)
-            if PY3:
+            if PY3:  # pragma: no cover
                 writer.writerow(keys)
             else:
                 writer.writerow([k.encode('utf-8') for k in keys])
diff --git a/dataset/freeze/format/fjson.py b/dataset/freeze/format/fjson.py
index 92f7b90..0aca5ca 100644
--- a/dataset/freeze/format/fjson.py
+++ b/dataset/freeze/format/fjson.py
@@ -1,14 +1,16 @@
 import json
-from datetime import datetime
+from datetime import datetime, date
 from collections import defaultdict
 
+from six import PY3
+
 from dataset.freeze.format.common import Serializer
 
 
 class JSONEncoder(json.JSONEncoder):
 
     def default(self, obj):
-        if isinstance(obj, datetime):
+        if isinstance(obj, (datetime, date)):
             return obj.isoformat()
 
 
@@ -37,7 +39,13 @@ class JSONSerializer(Serializer):
         for path, result in self.buckets.items():
             result = self.wrap(result)
 
-            fh = open(path, 'wb') if self.fileobj is None else self.fileobj
+            if self.fileobj is None:
+                if PY3:  # pragma: no cover
+                    fh = open(path, 'w', encoding='utf8')
+                else:
+                    fh = open(path, 'wb')
+            else:
+                fh = self.fileobj
 
             data = json.dumps(result,
                               cls=JSONEncoder,
diff --git a/dataset/persistence/database.py b/dataset/persistence/database.py
index 0687681..745063f 100644
--- a/dataset/persistence/database.py
+++ b/dataset/persistence/database.py
@@ -3,12 +3,7 @@ import threading
 import re
 from sqlalchemy.util import safe_reraise
 
-try:
-    from urllib.parse import urlencode
-    from urllib.parse import parse_qs
-except ImportError:
-    from urllib import urlencode
-    from urlparse import parse_qs
+from six.moves.urllib.parse import urlencode, parse_qs
 
 from sqlalchemy import create_engine
 from sqlalchemy.pool import NullPool
diff --git a/dataset/persistence/util.py b/dataset/persistence/util.py
index b41ef29..5d3d37d 100644
--- a/dataset/persistence/util.py
+++ b/dataset/persistence/util.py
@@ -3,7 +3,7 @@ from inspect import isgenerator
 
 try:
     from collections import OrderedDict
-except ImportError:
+except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict
 
 from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event
diff --git a/setup.cfg b/setup.cfg
index b88034e..6ca3a72 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
 [metadata]
 description-file = README.md
+
+[flake8]
+ignore = E501,E123,E124,E126,E127,E128 
diff --git a/test/test_freeze.py b/test/test_freeze.py
index b624155..fc164ae 100644
--- a/test/test_freeze.py
+++ b/test/test_freeze.py
@@ -60,3 +60,18 @@ class FreezeTestCase(unittest.TestCase):
                     else:
                         v1 = '%s' % v1
                 self.assertEqual(v2, v1)
+
+
+class SerializerTestCase(unittest.TestCase):
+    def test_Serializer(self):
+        from dataset.freeze.format.common import Serializer
+        from dataset.freeze.config import Export
+        from dataset.util import FreezeException
+
+        self.assertRaises(FreezeException, Serializer, {}, {})
+        s = Serializer(Export({'filename': 'f'}, {'mode': 'nomode'}), '')
+        self.assertRaises(FreezeException, getattr, s, 'wrap')
+        s = Serializer(Export({'filename': 'f'}, {}), '')
+        s.wrap
+        s = Serializer(Export({'filename': '-'}, {}), '')
+        self.assert_(s.fileobj)
diff --git a/test/test_freeze_app.py b/test/test_freeze_app.py
new file mode 100644
index 0000000..97e99d4
--- /dev/null
+++ b/test/test_freeze_app.py
@@ -0,0 +1,63 @@
+"""
+Test CLI following the recipe at http://dustinrcollins.com/testing-python-command-line-apps
+"""
+import os
+from unittest import TestCase
+from tempfile import mkdtemp
+from shutil import rmtree
+from copy import copy
+
+from six import StringIO
+
+from dataset import connect
+from dataset.util import FreezeException
+from dataset.freeze.config import Configuration, Export
+from dataset.freeze.app import create_parser, freeze_with_config, freeze_export
+from .sample_data import TEST_DATA
+
+
+class FreezeTestCase(TestCase):
+    """
+    Base TestCase class, sets up a CLI parser
+    """
+    def setUp(self):
+        parser = create_parser()
+        self.parser = parser
+        self.d = mkdtemp()
+        self.db_path = os.path.abspath(os.path.join(self.d, 'db.sqlite'))
+        self.db = 'sqlite:///' + self.db_path
+        _db = connect(self.db)
+        tbl = _db['weather']
+        for i, row in enumerate(TEST_DATA):
+            _row = copy(row)
+            _row['count'] = i
+            _row['bool'] = True
+            _row['none'] = None
+            tbl.insert(_row)
+
+    def tearDown(self):
+        rmtree(self.d, ignore_errors=True)
+
+    def test_with_empty_args(self):
+        """
+        User passes no args, should fail with SystemExit
+        """
+        with self.assertRaises(SystemExit):
+            self.parser.parse_args([])
+
+    def test_with_config(self):
+        """
+        """
+        cfg = Configuration(os.path.join(os.path.dirname(__file__), 'Freezefile.yaml'))
+        cfg.data['common']['database'] = self.db
+        cfg.data['common']['prefix'] = self.d
+        cfg.data['common']['query'] = 'SELECT * FROM weather'
+        cfg.data['exports'] = [
+            {'filename': '{{identity:count}}.json', 'mode': 'item', 'transform': {'bool': 'identity'}},
+            {'filename': 'weather.json', 'format': 'tabson'},
+            {'filename': 'weather.csv', 'fileobj': StringIO(), 'format': 'csv'},
+            {'filename': 'weather.json', 'fileobj': StringIO(), 'format': 'tabson'},
+            {'filename': 'weather.json', 'format': 'tabson', 'callback': 'read'},
+            {'skip': True}]
+        freeze_with_config(cfg, db=self.db)
+        self.assertRaises(FreezeException, freeze_export, Export(cfg.data['common'], {'query': 'SELECT * FROM notable'}))
diff --git a/test/test_persistence.py b/test/test_persistence.py
index c009c42..601ac26 100644
--- a/test/test_persistence.py
+++ b/test/test_persistence.py
@@ -4,7 +4,7 @@ from datetime import datetime
 
 try:
     from collections import OrderedDict
-except ImportError:
+except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict  # Python < 2.7 drop-in
 
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError

From f68866abc94a50ea8530cefd3490f67866555e4a Mon Sep 17 00:00:00 2001
From: xrotwang <xrotwang@googlemail.com>
Date: Thu, 6 Nov 2014 14:29:05 +0100
Subject: [PATCH 2/2] fixed python2.6 incompatible usage of assertRaises as
 context manager.

---
 test/test_freeze_app.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/test_freeze_app.py b/test/test_freeze_app.py
index 97e99d4..907f829 100644
--- a/test/test_freeze_app.py
+++ b/test/test_freeze_app.py
@@ -42,8 +42,7 @@ class FreezeTestCase(TestCase):
         """
         User passes no args, should fail with SystemExit
         """
-        with self.assertRaises(SystemExit):
-            self.parser.parse_args([])
+        self.assertRaises(SystemExit, self.parser.parse_args, [])
 
     def test_with_config(self):
         """