retoor/dataset
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge.

Browse Source
This commit is contained in:
Friedrich Lindenberg
2013-04-03 22:27:06 +02:00
parent 82153522cb 6d83953de3
commit f3533de1a7
19 changed files with 456 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dataset/__init__.py
+4 -3
View File
@@ -8,10 +8,11 @@ from dataset.persistence.table import Table
def connect(url):
""" Opens a new connection to a database. *url* can be any valid `SQLAlchemy engine URL`_. Returns
an instance of :py:class:`dataset.Database.
"""
Opens a new connection to a database. *url* can be any valid `SQLAlchemy engine URL`_. Returns
an instance of :py:class:`Database <dataset.Database>`.
::
db = dataset.connect('sqlite:///factbook.db')
.. _SQLAlchemy Engine URL: http://docs.sqlalchemy.org/en/latest/core/engines.html#sqlalchemy.create_engine
dataset/persistence/database.py
+39 -18
View File
@@ -32,15 +32,23 @@ class Database(object):
@property
def tables(self):
""" Get a listing of all tables that exist in the database. """
""" Get a listing of all tables that exist in the database.
>>> print db.tables
set([u'user', u'action'])
"""
return set(self.metadata.tables.keys() + self._tables.keys())
def create_table(self, table_name):
""" Creates a new table. The new table will automatically have
an `id` column, which is set to be an auto-incrementing integer
as the primary key of the table.
"""
Creates a new table. The new table will automatically have an `id` column, which is
set to be an auto-incrementing integer as the primary key of the table.
Returns a :py:class:`dataset.Table` instance."""
Returns a :py:class:`Table <dataset.Table>` instance.
::
table = db.create_table('population')
"""
with self.lock:
log.debug("Creating table: %s on %r" % (table_name, self.engine))
table = SQLATable(table_name, self.metadata)
@@ -51,12 +59,17 @@ class Database(object):
return Table(self, table)
def load_table(self, table_name):
""" Loads a table. This will fail if the tables does not already
"""
Loads a table. This will fail if the tables does not already
exist in the database. If the table exists, its columns will be
reflected and are available on the :py:class:`dataset.Table`
reflected and are available on the :py:class:`Table <dataset.Table>`
object.
Returns a :py:class:`dataset.Table` instance."""
Returns a :py:class:`Table <dataset.Table>` instance.
::
table = db.load_table('population')
"""
with self.lock:
log.debug("Loading table: %s on %r" % (table_name, self))
table = SQLATable(table_name, self.metadata, autoload=True)
@@ -64,9 +77,17 @@ class Database(object):
return Table(self, table)
def get_table(self, table_name):
""" Loads a table or creates it if it doesn't exist yet.
Returns a :py:class:`dataset.Table` instance. Alternatively to *get_table*
you can also get tables using the dict syntax."""
"""
Smart wrapper around *load_table* and *create_table*. Either loads a table
or creates it if it doesn't exist yet.
Returns a :py:class:`Table <dataset.Table>` instance.
::
table = db.get_table('population')
# you can also use the short-hand syntax:
table = db['population']
"""
with self.lock:
if table_name in self._tables:
return Table(self, self._tables[table_name])
@@ -79,16 +100,16 @@ class Database(object):
return self.get_table(table_name)
def query(self, query):
""" Run a statement on the database directly, allowing for the
"""
Run a statement on the database directly, allowing for the
execution of arbitrary read/write queries. A query can either be
a plain text string, or a SQLAlchemy expression. The returned
a plain text string, or a `SQLAlchemy expression <http://docs.sqlalchemy.org/ru/latest/core/tutorial.html#selecting>`_. The returned
iterator will yield each result sequentially.
::
.. code-block:: python
result = db.query('SELECT * FROM population WHERE population > 10000000')
for row in result:
print row
res = db.query('SELECT user, COUNT(*) c FROM photos GROUP BY user')
for row in res:
print row['user'], row['c']
"""
return resultiter(self.engine.execute(query))
dataset/persistence/table.py
+153 -50
View File
@@ -17,53 +17,69 @@ class Table(object):
self.database = database
self.table = table
@property
def columns(self):
"""
Get a listing of all columns that exist in the table.
>>> print 'age' in table.columns
True
"""
return set(self.table.columns.keys())
def drop(self):
""" Drop the table from the database, deleting both the schema
"""
Drop the table from the database, deleting both the schema
and all the contents within it.
Note: the object will be in an unusable state after using this
command and should not be used again. If you want to re-create
the table, make sure to get a fresh instance from the
:py:class:`dataset.Database`. """
:py:class:`Database <dataset.Database>`.
"""
with self.database.lock:
self.database.tables.pop(self.table.name, None)
self.table.drop(engine)
def insert(self, row, ensure=True, types={}):
""" Add a row (type: dict) by inserting it into the database.
"""
Add a row (type: dict) by inserting it into the table.
If ``ensure`` is set, any of the keys of the row are not
table columns, they will be created automatically.
table columns, they will be created automatically.
During column creation, ``types`` will be checked for a key
matching the name of a column to be created, and the given
matching the name of a column to be created, and the given
SQLAlchemy column type will be used. Otherwise, the type is
guessed from the row's value, defaulting to a simple unicode
field. """
guessed from the row value, defaulting to a simple unicode
field.
::
data = dict(id=10, title='I am a banana!')
table.insert(data, ['id'])
"""
if ensure:
self._ensure_columns(row, types=types)
self.database.engine.execute(self.table.insert(row))
def update(self, row, unique, ensure=True, types={}):
""" Update a row in the database. The update is managed via
the set of column names stated in ``unique``: they will be
def update(self, row, keys, ensure=True, types={}):
"""
Update a row in the table. The update is managed via
the set of column names stated in ``keys``: they will be
used as filters for the data to be updated, using the values
in ``row``. Example:
.. code-block:: python
in ``row``.
::
# update all entries with id matching 10, setting their title columns
data = dict(id=10, title='I am a banana!')
table.update(data, ['id'])
This will update all entries matching the given ``id``, setting
their ``title`` column.
If keys in ``row`` update columns not present in the table,
they will be created based on the settings of ``ensure`` and
``types``, matching the behaviour of ``insert``.
If keys in ``row`` update columns not present in the table,
they will be created based on the settings of ``ensure`` and
``types``, matching the behaviour of :py:meth:`insert() <dataset.Table.insert>`.
"""
if not len(unique):
if not len(keys):
return False
clause = [(u, row.get(u)) for u in unique]
clause = [(u, row.get(u)) for u in keys]
if ensure:
self._ensure_columns(row, types=types)
try:
@@ -71,17 +87,25 @@ class Table(object):
stmt = self.table.update(filters, row)
rp = self.database.engine.execute(stmt)
return rp.rowcount > 0
except KeyError, ke:
except KeyError:
return False
def upsert(self, row, unique, ensure=True, types={}):
if ensure:
self.create_index(unique)
def upsert(self, row, keys, ensure=True, types={}):
"""
An UPSERT is a smart combination of insert and update. If rows with matching ``keys`` exist
they will be updated, otherwise a new row is inserted in the table.
::
if not self.update(row, unique, ensure=ensure, types=types):
data = dict(id=10, title='I am a banana!')
table.upsert(data, ['id'])
"""
if ensure:
self.create_index(keys)
if not self.update(row, keys, ensure=ensure, types=types):
self.insert(row, ensure=ensure, types=types)
def delete(self, **kw):
def delete(self, **filter):
""" Delete rows from the table. Keyword arguments can be used
to add column-based filters. The filter criterion will always
be equality:
@@ -92,7 +116,7 @@ class Table(object):
If no arguments are given, all records are deleted.
"""
q = self._args_to_clause(kw)
q = self._args_to_clause(filter)
stmt = self.table.delete(q)
self.database.engine.execute(stmt)
@@ -102,8 +126,8 @@ class Table(object):
_type = types[column]
else:
_type = guess_type(row[column])
log.debug("Creating column: %s (%s) on %r" % (column,
_type, self.table.name))
log.debug("Creating column: %s (%s) on %r" % (column,
_type, self.table.name))
self.create_column(column, _type)
def _args_to_clause(self, args):
@@ -114,13 +138,26 @@ class Table(object):
return and_(*clauses)
def create_column(self, name, type):
"""
Explicitely create a new column ``name`` of a specified type.
``type`` must be a `SQLAlchemy column type <http://docs.sqlalchemy.org/en/rel_0_8/core/types.html>`_.
::
table.create_column('person', sqlalchemy.String)
"""
with self.database.lock:
if name not in self.table.columns.keys():
col = Column(name, type)
col.create(self.table,
connection=self.database.engine)
connection=self.database.engine)
def create_index(self, columns, name=None):
"""
Create an index to speed up queries on a table. If no ``name`` is given a random name is created.
::
table.create_index(['name', 'country'])
"""
with self.database.lock:
if not name:
sig = abs(hash('||'.join(columns)))
@@ -136,53 +173,119 @@ class Table(object):
self.indexes[name] = idx
return idx
def find_one(self, **kw):
res = list(self.find(_limit=1, **kw))
def find_one(self, **filter):
"""
Works just like :py:meth:`find() <dataset.Table.find>` but returns only one result.
::
row = table.find_one(country='United States')
"""
res = list(self.find(_limit=1, **filter))
if not len(res):
return None
return res[0]
def find(self, _limit=None, _step=5000, _offset=0,
order_by='id', **kw):
order_by = [self.table.c[order_by].asc()]
args = self._args_to_clause(kw)
def _args_to_order_by(self, order_by):
if order_by[0] == '-':
return self.table.c[order_by[1:]].desc()
else:
return self.table.c[order_by].asc()
def find(self, _limit=None, _offset=0, _step=5000,
order_by='id', **filter):
"""
Performs a simple search on the table. Simply pass keyword arguments as ``filter``.
::
results = table.find(country='France')
results = table.find(country='France', year=1980)
Using ``_limit``::
# just return the first 10 rows
results = table.find(country='France', _limit=10)
You can sort the results by single or multiple columns. Append a minus sign
to the column name for descending order::
# sort results by a column 'year'
results = table.find(country='France', order_by='year')
# return all rows sorted by multiple columns (by year in descending order)
results = table.find(order_by=['country', '-year'])
For more complex queries, please use :py:meth:`db.query() <dataset.Database.query>`
instead."""
if isinstance(order_by, (str, unicode)):
order_by = [order_by]
order_by = [self._args_to_order_by(o) for o in order_by]
args = self._args_to_clause(filter)
for i in count():
qoffset = _offset + (_step * i)
qlimit = _step
if _limit is not None:
qlimit = min(_limit-(_step*i), _step)
qlimit = min(_limit - (_step * i), _step)
if qlimit <= 0:
break
q = self.table.select(whereclause=args, limit=qlimit,
offset=qoffset, order_by=order_by)
offset=qoffset, order_by=order_by)
rows = list(self.database.query(q))
if not len(rows):
return
return
for row in rows:
yield row
def __len__(self):
"""
Returns the number of rows in the table.
"""
d = self.database.query(self.table.count()).next()
return d.values().pop()
def distinct(self, *columns, **kw):
def distinct(self, *columns, **filter):
"""
Returns all rows of a table, but removes rows in with duplicate values in ``columns``.
Interally this creates a `DISTINCT statement <http://www.w3schools.com/sql/sql_distinct.asp>`_.
::
# returns only one row per year, ignoring the rest
table.distinct('year')
# works with multiple columns, too
table.distinct('year', 'country')
# you can also combine this with a filter
table.distinct('year', country='China')
"""
qargs = []
try:
columns = [self.table.c[c] for c in columns]
for col, val in kw.items():
qargs.append(self.table.c[col]==val)
for col, val in filter.items():
qargs.append(self.table.c[col] == val)
except KeyError:
return []
q = expression.select(columns, distinct=True,
whereclause=and_(*qargs),
order_by=[c.asc() for c in columns])
whereclause=and_(*qargs),
order_by=[c.asc() for c in columns])
return self.database.query(q)
def all(self):
""" Return all records in the table, ordered by their
``id``. This is an alias for calling ``find`` without
any arguments. """
"""
Returns all rows of the table as simple dictionaries. This is simply a shortcut
to *find()* called with no arguments.
::
rows = table.all()"""
return self.find()
def __iter__(self):
"""
Allows for iterating over all rows in the table without explicetly
calling :py:meth:`all() <dataset.Table.all>`.
::
for row in table:
print row
"""
for row in self.all():
yield row