Reduce dependence on internal metadata caching, refs #208.

Browse Source
This commit is contained in:
Friedrich Lindenberg 2017-09-02 19:35:01 +02:00
parent 672b0bc8c4
commit a2748b7fde
3 changed files with 32 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
dataset/persistence/database.py
View File

@ -10,6 +10,8 @@ from sqlalchemy.schema import MetaData, Column
from sqlalchemy.schema import Table as SQLATable
from sqlalchemy.pool import StaticPool
from sqlalchemy.util import safe_reraise
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy.engine.reflection import Inspector
from alembic.migration import MigrationContext
from alembic.operations import Operations
@ -53,14 +55,7 @@ class Database(object):
self.url = url
self.row_type = row_type
self.ensure_schema = ensure_schema
self._tables = {}
self.metadata = MetaData(schema=schema)
self.metadata.bind = self.engine
if reflect_metadata:
self.metadata.reflect(self.engine, views=reflect_views)
for table_name in self.metadata.tables.keys():
self.load_table(self.metadata.tables[table_name].name)
self.reflect_views = reflect_views
@property
def executable(self):
@ -75,6 +70,11 @@ class Database(object):
ctx = MigrationContext.configure(self.executable)
return Operations(ctx)
@property
def metadata(self):
"""Return a SQLAlchemy schema cache object."""
return MetaData(schema=self.schema, bind=self.executable)
def _acquire(self):
self.lock.acquire()
@ -132,7 +132,8 @@ class Database(object):
@property
def tables(self):
"""Get a listing of all tables that exist in the database."""
return list(self._tables.keys())
inspector = Inspector.from_engine(self.executable)
return inspector.get_table_names(schema=self.schema)
def __contains__(self, member):
"""Check if the given table name exists in the database."""
@ -185,20 +186,19 @@ class Database(object):
if primary_id is not False:
primary_id = primary_id or Table.PRIMARY_DEFAULT
primary_type = primary_type or self.types.integer
autoincrement = primary_id == Table.PRIMARY_DEFAULT
autoincrement = primary_type in [self.types.integer,
self.types.bigint]
col = Column(primary_id, primary_type,
primary_key=True,
autoincrement=autoincrement)
table.append_column(col)
table.create(self.executable)
self._tables[table_name] = table
return Table(self, table)
finally:
self._release()
def load_table(self, table_name):
"""
Load a table.
"""Load a table.
This will fail if the tables does not already exist in the database. If the
table exists, its columns will be reflected and are available on the
@ -210,25 +210,21 @@ class Database(object):
table = db.load_table('population')
"""
table_name = self._valid_table_name(table_name)
self._acquire()
try:
log.debug("Loading table: %s on %r" % (table_name, self))
table = SQLATable(table_name, self.metadata,
schema=self.schema, autoload=True)
self._tables[table_name] = table
log.debug("Loading table: %s", table_name)
table = self._reflect_table(table_name)
if table is not None:
return Table(self, table)
finally:
self._release()
def update_table(self, table_name):
def _reflect_table(self, table_name):
"""Reload a table schema from the database."""
table_name = self._valid_table_name(table_name)
self.metadata = MetaData(schema=self.schema)
self.metadata.bind = self.executable
self.metadata.reflect(self.executable)
self._tables[table_name] = SQLATable(table_name, self.metadata,
schema=self.schema)
return self._tables[table_name]
try:
return SQLATable(table_name,
self.metadata,
schema=self.schema,
autoload=True)
except NoSuchTableError:
return None
def get_table(self, table_name, primary_id=None, primary_type=None):
"""
@ -246,24 +242,17 @@ class Database(object):
table = db['population']
"""
if table_name in self._tables:
return Table(self, self._tables[table_name])
self._acquire()
try:
if self.engine.has_table(table_name, schema=self.schema):
return self.load_table(table_name)
else:
return self.create_table(table_name, primary_id, primary_type)
finally:
self._release()
table = self._reflect_table(table_name)
if table is not None:
return Table(self, table)
return self.create_table(table_name, primary_id, primary_type)
def __getitem__(self, table_name):
"""Get a given table."""
return self.get_table(table_name)
def query(self, query, **kw):
"""
Run a statement on the database directly.
"""Run a statement on the database directly.
Allows for the execution of arbitrary read/write queries. A query can either be
a plain text string, or a `SQLAlchemy expression <http://docs.sqlalchemy.org/en/latest/core/tutorial.html#selecting>`_.

5
dataset/persistence/table.py
View File

@ -43,12 +43,11 @@ class Table(object):
dropping the table. If you want to re-create the table, make
sure to get a fresh instance from the :py:class:`Database <dataset.Database>`.
"""
self._check_dropped()
self.database._acquire()
try:
self._is_dropped = True
self.database._tables.pop(self.table.name, None)
self.table.drop(self.database.executable)
return True
finally:
self.database._release()
@ -313,7 +312,7 @@ class Table(object):
Column(name, type),
self.table.schema
)
self.table = self.database.update_table(self.table.name)
self.table = self.database._reflect_table(self.table.name)
finally:
self.database._release()

2
test/test_persistence.py
View File

@ -135,7 +135,7 @@ class DatabaseTestCase(unittest.TestCase):
def test_load_table(self):
tbl = self.db.load_table('weather')
assert tbl.table == self.tbl.table
assert tbl.table.name == self.tbl.table.name
def test_query(self):
r = self.db.query('SELECT COUNT(*) AS num FROM weather').next()