Reduce dependence on internal metadata caching, refs #208.

Browse Source
This commit is contained in:
Friedrich Lindenberg 2017-09-02 19:35:01 +02:00
parent 672b0bc8c4
commit a2748b7fde
3 changed files with 32 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
dataset/persistence/database.py
View File

@ -10,6 +10,8 @@ from sqlalchemy.schema import MetaData, Column
from sqlalchemy.schema import Table as SQLATable from sqlalchemy.schema import Table as SQLATable
from sqlalchemy.pool import StaticPool from sqlalchemy.pool import StaticPool
from sqlalchemy.util import safe_reraise from sqlalchemy.util import safe_reraise
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy.engine.reflection import Inspector
from alembic.migration import MigrationContext from alembic.migration import MigrationContext
from alembic.operations import Operations from alembic.operations import Operations
@ -53,14 +55,7 @@ class Database(object):
self.url = url self.url = url
self.row_type = row_type self.row_type = row_type
self.ensure_schema = ensure_schema self.ensure_schema = ensure_schema
self._tables = {} self.reflect_views = reflect_views
self.metadata = MetaData(schema=schema)
self.metadata.bind = self.engine
if reflect_metadata:
self.metadata.reflect(self.engine, views=reflect_views)
for table_name in self.metadata.tables.keys():
self.load_table(self.metadata.tables[table_name].name)
@property @property
def executable(self): def executable(self):
@ -75,6 +70,11 @@ class Database(object):
ctx = MigrationContext.configure(self.executable) ctx = MigrationContext.configure(self.executable)
return Operations(ctx) return Operations(ctx)
@property
def metadata(self):
"""Return a SQLAlchemy schema cache object."""
return MetaData(schema=self.schema, bind=self.executable)
def _acquire(self): def _acquire(self):
self.lock.acquire() self.lock.acquire()
@ -132,7 +132,8 @@ class Database(object):
@property @property
def tables(self): def tables(self):
"""Get a listing of all tables that exist in the database.""" """Get a listing of all tables that exist in the database."""
return list(self._tables.keys()) inspector = Inspector.from_engine(self.executable)
return inspector.get_table_names(schema=self.schema)
def __contains__(self, member): def __contains__(self, member):
"""Check if the given table name exists in the database.""" """Check if the given table name exists in the database."""
@ -185,20 +186,19 @@ class Database(object):
if primary_id is not False: if primary_id is not False:
primary_id = primary_id or Table.PRIMARY_DEFAULT primary_id = primary_id or Table.PRIMARY_DEFAULT
primary_type = primary_type or self.types.integer primary_type = primary_type or self.types.integer
autoincrement = primary_id == Table.PRIMARY_DEFAULT autoincrement = primary_type in [self.types.integer,
self.types.bigint]
col = Column(primary_id, primary_type, col = Column(primary_id, primary_type,
primary_key=True, primary_key=True,
autoincrement=autoincrement) autoincrement=autoincrement)
table.append_column(col) table.append_column(col)
table.create(self.executable) table.create(self.executable)
self._tables[table_name] = table
return Table(self, table) return Table(self, table)
finally: finally:
self._release() self._release()
def load_table(self, table_name): def load_table(self, table_name):
""" """Load a table.
Load a table.
This will fail if the tables does not already exist in the database. If the This will fail if the tables does not already exist in the database. If the
table exists, its columns will be reflected and are available on the table exists, its columns will be reflected and are available on the
@ -210,25 +210,21 @@ class Database(object):
table = db.load_table('population') table = db.load_table('population')
""" """
table_name = self._valid_table_name(table_name) table_name = self._valid_table_name(table_name)
self._acquire() log.debug("Loading table: %s", table_name)
try: table = self._reflect_table(table_name)
log.debug("Loading table: %s on %r" % (table_name, self)) if table is not None:
table = SQLATable(table_name, self.metadata,
schema=self.schema, autoload=True)
self._tables[table_name] = table
return Table(self, table) return Table(self, table)
finally:
self._release()
def update_table(self, table_name): def _reflect_table(self, table_name):
"""Reload a table schema from the database.""" """Reload a table schema from the database."""
table_name = self._valid_table_name(table_name) table_name = self._valid_table_name(table_name)
self.metadata = MetaData(schema=self.schema) try:
self.metadata.bind = self.executable return SQLATable(table_name,
self.metadata.reflect(self.executable) self.metadata,
self._tables[table_name] = SQLATable(table_name, self.metadata, schema=self.schema,
schema=self.schema) autoload=True)
return self._tables[table_name] except NoSuchTableError:
return None
def get_table(self, table_name, primary_id=None, primary_type=None): def get_table(self, table_name, primary_id=None, primary_type=None):
""" """
@ -246,24 +242,17 @@ class Database(object):
table = db['population'] table = db['population']
""" """
if table_name in self._tables: table = self._reflect_table(table_name)
return Table(self, self._tables[table_name]) if table is not None:
self._acquire() return Table(self, table)
try: return self.create_table(table_name, primary_id, primary_type)
if self.engine.has_table(table_name, schema=self.schema):
return self.load_table(table_name)
else:
return self.create_table(table_name, primary_id, primary_type)
finally:
self._release()
def __getitem__(self, table_name): def __getitem__(self, table_name):
"""Get a given table.""" """Get a given table."""
return self.get_table(table_name) return self.get_table(table_name)
def query(self, query, **kw): def query(self, query, **kw):
""" """Run a statement on the database directly.
Run a statement on the database directly.
Allows for the execution of arbitrary read/write queries. A query can either be Allows for the execution of arbitrary read/write queries. A query can either be
a plain text string, or a `SQLAlchemy expression <http://docs.sqlalchemy.org/en/latest/core/tutorial.html#selecting>`_. a plain text string, or a `SQLAlchemy expression <http://docs.sqlalchemy.org/en/latest/core/tutorial.html#selecting>`_.

5
dataset/persistence/table.py
View File

@ -43,12 +43,11 @@ class Table(object):
dropping the table. If you want to re-create the table, make dropping the table. If you want to re-create the table, make
sure to get a fresh instance from the :py:class:`Database <dataset.Database>`. sure to get a fresh instance from the :py:class:`Database <dataset.Database>`.
""" """
self._check_dropped()
self.database._acquire() self.database._acquire()
try: try:
self._is_dropped = True self._is_dropped = True
self.database._tables.pop(self.table.name, None)
self.table.drop(self.database.executable) self.table.drop(self.database.executable)
return True
finally: finally:
self.database._release() self.database._release()
@ -313,7 +312,7 @@ class Table(object):
Column(name, type), Column(name, type),
self.table.schema self.table.schema
) )
self.table = self.database.update_table(self.table.name) self.table = self.database._reflect_table(self.table.name)
finally: finally:
self.database._release() self.database._release()

2
test/test_persistence.py
View File

@ -135,7 +135,7 @@ class DatabaseTestCase(unittest.TestCase):
def test_load_table(self): def test_load_table(self):
tbl = self.db.load_table('weather') tbl = self.db.load_table('weather')
assert tbl.table == self.tbl.table assert tbl.table.name == self.tbl.table.name
def test_query(self): def test_query(self):
r = self.db.query('SELECT COUNT(*) AS num FROM weather').next() r = self.db.query('SELECT COUNT(*) AS num FROM weather').next()