Update schema to be thread-safe

Browse Source
This commit is contained in:
Friedrich Lindenberg 2012-07-23 15:27:42 +02:00
parent 2c0f88ea74
commit ddcdfea48f
1 changed files with 53 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
sqlaload/schema.py
View File

@ -1,6 +1,7 @@
import logging import logging
from datetime import datetime from datetime import datetime
from collections import defaultdict from collections import defaultdict
from threading import RLock
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean
@ -9,6 +10,7 @@ from sqlalchemy.sql import and_, expression
from migrate.versioning.util import construct_engine from migrate.versioning.util import construct_engine
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
lock = RLock()
TABLES = defaultdict(dict) TABLES = defaultdict(dict)
INDEXES = dict() INDEXES = dict()
@ -16,8 +18,8 @@ INDEXES = dict()
def connect(url): def connect(url):
""" Create an engine for the given database URL. """ """ Create an engine for the given database URL. """
kw = {} kw = {}
if url.startswith('postgres'): #if url.startswith('postgres'):
kw['pool_size'] = 1 # kw['pool_size'] = 10
engine = create_engine(url, **kw) engine = create_engine(url, **kw)
engine = construct_engine(engine) engine = construct_engine(engine)
meta = MetaData() meta = MetaData()
@ -26,6 +28,7 @@ def connect(url):
return engine return engine
def create_table(engine, table_name): def create_table(engine, table_name):
with lock:
log.debug("Creating table: %s on %r" % (table_name, engine)) log.debug("Creating table: %s on %r" % (table_name, engine))
table = Table(table_name, engine._metadata) table = Table(table_name, engine._metadata)
col = Column('id', Integer, primary_key=True) col = Column('id', Integer, primary_key=True)
@ -35,6 +38,7 @@ def create_table(engine, table_name):
return table return table
def load_table(engine, table_name): def load_table(engine, table_name):
with lock:
log.debug("Loading table: %s on %r" % (table_name, engine)) log.debug("Loading table: %s on %r" % (table_name, engine))
table = Table(table_name, engine._metadata, autoload=True) table = Table(table_name, engine._metadata, autoload=True)
TABLES[engine][table_name] = table TABLES[engine][table_name] = table
@ -45,6 +49,7 @@ def get_table(engine, table_name):
if hasattr(engine, 'engine'): if hasattr(engine, 'engine'):
engine = engine.engine engine = engine.engine
with lock:
if table_name in TABLES[engine]: if table_name in TABLES[engine]:
return TABLES[engine][table_name] return TABLES[engine][table_name]
if engine.has_table(table_name): if engine.has_table(table_name):
@ -56,6 +61,8 @@ def drop_table(engine, table_name):
# Accept Connection objects here # Accept Connection objects here
if hasattr(engine, 'engine'): if hasattr(engine, 'engine'):
engine = engine.engine engine = engine.engine
with lock:
if table_name in TABLES[engine]: if table_name in TABLES[engine]:
table = TABLES[engine][table_name] table = TABLES[engine][table_name]
elif engine.has_table(table_name): elif engine.has_table(table_name):
@ -94,11 +101,13 @@ def _args_to_clause(table, args):
return and_(*clauses) return and_(*clauses)
def create_column(engine, table, name, type): def create_column(engine, table, name, type):
with lock:
if name not in table.columns.keys(): if name not in table.columns.keys():
col = Column(name, type) col = Column(name, type)
col.create(table, connection=engine) col.create(table, connection=engine)
def create_index(engine, table, columns, name=None): def create_index(engine, table, columns, name=None):
with lock:
if not name: if not name:
sig = abs(hash('||'.join(columns))) sig = abs(hash('||'.join(columns)))
name = 'ix_%s_%s' % (table.name, sig) name = 'ix_%s_%s' % (table.name, sig)