Add parallel checker execution with connection pooling
Implements Level 2 parallelization for row_count, schema, and
aggregate checkers, improving performance by 2-3x for tables with
multiple enabled checks.
Changes:
- Add max_workers config option (default: 4)
- Add ConnectionPool module with SQLAlchemy QueuePool
- Add URL encoding for connection strings
- Implement parallel checker execution with ThreadPoolExecutor
- Add fail-fast behavior on checker errors
- Update executor for SQLAlchemy 2.0 compatibility
- Fix engine disposal resource leak
- Cache pooled engines in ConnectionManager
- Add disconnect() cleanup for pooled engines
Performance:
- Sequential: 3 checkers × 100ms = 300ms
- Parallel: 3 checkers ≈ 100ms (2-3x speedup)
Configuration:
execution:
max_workers: 4 # Controls parallel checker execution
continue_on_error: true
This commit is contained in:
88
tests/test_parallelization.py
Normal file
88
tests/test_parallelization.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Test parallelization features."""
|
||||
|
||||
import pytest
|
||||
from drt.config.models import Config, ExecutionConfig
|
||||
|
||||
|
||||
class TestParallelizationConfig:
|
||||
"""Test parallelization configuration."""
|
||||
|
||||
def test_default_max_workers(self):
|
||||
"""Test default max_workers value."""
|
||||
config = Config()
|
||||
assert config.execution.max_workers == 4
|
||||
|
||||
def test_custom_max_workers(self):
|
||||
"""Test custom max_workers value."""
|
||||
config = Config(execution=ExecutionConfig(max_workers=8))
|
||||
assert config.execution.max_workers == 8
|
||||
|
||||
def test_max_workers_positive(self):
|
||||
"""Test that max_workers is positive."""
|
||||
config = Config()
|
||||
assert config.execution.max_workers > 0
|
||||
|
||||
def test_continue_on_error_default(self):
|
||||
"""Test default continue_on_error value."""
|
||||
config = Config()
|
||||
assert config.execution.continue_on_error is True
|
||||
|
||||
|
||||
def test_imports():
|
||||
"""Test that all parallelization modules can be imported."""
|
||||
from drt.config.models import Config, ExecutionConfig
|
||||
from urllib.parse import quote_plus
|
||||
from sqlalchemy import create_engine, text, QueuePool
|
||||
|
||||
assert Config is not None
|
||||
assert ExecutionConfig is not None
|
||||
assert quote_plus is not None
|
||||
assert create_engine is not None
|
||||
assert text is not None
|
||||
assert QueuePool is not None
|
||||
|
||||
|
||||
def test_url_encoding():
|
||||
"""Test URL encoding for connection strings."""
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
test_conn_str = "DRIVER={ODBC Driver};SERVER=localhost;PWD=test@pass#123"
|
||||
encoded = quote_plus(test_conn_str)
|
||||
|
||||
# Special characters should be encoded
|
||||
assert "%40" in encoded # @ encoded
|
||||
assert "%23" in encoded # # encoded
|
||||
assert "%3D" in encoded # = encoded
|
||||
assert encoded != test_conn_str # Should be different after encoding
|
||||
|
||||
# Test that it can be decoded back
|
||||
from urllib.parse import unquote_plus
|
||||
decoded = unquote_plus(encoded)
|
||||
assert decoded == test_conn_str
|
||||
|
||||
|
||||
def test_config_load_minimal():
|
||||
"""Test loading a minimal config with parallelization settings."""
|
||||
config_dict = {
|
||||
"database_pairs": [
|
||||
{
|
||||
"name": "Test",
|
||||
"enabled": True,
|
||||
"baseline": {"server": "S1", "database": "D1"},
|
||||
"target": {"server": "S1", "database": "D2"}
|
||||
}
|
||||
],
|
||||
"execution": {
|
||||
"max_workers": 6,
|
||||
"continue_on_error": False
|
||||
},
|
||||
"tables": []
|
||||
}
|
||||
|
||||
config = Config(**config_dict)
|
||||
assert config.execution.max_workers == 6
|
||||
assert config.execution.continue_on_error is False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user