261 lines
12 KiB
Python
261 lines
12 KiB
Python
|
"""
|
||
|
|
Integration tests for safety workflow scenarios.
|
||
|
|
|
||
|
|
Tests safety limit violations, emergency stop workflows, and failsafe mode operations.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import pytest_asyncio
|
||
|
|
from unittest.mock import Mock, AsyncMock
|
||
|
|
import asyncio
|
||
|
|
|
||
|
|
from src.database.flexible_client import FlexibleDatabaseClient
|
||
|
|
from src.core.auto_discovery import AutoDiscovery
|
||
|
|
from src.core.safety import SafetyLimitEnforcer
|
||
|
|
from src.core.emergency_stop import EmergencyStopManager
|
||
|
|
from src.core.setpoint_manager import SetpointManager
|
||
|
|
from src.core.security import SecurityManager
|
||
|
|
from src.core.compliance_audit import ComplianceAuditLogger
|
||
|
|
from src.monitoring.watchdog import DatabaseWatchdog
|
||
|
|
|
||
|
|
|
||
|
|
class TestSafetyWorkflows:
|
||
|
|
"""Test safety-related workflows and scenarios."""
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def db_client(self):
|
||
|
|
"""Create database client with safety test data."""
|
||
|
|
client = FlexibleDatabaseClient(
|
||
|
|
database_url="sqlite:///:memory:",
|
||
|
|
pool_size=5,
|
||
|
|
max_overflow=10,
|
||
|
|
pool_timeout=30
|
||
|
|
)
|
||
|
|
|
||
|
|
await client.connect()
|
||
|
|
client.create_tables()
|
||
|
|
|
||
|
|
# Insert safety test data
|
||
|
|
self._insert_safety_test_data(client)
|
||
|
|
return client
|
||
|
|
|
||
|
|
def _insert_safety_test_data(self, db_client):
|
||
|
|
"""Insert comprehensive safety test data."""
|
||
|
|
# Insert stations
|
||
|
|
db_client.execute(
|
||
|
|
"""INSERT INTO pump_stations (station_id, station_name, location) VALUES
|
||
|
|
('SAFETY_STATION_001', 'Safety Test Station 1', 'Test Location A'),
|
||
|
|
('SAFETY_STATION_002', 'Safety Test Station 2', 'Test Location B')"""
|
||
|
|
)
|
||
|
|
|
||
|
|
# Insert pumps with different safety configurations
|
||
|
|
db_client.execute(
|
||
|
|
"""INSERT INTO pumps (station_id, pump_id, pump_name, control_type,
|
||
|
|
min_speed_hz, max_speed_hz, default_setpoint_hz) VALUES
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_001', 'Safety Pump 1', 'DIRECT_SPEED', 20.0, 60.0, 35.0),
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_002', 'Safety Pump 2', 'LEVEL_CONTROLLED', 25.0, 55.0, 40.0),
|
||
|
|
('SAFETY_STATION_002', 'SAFETY_PUMP_003', 'Safety Pump 3', 'POWER_CONTROLLED', 30.0, 50.0, 38.0)"""
|
||
|
|
)
|
||
|
|
|
||
|
|
# Insert safety limits with different scenarios
|
||
|
|
db_client.execute(
|
||
|
|
"""INSERT INTO pump_safety_limits (station_id, pump_id, hard_min_speed_hz, hard_max_speed_hz,
|
||
|
|
hard_min_level_m, hard_max_level_m, emergency_stop_level_m, dry_run_protection_level_m,
|
||
|
|
hard_max_power_kw, hard_max_flow_m3h, max_starts_per_hour, min_run_time_seconds,
|
||
|
|
max_continuous_run_hours, max_speed_change_hz_per_min, set_by, approved_by) VALUES
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_001', 20.0, 60.0, 1.0, 5.0, 0.5, 1.5, 15.0, 200.0, 6, 300, 24, 10.0, 'admin', 'admin'),
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_002', 25.0, 55.0, 1.5, 4.5, 1.0, 2.0, 12.0, 180.0, 6, 300, 24, 10.0, 'admin', 'admin'),
|
||
|
|
('SAFETY_STATION_002', 'SAFETY_PUMP_003', 30.0, 50.0, 2.0, 4.0, 1.5, 2.5, 10.0, 160.0, 6, 300, 24, 10.0, 'admin', 'admin')"""
|
||
|
|
)
|
||
|
|
|
||
|
|
# Insert optimization plans that might trigger safety limits
|
||
|
|
import datetime
|
||
|
|
now = datetime.datetime.now()
|
||
|
|
db_client.execute(
|
||
|
|
f"""INSERT INTO pump_plans (station_id, pump_id, interval_start, interval_end,
|
||
|
|
suggested_speed_hz, target_level_m, target_power_kw, plan_status) VALUES
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_001', '{now}', '{now + datetime.timedelta(hours=1)}', 65.0, 3.0, 12.0, 'ACTIVE'),
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_002', '{now}', '{now + datetime.timedelta(hours=1)}', 15.0, 2.5, 10.0, 'ACTIVE'),
|
||
|
|
('SAFETY_STATION_002', 'SAFETY_PUMP_003', '{now}', '{now + datetime.timedelta(hours=1)}', 45.0, 3.5, 8.0, 'ACTIVE')"""
|
||
|
|
)
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def safety_components(self, db_client):
|
||
|
|
"""Create safety-related components for testing."""
|
||
|
|
# Create auto discovery
|
||
|
|
auto_discovery = AutoDiscovery(db_client)
|
||
|
|
await auto_discovery.discover()
|
||
|
|
|
||
|
|
# Create safety components
|
||
|
|
safety_enforcer = SafetyLimitEnforcer(db_client)
|
||
|
|
emergency_stop_manager = EmergencyStopManager(db_client)
|
||
|
|
|
||
|
|
# Load safety limits
|
||
|
|
await safety_enforcer.load_safety_limits()
|
||
|
|
|
||
|
|
# Create mock alert manager for watchdog
|
||
|
|
mock_alert_manager = Mock()
|
||
|
|
watchdog = DatabaseWatchdog(db_client, mock_alert_manager)
|
||
|
|
|
||
|
|
# Create setpoint manager
|
||
|
|
setpoint_manager = SetpointManager(
|
||
|
|
discovery=auto_discovery,
|
||
|
|
db_client=db_client,
|
||
|
|
safety_enforcer=safety_enforcer,
|
||
|
|
emergency_stop_manager=emergency_stop_manager,
|
||
|
|
watchdog=watchdog
|
||
|
|
)
|
||
|
|
|
||
|
|
# Create security components
|
||
|
|
security_manager = SecurityManager()
|
||
|
|
audit_logger = ComplianceAuditLogger(db_client)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'auto_discovery': auto_discovery,
|
||
|
|
'safety_enforcer': safety_enforcer,
|
||
|
|
'emergency_stop_manager': emergency_stop_manager,
|
||
|
|
'setpoint_manager': setpoint_manager,
|
||
|
|
'security_manager': security_manager,
|
||
|
|
'audit_logger': audit_logger,
|
||
|
|
'watchdog': watchdog
|
||
|
|
}
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_safety_limit_violation_workflow(self, safety_components):
|
||
|
|
"""Test complete workflow for safety limit violations."""
|
||
|
|
setpoint_manager = safety_components['setpoint_manager']
|
||
|
|
safety_enforcer = safety_components['safety_enforcer']
|
||
|
|
|
||
|
|
# Test pump with optimization plan that exceeds safety limits
|
||
|
|
station_id = 'SAFETY_STATION_001'
|
||
|
|
pump_id = 'SAFETY_PUMP_001'
|
||
|
|
|
||
|
|
# Get setpoint (should be enforced to max limit)
|
||
|
|
setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
|
||
|
|
# Verify setpoint is within safety limits
|
||
|
|
pump_info = setpoint_manager.discovery.get_pump(station_id, pump_id)
|
||
|
|
max_speed = pump_info.get('max_speed_hz', 60.0)
|
||
|
|
|
||
|
|
# The setpoint should be enforced to the maximum safety limit (60.0)
|
||
|
|
# even though the optimization plan requests 65.0
|
||
|
|
assert setpoint <= max_speed, f"Setpoint {setpoint} exceeds safety limit {max_speed}"
|
||
|
|
assert setpoint == 60.0, f"Setpoint should be enforced to safety limit, got {setpoint}"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_emergency_stop_clearance_workflow(self, safety_components):
|
||
|
|
"""Test emergency stop activation and clearance workflow."""
|
||
|
|
setpoint_manager = safety_components['setpoint_manager']
|
||
|
|
emergency_stop_manager = safety_components['emergency_stop_manager']
|
||
|
|
|
||
|
|
station_id = 'SAFETY_STATION_001'
|
||
|
|
pump_id = 'SAFETY_PUMP_002'
|
||
|
|
|
||
|
|
# Get normal setpoint
|
||
|
|
normal_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
assert normal_setpoint is not None
|
||
|
|
|
||
|
|
# Activate emergency stop
|
||
|
|
emergency_stop_manager.emergency_stop_pump(station_id, pump_id, "test_workflow")
|
||
|
|
|
||
|
|
# Verify setpoint is 0 during emergency stop
|
||
|
|
emergency_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
assert emergency_setpoint == 0.0, "Setpoint not zero during emergency stop"
|
||
|
|
|
||
|
|
# Clear emergency stop
|
||
|
|
emergency_stop_manager.clear_emergency_stop_pump(station_id, pump_id, "test_clearance")
|
||
|
|
|
||
|
|
# Verify setpoint returns to normal
|
||
|
|
recovered_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
assert recovered_setpoint == normal_setpoint, "Setpoint did not recover after emergency stop clearance"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_multiple_safety_violations(self, safety_components):
|
||
|
|
"""Test handling of multiple simultaneous safety violations."""
|
||
|
|
setpoint_manager = safety_components['setpoint_manager']
|
||
|
|
safety_enforcer = safety_components['safety_enforcer']
|
||
|
|
|
||
|
|
# Test multiple pumps with different safety violations
|
||
|
|
test_cases = [
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_001', 60.0), # Exceeds max speed (65.0 -> 60.0)
|
||
|
|
('SAFETY_STATION_001', 'SAFETY_PUMP_002', 25.0), # Below min speed (15.0 -> 25.0)
|
||
|
|
('SAFETY_STATION_002', 'SAFETY_PUMP_003', 45.0), # Within limits (45.0 -> 45.0)
|
||
|
|
]
|
||
|
|
|
||
|
|
for station_id, pump_id, expected_enforced_setpoint in test_cases:
|
||
|
|
setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
|
||
|
|
# Verify setpoint is within safety limits
|
||
|
|
pump_info = setpoint_manager.discovery.get_pump(station_id, pump_id)
|
||
|
|
min_speed = pump_info.get('min_speed_hz', 20.0)
|
||
|
|
max_speed = pump_info.get('max_speed_hz', 60.0)
|
||
|
|
|
||
|
|
assert min_speed <= setpoint <= max_speed, \
|
||
|
|
f"Setpoint {setpoint} out of range [{min_speed}, {max_speed}] for {station_id}/{pump_id}"
|
||
|
|
|
||
|
|
# Verify specific enforcement
|
||
|
|
assert setpoint == expected_enforced_setpoint, \
|
||
|
|
f"Setpoint enforcement failed for {station_id}/{pump_id}: expected {expected_enforced_setpoint}, got {setpoint}"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_safety_limit_dynamic_updates(self, safety_components, db_client):
|
||
|
|
"""Test that safety limit updates are reflected in real-time."""
|
||
|
|
setpoint_manager = safety_components['setpoint_manager']
|
||
|
|
safety_enforcer = safety_components['safety_enforcer']
|
||
|
|
|
||
|
|
station_id = 'SAFETY_STATION_001'
|
||
|
|
pump_id = 'SAFETY_PUMP_001'
|
||
|
|
|
||
|
|
# Get initial setpoint
|
||
|
|
initial_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
|
||
|
|
# Update safety limits to be more restrictive
|
||
|
|
db_client.execute(
|
||
|
|
"""UPDATE pump_safety_limits
|
||
|
|
SET hard_max_speed_hz = 45.0
|
||
|
|
WHERE station_id = 'SAFETY_STATION_001' AND pump_id = 'SAFETY_PUMP_001'"""
|
||
|
|
)
|
||
|
|
|
||
|
|
# Reload safety limits
|
||
|
|
await safety_enforcer.load_safety_limits()
|
||
|
|
|
||
|
|
# Get updated setpoint
|
||
|
|
updated_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
|
||
|
|
# Verify setpoint is now enforced to new limit
|
||
|
|
assert updated_setpoint <= 45.0, f"Setpoint {updated_setpoint} not enforced to new limit 45.0"
|
||
|
|
assert updated_setpoint < initial_setpoint, f"Setpoint should be lower after safety limit update: {updated_setpoint} < {initial_setpoint}"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_emergency_stop_cascade_effects(self, safety_components):
|
||
|
|
"""Test cascade effects of emergency stops on related systems."""
|
||
|
|
setpoint_manager = safety_components['setpoint_manager']
|
||
|
|
emergency_stop_manager = safety_components['emergency_stop_manager']
|
||
|
|
|
||
|
|
# Activate station-wide emergency stop
|
||
|
|
station_id = 'SAFETY_STATION_001'
|
||
|
|
emergency_stop_manager.emergency_stop_station(station_id, "station_emergency_test")
|
||
|
|
|
||
|
|
# Verify all pumps in the station are stopped
|
||
|
|
pumps = setpoint_manager.discovery.get_pumps(station_id)
|
||
|
|
for pump in pumps:
|
||
|
|
pump_id = pump['pump_id']
|
||
|
|
setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
assert setpoint == 0.0, f"Pump {pump_id} not stopped during station emergency"
|
||
|
|
|
||
|
|
# Verify pumps in other stations are unaffected
|
||
|
|
other_station_id = 'SAFETY_STATION_002'
|
||
|
|
other_pumps = setpoint_manager.discovery.get_pumps(other_station_id)
|
||
|
|
for pump in other_pumps:
|
||
|
|
pump_id = pump['pump_id']
|
||
|
|
setpoint = setpoint_manager.get_current_setpoint(other_station_id, pump_id)
|
||
|
|
assert setpoint > 0.0, f"Pump {pump_id} incorrectly stopped during unrelated station emergency"
|
||
|
|
|
||
|
|
# Clear station emergency stop
|
||
|
|
emergency_stop_manager.clear_emergency_stop_station(station_id, "station_clearance_test")
|
||
|
|
|
||
|
|
# Verify pumps return to normal operation
|
||
|
|
for pump in pumps:
|
||
|
|
pump_id = pump['pump_id']
|
||
|
|
setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
|
||
|
|
assert setpoint > 0.0, f"Pump {pump_id} not recovered after station emergency clearance"
|