""" Integration tests for failsafe mode operations. Tests failsafe mode activation, operation, and recovery scenarios. """ import pytest import pytest_asyncio from unittest.mock import Mock, AsyncMock import asyncio from src.database.flexible_client import FlexibleDatabaseClient from src.core.auto_discovery import AutoDiscovery from src.core.safety import SafetyLimitEnforcer from src.core.emergency_stop import EmergencyStopManager from src.core.setpoint_manager import SetpointManager from src.core.security import SecurityManager from src.core.compliance_audit import ComplianceAuditLogger from src.monitoring.watchdog import DatabaseWatchdog class TestFailsafeOperations: """Test failsafe mode operations and scenarios.""" @pytest_asyncio.fixture async def db_client(self): """Create database client with failsafe test data.""" client = FlexibleDatabaseClient( database_url="sqlite:///:memory:", pool_size=5, max_overflow=10, pool_timeout=30 ) await client.connect() client.create_tables() # Insert failsafe test data self._insert_failsafe_test_data(client) return client def _insert_failsafe_test_data(self, db_client): """Insert comprehensive failsafe test data.""" # Insert stations db_client.execute( """INSERT INTO pump_stations (station_id, station_name, location) VALUES ('FAILSAFE_STATION_001', 'Failsafe Test Station 1', 'Test Location A'), ('FAILSAFE_STATION_002', 'Failsafe Test Station 2', 'Test Location B')""" ) # Insert pumps with different failsafe configurations db_client.execute( """INSERT INTO pumps (station_id, pump_id, pump_name, control_type, min_speed_hz, max_speed_hz, default_setpoint_hz) VALUES ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', 'Failsafe Pump 1', 'DIRECT_SPEED', 20.0, 60.0, 35.0), ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', 'Failsafe Pump 2', 'LEVEL_CONTROLLED', 25.0, 55.0, 40.0), ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', 'Failsafe Pump 3', 'POWER_CONTROLLED', 30.0, 50.0, 38.0)""" ) # Insert safety limits db_client.execute( """INSERT INTO pump_safety_limits (station_id, pump_id, hard_min_speed_hz, hard_max_speed_hz, hard_min_level_m, hard_max_level_m, emergency_stop_level_m, dry_run_protection_level_m, hard_max_power_kw, hard_max_flow_m3h, max_starts_per_hour, min_run_time_seconds, max_continuous_run_hours, max_speed_change_hz_per_min, set_by, approved_by) VALUES ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', 20.0, 60.0, 1.0, 5.0, 0.5, 1.5, 15.0, 200.0, 6, 300, 24, 10.0, 'admin', 'admin'), ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', 25.0, 55.0, 1.5, 4.5, 1.0, 2.0, 12.0, 180.0, 6, 300, 24, 10.0, 'admin', 'admin'), ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', 30.0, 50.0, 2.0, 4.0, 1.5, 2.5, 10.0, 160.0, 6, 300, 24, 10.0, 'admin', 'admin')""" ) # Insert optimization plans import datetime now = datetime.datetime.now() db_client.execute( f"""INSERT INTO pump_plans (station_id, pump_id, interval_start, interval_end, suggested_speed_hz, target_level_m, target_power_kw, plan_status) VALUES ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', '{now}', '{now + datetime.timedelta(hours=1)}', 42.5, 3.0, 12.0, 'ACTIVE'), ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', '{now}', '{now + datetime.timedelta(hours=1)}', 38.0, 2.5, 10.0, 'ACTIVE'), ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', '{now}', '{now + datetime.timedelta(hours=1)}', 36.0, 3.5, 8.0, 'ACTIVE')""" ) @pytest_asyncio.fixture async def failsafe_components(self, db_client): """Create failsafe-related components for testing.""" # Create auto discovery auto_discovery = AutoDiscovery(db_client) await auto_discovery.discover() # Create safety components safety_enforcer = SafetyLimitEnforcer(db_client) emergency_stop_manager = EmergencyStopManager(db_client) # Load safety limits await safety_enforcer.load_safety_limits() # Create mock alert manager for watchdog mock_alert_manager = Mock() watchdog = DatabaseWatchdog(db_client, mock_alert_manager) # Create setpoint manager setpoint_manager = SetpointManager( discovery=auto_discovery, db_client=db_client, safety_enforcer=safety_enforcer, emergency_stop_manager=emergency_stop_manager, watchdog=watchdog ) # Create security components security_manager = SecurityManager() audit_logger = ComplianceAuditLogger(db_client) return { 'auto_discovery': auto_discovery, 'safety_enforcer': safety_enforcer, 'emergency_stop_manager': emergency_stop_manager, 'setpoint_manager': setpoint_manager, 'security_manager': security_manager, 'audit_logger': audit_logger, 'watchdog': watchdog } @pytest.mark.asyncio async def test_failsafe_mode_activation(self, failsafe_components): """Test failsafe mode activation for individual pumps.""" setpoint_manager = failsafe_components['setpoint_manager'] watchdog = failsafe_components['watchdog'] station_id = 'FAILSAFE_STATION_001' pump_id = 'FAILSAFE_PUMP_001' # Get normal setpoint normal_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert normal_setpoint is not None # Activate failsafe mode await watchdog.activate_failsafe_mode(station_id, pump_id, "communication_loss") # Verify setpoint switches to failsafe mode failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) # In failsafe mode, should use default setpoint (35.0) as fallback # since we don't have a specific failsafe_setpoint_hz in the database assert failsafe_setpoint == 35.0, f"Setpoint not in failsafe mode: {failsafe_setpoint}" @pytest.mark.asyncio async def test_failsafe_mode_recovery(self, failsafe_components): """Test failsafe mode recovery and return to normal operation.""" setpoint_manager = failsafe_components['setpoint_manager'] watchdog = failsafe_components['watchdog'] station_id = 'FAILSAFE_STATION_001' pump_id = 'FAILSAFE_PUMP_002' # Get normal setpoint normal_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) # Activate failsafe mode await watchdog.activate_failsafe_mode(station_id, pump_id, "sensor_failure") # Verify failsafe mode failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert failsafe_setpoint == 40.0, f"Setpoint not in failsafe mode: {failsafe_setpoint}" # Clear failsafe mode await watchdog.clear_failsafe_mode(station_id, pump_id) # Verify return to normal operation recovered_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert recovered_setpoint == normal_setpoint, "Setpoint did not recover after failsafe clearance" @pytest.mark.asyncio async def test_failsafe_mode_station_wide(self, failsafe_components): """Test station-wide failsafe mode activation.""" setpoint_manager = failsafe_components['setpoint_manager'] watchdog = failsafe_components['watchdog'] station_id = 'FAILSAFE_STATION_001' # Activate station-wide failsafe mode await watchdog.activate_failsafe_mode_station(station_id, "station_communication_loss") # Verify all pumps in station are in failsafe mode pumps = setpoint_manager.discovery.get_pumps(station_id) for pump in pumps: pump_id = pump['pump_id'] setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) # Should use default setpoint as failsafe default_setpoint = pump.get('default_setpoint_hz', 35.0) assert setpoint == default_setpoint, \ f"Pump {pump_id} not in failsafe mode: {setpoint} != {default_setpoint}" # Verify pumps in other stations are unaffected other_station_id = 'FAILSAFE_STATION_002' other_pumps = setpoint_manager.discovery.get_pumps(other_station_id) for pump in other_pumps: pump_id = pump['pump_id'] setpoint = setpoint_manager.get_current_setpoint(other_station_id, pump_id) assert setpoint != pump.get('default_setpoint', 35.0), \ f"Pump {pump_id} incorrectly in failsafe mode" # Clear station-wide failsafe mode await watchdog.clear_failsafe_mode_station(station_id) # Verify pumps return to normal operation for pump in pumps: pump_id = pump['pump_id'] setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert setpoint != pump.get('default_setpoint', 35.0), \ f"Pump {pump_id} not recovered from failsafe mode" @pytest.mark.asyncio async def test_failsafe_priority_over_optimization(self, failsafe_components): """Test that failsafe mode takes priority over optimization plans.""" setpoint_manager = failsafe_components['setpoint_manager'] watchdog = failsafe_components['watchdog'] station_id = 'FAILSAFE_STATION_002' pump_id = 'FAILSAFE_PUMP_003' # Get optimization-based setpoint optimization_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert optimization_setpoint is not None # Activate failsafe mode await watchdog.activate_failsafe_mode(station_id, pump_id, "optimizer_failure") # Verify failsafe mode overrides optimization failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert failsafe_setpoint == 38.0, f"Failsafe mode not overriding optimization: {failsafe_setpoint}" assert failsafe_setpoint != optimization_setpoint, "Failsafe mode should differ from optimization" @pytest.mark.asyncio async def test_emergency_stop_priority_over_failsafe(self, failsafe_components): """Test that emergency stop takes priority over failsafe mode.""" setpoint_manager = failsafe_components['setpoint_manager'] emergency_stop_manager = failsafe_components['emergency_stop_manager'] watchdog = failsafe_components['watchdog'] station_id = 'FAILSAFE_STATION_001' pump_id = 'FAILSAFE_PUMP_001' # Activate failsafe mode await watchdog.activate_failsafe_mode(station_id, pump_id, "test_priority") # Verify failsafe mode is active failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert failsafe_setpoint == 35.0, "Failsafe mode not active" # Activate emergency stop emergency_stop_manager.emergency_stop_pump(station_id, pump_id, "emergency_override_test") # Verify emergency stop overrides failsafe mode emergency_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert emergency_setpoint == 0.0, "Emergency stop not overriding failsafe mode" # Clear emergency stop emergency_stop_manager.clear_emergency_stop_pump(station_id, pump_id, "clearance_test") # Verify failsafe mode is restored restored_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert restored_setpoint == 35.0, "Failsafe mode not restored after emergency stop clearance" @pytest.mark.asyncio async def test_failsafe_mode_audit_logging(self, failsafe_components, db_client): """Test that failsafe mode activations are properly logged.""" setpoint_manager = failsafe_components['setpoint_manager'] watchdog = failsafe_components['watchdog'] audit_logger = failsafe_components['audit_logger'] station_id = 'FAILSAFE_STATION_001' pump_id = 'FAILSAFE_PUMP_002' # Activate failsafe mode await watchdog.activate_failsafe_mode(station_id, pump_id, "audit_test_reason") # Verify failsafe mode is active setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert setpoint == 40.0, "Failsafe mode not active for audit test" # Note: In a real implementation, we would verify that the audit logger # recorded the failsafe activation. For now, we'll just verify the # functional behavior. # Clear failsafe mode await watchdog.clear_failsafe_mode(station_id, pump_id) # Verify normal operation restored recovered_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id) assert recovered_setpoint != 40.0, "Failsafe mode not cleared"