CalejoControl/tests/integration/test_failsafe_operations.py

"""
Integration tests for failsafe mode operations.

Tests failsafe mode activation, operation, and recovery scenarios.
"""

import pytest
import pytest_asyncio
from unittest.mock import Mock, AsyncMock
import asyncio

from src.database.flexible_client import FlexibleDatabaseClient
from src.core.auto_discovery import AutoDiscovery
from src.core.safety import SafetyLimitEnforcer
from src.core.emergency_stop import EmergencyStopManager
from src.core.setpoint_manager import SetpointManager
from src.core.security import SecurityManager
from src.core.compliance_audit import ComplianceAuditLogger
from src.monitoring.watchdog import DatabaseWatchdog


class TestFailsafeOperations:
    """Test failsafe mode operations and scenarios."""

    @pytest_asyncio.fixture
    async def db_client(self):
        """Create database client with failsafe test data."""
        client = FlexibleDatabaseClient(
            database_url="sqlite:///:memory:",
            pool_size=5,
            max_overflow=10,
            pool_timeout=30
        )

        await client.connect()
        client.create_tables()

        # Insert failsafe test data
        self._insert_failsafe_test_data(client)
        return client

    def _insert_failsafe_test_data(self, db_client):
        """Insert comprehensive failsafe test data."""
        # Insert stations
        db_client.execute(
            """INSERT INTO pump_stations (station_id, station_name, location) VALUES
               ('FAILSAFE_STATION_001', 'Failsafe Test Station 1', 'Test Location A'),
               ('FAILSAFE_STATION_002', 'Failsafe Test Station 2', 'Test Location B')"""
        )

        # Insert pumps with different failsafe configurations
        db_client.execute(
            """INSERT INTO pumps (station_id, pump_id, pump_name, control_type, 
                   min_speed_hz, max_speed_hz, default_setpoint_hz) VALUES
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', 'Failsafe Pump 1', 'DIRECT_SPEED', 20.0, 60.0, 35.0),
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', 'Failsafe Pump 2', 'LEVEL_CONTROLLED', 25.0, 55.0, 40.0),
               ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', 'Failsafe Pump 3', 'POWER_CONTROLLED', 30.0, 50.0, 38.0)"""
        )

        # Insert safety limits
        db_client.execute(
            """INSERT INTO pump_safety_limits (station_id, pump_id, hard_min_speed_hz, hard_max_speed_hz, 
                   hard_min_level_m, hard_max_level_m, emergency_stop_level_m, dry_run_protection_level_m, 
                   hard_max_power_kw, hard_max_flow_m3h, max_starts_per_hour, min_run_time_seconds, 
                   max_continuous_run_hours, max_speed_change_hz_per_min, set_by, approved_by) VALUES
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', 20.0, 60.0, 1.0, 5.0, 0.5, 1.5, 15.0, 200.0, 6, 300, 24, 10.0, 'admin', 'admin'),
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', 25.0, 55.0, 1.5, 4.5, 1.0, 2.0, 12.0, 180.0, 6, 300, 24, 10.0, 'admin', 'admin'),
               ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', 30.0, 50.0, 2.0, 4.0, 1.5, 2.5, 10.0, 160.0, 6, 300, 24, 10.0, 'admin', 'admin')"""
        )

        # Insert optimization plans
        import datetime
        now = datetime.datetime.now()
        db_client.execute(
            f"""INSERT INTO pump_plans (station_id, pump_id, interval_start, interval_end, 
                   suggested_speed_hz, target_level_m, target_power_kw, plan_status) VALUES
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_001', '{now}', '{now + datetime.timedelta(hours=1)}', 42.5, 3.0, 12.0, 'ACTIVE'),
               ('FAILSAFE_STATION_001', 'FAILSAFE_PUMP_002', '{now}', '{now + datetime.timedelta(hours=1)}', 38.0, 2.5, 10.0, 'ACTIVE'),
               ('FAILSAFE_STATION_002', 'FAILSAFE_PUMP_003', '{now}', '{now + datetime.timedelta(hours=1)}', 36.0, 3.5, 8.0, 'ACTIVE')"""
        )

    @pytest_asyncio.fixture
    async def failsafe_components(self, db_client):
        """Create failsafe-related components for testing."""
        # Create auto discovery
        auto_discovery = AutoDiscovery(db_client)
        await auto_discovery.discover()

        # Create safety components
        safety_enforcer = SafetyLimitEnforcer(db_client)
        emergency_stop_manager = EmergencyStopManager(db_client)

        # Load safety limits
        await safety_enforcer.load_safety_limits()

        # Create mock alert manager for watchdog
        mock_alert_manager = Mock()
        watchdog = DatabaseWatchdog(db_client, mock_alert_manager)

        # Create setpoint manager
        setpoint_manager = SetpointManager(
            discovery=auto_discovery,
            db_client=db_client,
            safety_enforcer=safety_enforcer,
            emergency_stop_manager=emergency_stop_manager,
            watchdog=watchdog
        )

        # Create security components
        security_manager = SecurityManager()
        audit_logger = ComplianceAuditLogger(db_client)

        return {
            'auto_discovery': auto_discovery,
            'safety_enforcer': safety_enforcer,
            'emergency_stop_manager': emergency_stop_manager,
            'setpoint_manager': setpoint_manager,
            'security_manager': security_manager,
            'audit_logger': audit_logger,
            'watchdog': watchdog
        }

    @pytest.mark.asyncio
    async def test_failsafe_mode_activation(self, failsafe_components):
        """Test failsafe mode activation for individual pumps."""
        setpoint_manager = failsafe_components['setpoint_manager']
        watchdog = failsafe_components['watchdog']

        station_id = 'FAILSAFE_STATION_001'
        pump_id = 'FAILSAFE_PUMP_001'

        # Get normal setpoint
        normal_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert normal_setpoint is not None

        # Activate failsafe mode
        await watchdog.activate_failsafe_mode(station_id, pump_id, "communication_loss")

        # Verify setpoint switches to failsafe mode
        failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        
        # In failsafe mode, should use default setpoint (35.0) as fallback
        # since we don't have a specific failsafe_setpoint_hz in the database
        assert failsafe_setpoint == 35.0, f"Setpoint not in failsafe mode: {failsafe_setpoint}"

    @pytest.mark.asyncio
    async def test_failsafe_mode_recovery(self, failsafe_components):
        """Test failsafe mode recovery and return to normal operation."""
        setpoint_manager = failsafe_components['setpoint_manager']
        watchdog = failsafe_components['watchdog']

        station_id = 'FAILSAFE_STATION_001'
        pump_id = 'FAILSAFE_PUMP_002'

        # Get normal setpoint
        normal_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)

        # Activate failsafe mode
        await watchdog.activate_failsafe_mode(station_id, pump_id, "sensor_failure")

        # Verify failsafe mode
        failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert failsafe_setpoint == 40.0, f"Setpoint not in failsafe mode: {failsafe_setpoint}"

        # Clear failsafe mode
        await watchdog.clear_failsafe_mode(station_id, pump_id)

        # Verify return to normal operation
        recovered_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert recovered_setpoint == normal_setpoint, "Setpoint did not recover after failsafe clearance"

    @pytest.mark.asyncio
    async def test_failsafe_mode_station_wide(self, failsafe_components):
        """Test station-wide failsafe mode activation."""
        setpoint_manager = failsafe_components['setpoint_manager']
        watchdog = failsafe_components['watchdog']

        station_id = 'FAILSAFE_STATION_001'

        # Activate station-wide failsafe mode
        await watchdog.activate_failsafe_mode_station(station_id, "station_communication_loss")

        # Verify all pumps in station are in failsafe mode
        pumps = setpoint_manager.discovery.get_pumps(station_id)
        for pump in pumps:
            pump_id = pump['pump_id']
            setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
            
            # Should use default setpoint as failsafe
            default_setpoint = pump.get('default_setpoint_hz', 35.0)
            assert setpoint == default_setpoint, \
                f"Pump {pump_id} not in failsafe mode: {setpoint} != {default_setpoint}"

        # Verify pumps in other stations are unaffected
        other_station_id = 'FAILSAFE_STATION_002'
        other_pumps = setpoint_manager.discovery.get_pumps(other_station_id)
        for pump in other_pumps:
            pump_id = pump['pump_id']
            setpoint = setpoint_manager.get_current_setpoint(other_station_id, pump_id)
            assert setpoint != pump.get('default_setpoint', 35.0), \
                f"Pump {pump_id} incorrectly in failsafe mode"

        # Clear station-wide failsafe mode
        await watchdog.clear_failsafe_mode_station(station_id)

        # Verify pumps return to normal operation
        for pump in pumps:
            pump_id = pump['pump_id']
            setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
            assert setpoint != pump.get('default_setpoint', 35.0), \
                f"Pump {pump_id} not recovered from failsafe mode"

    @pytest.mark.asyncio
    async def test_failsafe_priority_over_optimization(self, failsafe_components):
        """Test that failsafe mode takes priority over optimization plans."""
        setpoint_manager = failsafe_components['setpoint_manager']
        watchdog = failsafe_components['watchdog']

        station_id = 'FAILSAFE_STATION_002'
        pump_id = 'FAILSAFE_PUMP_003'

        # Get optimization-based setpoint
        optimization_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert optimization_setpoint is not None

        # Activate failsafe mode
        await watchdog.activate_failsafe_mode(station_id, pump_id, "optimizer_failure")

        # Verify failsafe mode overrides optimization
        failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert failsafe_setpoint == 38.0, f"Failsafe mode not overriding optimization: {failsafe_setpoint}"
        assert failsafe_setpoint != optimization_setpoint, "Failsafe mode should differ from optimization"

    @pytest.mark.asyncio
    async def test_emergency_stop_priority_over_failsafe(self, failsafe_components):
        """Test that emergency stop takes priority over failsafe mode."""
        setpoint_manager = failsafe_components['setpoint_manager']
        emergency_stop_manager = failsafe_components['emergency_stop_manager']
        watchdog = failsafe_components['watchdog']

        station_id = 'FAILSAFE_STATION_001'
        pump_id = 'FAILSAFE_PUMP_001'

        # Activate failsafe mode
        await watchdog.activate_failsafe_mode(station_id, pump_id, "test_priority")

        # Verify failsafe mode is active
        failsafe_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert failsafe_setpoint == 35.0, "Failsafe mode not active"

        # Activate emergency stop
        emergency_stop_manager.emergency_stop_pump(station_id, pump_id, "emergency_override_test")

        # Verify emergency stop overrides failsafe mode
        emergency_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert emergency_setpoint == 0.0, "Emergency stop not overriding failsafe mode"

        # Clear emergency stop
        emergency_stop_manager.clear_emergency_stop_pump(station_id, pump_id, "clearance_test")

        # Verify failsafe mode is restored
        restored_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert restored_setpoint == 35.0, "Failsafe mode not restored after emergency stop clearance"

    @pytest.mark.asyncio
    async def test_failsafe_mode_audit_logging(self, failsafe_components, db_client):
        """Test that failsafe mode activations are properly logged."""
        setpoint_manager = failsafe_components['setpoint_manager']
        watchdog = failsafe_components['watchdog']
        audit_logger = failsafe_components['audit_logger']

        station_id = 'FAILSAFE_STATION_001'
        pump_id = 'FAILSAFE_PUMP_002'

        # Activate failsafe mode
        await watchdog.activate_failsafe_mode(station_id, pump_id, "audit_test_reason")

        # Verify failsafe mode is active
        setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert setpoint == 40.0, "Failsafe mode not active for audit test"

        # Note: In a real implementation, we would verify that the audit logger
        # recorded the failsafe activation. For now, we'll just verify the
        # functional behavior.

        # Clear failsafe mode
        await watchdog.clear_failsafe_mode(station_id, pump_id)

        # Verify normal operation restored
        recovered_setpoint = setpoint_manager.get_current_setpoint(station_id, pump_id)
        assert recovered_setpoint != 40.0, "Failsafe mode not cleared"