CalejoControl/validate-deployment.sh

#!/bin/bash

# Calejo Control Adapter - Deployment Validation Script
# Validates that the deployment is healthy and ready for production

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
BASE_URL="http://localhost:8080"
DEPLOYMENT_DIR="/opt/calejo-control-adapter"

# Function to print colored output
print_status() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

print_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Function to check service health
check_service_health() {
    local service_name=$1
    local port=$2
    local endpoint=$3
    
    if curl -s -f "http://localhost:$port$endpoint" > /dev/null; then
        print_success "$service_name is healthy (port $port)"
        return 0
    else
        print_error "$service_name is not responding (port $port)"
        return 1
    fi
}

# Function to check container status
check_container_status() {
    print_status "Checking Docker container status..."
    
    if command -v docker-compose > /dev/null && [ -f "docker-compose.yml" ]; then
        cd $DEPLOYMENT_DIR
        
        if docker-compose ps | grep -q "Up"; then
            print_success "All Docker containers are running"
            docker-compose ps --format "table {{.Service}}\t{{.State}}\t{{.Ports}}"
            return 0
        else
            print_error "Some Docker containers are not running"
            docker-compose ps
            return 1
        fi
    else
        print_warning "Docker Compose not available or docker-compose.yml not found"
        return 0
    fi
}

# Function to check system resources
check_system_resources() {
    print_status "Checking system resources..."
    
    # Check disk space
    local disk_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    if [ $disk_usage -gt 90 ]; then
        print_error "Disk usage is high: ${disk_usage}%"
    elif [ $disk_usage -gt 80 ]; then
        print_warning "Disk usage is moderate: ${disk_usage}%"
    else
        print_success "Disk usage is normal: ${disk_usage}%"
    fi
    
    # Check memory
    local mem_info=$(free -h)
    print_status "Memory usage:"
    echo "$mem_info" | head -2
    
    # Check CPU load
    local load_avg=$(cat /proc/loadavg | awk '{print $1}')
    local cpu_cores=$(nproc)
    local load_percent=$(echo "scale=0; $load_avg * 100 / $cpu_cores" | bc)
    
    if [ $load_percent -gt 90 ]; then
        print_error "CPU load is high: ${load_avg} (${load_percent}% of capacity)"
    elif [ $load_percent -gt 70 ]; then
        print_warning "CPU load is moderate: ${load_avg} (${load_percent}% of capacity)"
    else
        print_success "CPU load is normal: ${load_avg} (${load_percent}% of capacity)"
    fi
}

# Function to check application endpoints
check_application_endpoints() {
    print_status "Checking application endpoints..."
    
    endpoints=(
        "/health"
        "/dashboard"
        "/api/v1/status"
        "/api/v1/dashboard/status"
        "/api/v1/dashboard/config"
        "/api/v1/dashboard/logs"
        "/api/v1/dashboard/actions"
    )
    
    all_healthy=true
    
    for endpoint in "${endpoints[@]}"; do
        if curl -s -f "$BASE_URL$endpoint" > /dev/null; then
            print_success "Endpoint $endpoint is accessible"
        else
            print_error "Endpoint $endpoint is not accessible"
            all_healthy=false
        fi
    done
    
    if $all_healthy; then
        print_success "All application endpoints are accessible"
        return 0
    else
        print_error "Some application endpoints are not accessible"
        return 1
    fi
}

# Function to check configuration
check_configuration() {
    print_status "Checking configuration..."
    
    # Check if configuration files exist
    config_files=(
        "$DEPLOYMENT_DIR/config/settings.py"
        "$DEPLOYMENT_DIR/docker-compose.yml"
        "$CONFIG_DIR/settings.py"
    )
    
    for config_file in "${config_files[@]}"; do
        if [ -f "$config_file" ]; then
            print_success "Configuration file exists: $config_file"
        else
            print_warning "Configuration file missing: $config_file"
        fi
    done
    
    # Check if configuration is valid
    if curl -s "$BASE_URL/api/v1/dashboard/config" | grep -q '"success":true'; then
        print_success "Configuration is valid and accessible"
        return 0
    else
        print_error "Configuration validation failed"
        return 1
    fi
}

# Function to check logs
check_logs() {
    print_status "Checking logs..."
    
    log_dirs=(
        "/var/log/calejo"
        "$DEPLOYMENT_DIR/logs"
    )
    
    for log_dir in "${log_dirs[@]}"; do
        if [ -d "$log_dir" ]; then
            local log_count=$(find "$log_dir" -name "*.log" -type f | wc -l)
            if [ $log_count -gt 0 ]; then
                print_success "Log directory contains $log_count log files: $log_dir"
                
                # Check for recent errors
                local error_count=$(find "$log_dir" -name "*.log" -type f -exec grep -l -i "error\|exception\|fail" {} \; | wc -l)
                if [ $error_count -gt 0 ]; then
                    print_warning "Found $error_count log files with errors"
                fi
            else
                print_warning "Log directory exists but contains no log files: $log_dir"
            fi
        else
            print_warning "Log directory does not exist: $log_dir"
        fi
    done
}

# Function to check security
check_security() {
    print_status "Checking security configuration..."
    
    # Check for default credentials warning
    if curl -s "$BASE_URL/api/v1/dashboard/config" | grep -q '"security_warning":true'; then
        print_warning "Security warning: Default credentials detected"
    else
        print_success "No security warnings detected"
    fi
    
    # Check if ports are properly exposed
    local open_ports=$(ss -tuln | grep -E ":(8080|4840|502|9090)" | wc -l)
    if [ $open_ports -gt 0 ]; then
        print_success "Required ports are open"
    else
        print_warning "Some required ports may not be open"
    fi
}

# Function to check backup configuration
check_backup_configuration() {
    print_status "Checking backup configuration..."
    
    if [ -f "$DEPLOYMENT_DIR/scripts/backup-full.sh" ]; then
        print_success "Backup script exists: $DEPLOYMENT_DIR/scripts/backup-full.sh"
        
        # Check if backup directory exists and is writable
        if [ -w "/var/backup/calejo" ]; then
            print_success "Backup directory is writable: /var/backup/calejo"
        else
            print_error "Backup directory is not writable: /var/backup/calejo"
        fi
    else
        print_error "Backup script not found"
    fi
}

# Function to generate validation report
generate_validation_report() {
    print_status "Generating validation report..."
    
    local report_file="/tmp/calejo-deployment-validation-$(date +%Y%m%d_%H%M%S).txt"
    
    cat > "$report_file" << EOF
Calejo Control Adapter - Deployment Validation Report
Generated: $(date)
System: $(hostname)

VALIDATION CHECKS:
EOF
    
    # Run checks and capture output
    {
        echo "1. System Resources:"
        check_system_resources 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "2. Container Status:"
        check_container_status 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "3. Application Endpoints:"
        check_application_endpoints 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "4. Configuration:"
        check_configuration 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "5. Logs:"
        check_logs 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "6. Security:"
        check_security 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "7. Backup Configuration:"
        check_backup_configuration 2>&1 | sed 's/^/   /'
        echo ""
        
        echo "SUMMARY:"
        echo "Deployment validation completed. Review any warnings or errors above."
        
    } >> "$report_file"
    
    print_success "Validation report generated: $report_file"
    
    # Display summary
    echo ""
    echo "=================================================="
    echo "            DEPLOYMENT VALIDATION SUMMARY"
    echo "=================================================="
    echo ""
    echo "📊 System Status:"
    check_system_resources | grep -E "(Disk usage|CPU load)"
    echo ""
    echo "🔧 Application Status:"
    check_application_endpoints > /dev/null 2>&1 && echo "   ✅ All endpoints accessible" || echo "   ❌ Some endpoints failed"
    echo ""
    echo "📋 Next Steps:"
    echo "   Review full report: $report_file"
    echo "   Address any warnings or errors"
    echo "   Run end-to-end tests: python test-e2e-deployment.py"
    echo ""
    echo "=================================================="
}

# Main validation function
main() {
    echo ""
    echo "🔍 Calejo Control Adapter - Deployment Validation"
    echo "=================================================="
    echo ""
    
    # Check if application is running
    if ! curl -s "$BASE_URL/health" > /dev/null 2>&1; then
        print_error "Application is not running or not accessible at $BASE_URL"
        echo ""
        echo "Please ensure the application is running before validation."
        echo "Start with: systemctl start calejo-control-adapter"
        exit 1
    fi
    
    # Run validation checks
    check_system_resources
    echo ""
    
    check_container_status
    echo ""
    
    check_application_endpoints
    echo ""
    
    check_configuration
    echo ""
    
    check_logs
    echo ""
    
    check_security
    echo ""
    
    check_backup_configuration
    echo ""
    
    # Generate comprehensive report
    generate_validation_report
    
    echo ""
    print_success "Deployment validation completed!"
}

# Run main function
main "$@"