provisioning/docs/book/user/examples/quick-start/local-development/verify.sh

#!/usr/bin/env bash
# Local Development Environment Verification Script
# This script verifies that the local development environment is working correctly

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SETTINGS_FILE="${SCRIPT_DIR}/settings.k"

# Function to print colored output
print_status() {
    local color=$1
    local message=$2
    echo -e "${color}${message}${NC}"
}

print_step() {
    print_status $BLUE "🔄 $1"
}

print_success() {
    print_status $GREEN "✅ $1"
}

print_warning() {
    print_status $YELLOW "⚠️  $1"
}

print_error() {
    print_status $RED "❌ $1"
}

# Verification results tracking
declare -A verification_results
total_checks=0
passed_checks=0

# Function to run a verification check
run_check() {
    local check_name=$1
    local check_command=$2
    local expected_result=${3:-0}

    ((total_checks++))

    print_step "Checking: $check_name"

    if eval "$check_command" &>/dev/null; then
        if [ $? -eq $expected_result ]; then
            print_success "$check_name"
            verification_results["$check_name"]="PASS"
            ((passed_checks++))
            return 0
        else
            print_error "$check_name - Command succeeded but returned unexpected result"
            verification_results["$check_name"]="FAIL"
            return 1
        fi
    else
        print_error "$check_name - Command failed"
        verification_results["$check_name"]="FAIL"
        return 1
    fi
}

# Function to check if servers exist
check_servers_exist() {
    print_step "Verifying servers exist..."

    local servers_output
    if servers_output=$(provisioning server list --settings "$SETTINGS_FILE" --out json 2>/dev/null); then
        local web_server_count
        local db_server_count

        web_server_count=$(echo "$servers_output" | jq -r '.[] | select(.name == "web-dev-01") | .name' | wc -l)
        db_server_count=$(echo "$servers_output" | jq -r '.[] | select(.name == "db-dev-01") | .name' | wc -l)

        if [ "$web_server_count" -eq 1 ] && [ "$db_server_count" -eq 1 ]; then
            print_success "Both servers exist (web-dev-01, db-dev-01)"
            return 0
        else
            print_error "Missing servers - found web: $web_server_count, db: $db_server_count"
            return 1
        fi
    else
        print_error "Could not retrieve server list"
        return 1
    fi
}

# Function to check server connectivity
check_server_connectivity() {
    print_step "Testing server SSH connectivity..."

    local connectivity_issues=0

    # Test web server SSH
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "echo 'SSH test successful'" &>/dev/null; then
        print_success "SSH to web-dev-01 working"
    else
        print_error "SSH to web-dev-01 failed"
        ((connectivity_issues++))
    fi

    # Test database server SSH
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "echo 'SSH test successful'" &>/dev/null; then
        print_success "SSH to db-dev-01 working"
    else
        print_error "SSH to db-dev-01 failed"
        ((connectivity_issues++))
    fi

    if [ $connectivity_issues -eq 0 ]; then
        return 0
    else
        return 1
    fi
}

# Function to check installed services
check_installed_services() {
    print_step "Verifying installed services..."

    local services_output
    if services_output=$(provisioning taskserv list --settings "$SETTINGS_FILE" --installed --out json 2>/dev/null); then
        print_success "Retrieved installed services list"

        # Show installed services
        echo "$services_output" | jq -r '.[] | "  • \(.name) on \(.servers | join(", "))"'

        return 0
    else
        print_error "Could not retrieve installed services"
        return 1
    fi
}

# Function to check web server functionality
check_web_server() {
    print_step "Testing web server functionality..."

    # Check if Nginx is running
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "systemctl is-active nginx" 2>/dev/null | grep -q "active"; then
        print_success "Nginx service is running"
    else
        print_warning "Nginx service not running or not installed"
    fi

    # Check if port 80 is listening
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "ss -tlnp | grep :80" &>/dev/null; then
        print_success "Web server listening on port 80"
    else
        print_warning "Web server not listening on port 80"
    fi

    # Try to make HTTP request
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "curl -s -o /dev/null -w '%{http_code}' http://localhost" 2>/dev/null | grep -q "200\|403\|404"; then
        print_success "Web server responding to HTTP requests"
        return 0
    else
        print_warning "Web server not responding to HTTP requests"
        return 1
    fi
}

# Function to check database server functionality
check_database_server() {
    print_step "Testing database server functionality..."

    # Check if PostgreSQL is running
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "systemctl is-active postgresql" 2>/dev/null | grep -q "active"; then
        print_success "PostgreSQL service is running"
    else
        print_warning "PostgreSQL service not running or not installed"
    fi

    # Check if PostgreSQL port is listening
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "ss -tlnp | grep :5432" &>/dev/null; then
        print_success "PostgreSQL listening on port 5432"
    else
        print_warning "PostgreSQL not listening on port 5432"
    fi

    # Test database connection
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "pg_isready -h localhost -p 5432" &>/dev/null; then
        print_success "PostgreSQL database is ready"
    else
        print_warning "PostgreSQL database not ready"
    fi

    # Check if Redis is running
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "systemctl is-active redis" 2>/dev/null | grep -q "active"; then
        print_success "Redis service is running"
    elif provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "systemctl is-active redis-server" 2>/dev/null | grep -q "active"; then
        print_success "Redis service is running"
    else
        print_warning "Redis service not running or not installed"
    fi

    # Test Redis connection
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "redis-cli ping" 2>/dev/null | grep -q "PONG"; then
        print_success "Redis is responding"
        return 0
    else
        print_warning "Redis not responding"
        return 1
    fi
}

# Function to check network connectivity between servers
check_network_connectivity() {
    print_step "Testing network connectivity between servers..."

    # Test web server can reach database server
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "ping -c 1 192.168.100.11" &>/dev/null; then
        print_success "Web server can ping database server"
    else
        print_warning "Web server cannot ping database server"
    fi

    # Test database server can reach web server
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "ping -c 1 192.168.100.10" &>/dev/null; then
        print_success "Database server can ping web server"
        return 0
    else
        print_warning "Database server cannot ping web server"
        return 1
    fi
}

# Function to check development tools
check_development_tools() {
    print_step "Checking development tools..."

    # Check Node.js on web server
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "node --version" &>/dev/null; then
        local node_version
        node_version=$(provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "node --version" 2>/dev/null)
        print_success "Node.js installed: $node_version"
    else
        print_warning "Node.js not installed on web server"
    fi

    # Check npm on web server
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "npm --version" &>/dev/null; then
        local npm_version
        npm_version=$(provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "npm --version" 2>/dev/null)
        print_success "npm installed: $npm_version"
    else
        print_warning "npm not installed on web server"
    fi

    # Check Git on both servers
    local git_issues=0

    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "git --version" &>/dev/null; then
        print_success "Git installed on web server"
    else
        print_warning "Git not installed on web server"
        ((git_issues++))
    fi

    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "git --version" &>/dev/null; then
        print_success "Git installed on database server"
    else
        print_warning "Git not installed on database server"
        ((git_issues++))
    fi

    return $git_issues
}

# Function to perform resource checks
check_resources() {
    print_step "Checking resource usage..."

    # Check memory usage on both servers
    print_step "Memory usage:"
    provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "echo 'Web server:' && free -h | grep Mem" 2>/dev/null || true
    provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "echo 'Database server:' && free -h | grep Mem" 2>/dev/null || true

    # Check disk usage
    print_step "Disk usage:"
    provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "echo 'Web server:' && df -h | head -2" 2>/dev/null || true
    provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "echo 'Database server:' && df -h | head -2" 2>/dev/null || true

    print_success "Resource check completed"
    return 0
}

# Function to run functional tests
run_functional_tests() {
    print_step "Running functional tests..."

    # Test 1: Create a test file on web server
    if provisioning server ssh web-dev-01 --settings "$SETTINGS_FILE" --command "echo 'Hello World' > /tmp/test.txt && cat /tmp/test.txt" &>/dev/null; then
        print_success "File operations working on web server"
    else
        print_warning "File operations failed on web server"
    fi

    # Test 2: Test database creation
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "createdb test_verification 2>/dev/null && dropdb test_verification" &>/dev/null; then
        print_success "Database operations working"
    else
        print_warning "Database operations failed"
    fi

    # Test 3: Test Redis operations
    if provisioning server ssh db-dev-01 --settings "$SETTINGS_FILE" --command "redis-cli set test_key 'test_value' && redis-cli get test_key && redis-cli del test_key" &>/dev/null; then
        print_success "Redis operations working"
        return 0
    else
        print_warning "Redis operations failed"
        return 1
    fi
}

# Function to show verification summary
show_verification_summary() {
    echo
    print_status $BLUE "📊 Verification Summary"
    print_status $BLUE "======================="

    echo "Total checks performed: $total_checks"
    echo "Checks passed: $passed_checks"
    echo "Success rate: $(( passed_checks * 100 / total_checks ))%"

    echo
    print_status $BLUE "Detailed results:"

    for check in "${!verification_results[@]}"; do
        local result="${verification_results[$check]}"
        if [ "$result" = "PASS" ]; then
            print_success "$check"
        else
            print_error "$check"
        fi
    done

    echo
    if [ $passed_checks -eq $total_checks ]; then
        print_success "All verifications passed! Your local development environment is ready."
        print_status $BLUE "🚀 You can now start developing:"
        echo "   • Deploy your applications to the web server"
        echo "   • Use the database for your data storage"
        echo "   • SSH to servers for manual configuration"
        echo "   • Check README.md for more usage examples"
    elif [ $passed_checks -gt $(( total_checks / 2 )) ]; then
        print_warning "Most verifications passed, but some issues were found."
        print_status $BLUE "💡 Your environment is mostly functional, but consider fixing the warnings above."
    else
        print_error "Multiple verification failures detected."
        print_status $BLUE "🔧 Please check the errors above and refer to the troubleshooting guide."
    fi
}

# Function to show quick start guide
show_quick_start() {
    echo
    print_status $BLUE "🚀 Quick Start Guide"
    print_status $BLUE "==================="
    echo
    echo "Now that your environment is verified, here are some things you can try:"
    echo
    print_status $GREEN "1. Deploy a simple web page:"
    echo "   provisioning server ssh web-dev-01 --settings settings.k --command \\"
    echo "     \"echo '<h1>Hello World</h1>' | sudo tee /var/www/html/index.html\""
    echo
    print_status $GREEN "2. Create a database:"
    echo "   provisioning server ssh db-dev-01 --settings settings.k --command \\"
    echo "     \"createdb myapp_dev\""
    echo
    print_status $GREEN "3. Test the web server:"
    echo "   provisioning server ssh web-dev-01 --settings settings.k --command \\"
    echo "     \"curl http://localhost\""
    echo
    print_status $GREEN "4. Connect to PostgreSQL:"
    echo "   provisioning server ssh db-dev-01 --settings settings.k --command \\"
    echo "     \"psql -U postgres myapp_dev\""
    echo
    print_status $BLUE "📚 Next steps:"
    echo "   • Explore other examples in ../../"
    echo "   • Read the Infrastructure Management guide"
    echo "   • Try deploying a real application"
    echo "   • When done, run ./cleanup.sh to clean up resources"
    echo
}

# Main verification function
main() {
    local start_time=$(date +%s)

    print_status $BLUE "🔍 Local Development Environment Verification"
    print_status $BLUE "============================================="
    echo

    # Set environment
    export PROVISIONING_ENV=development

    # Run all verification checks
    echo "Running comprehensive verification checks..."
    echo

    # Core functionality checks
    if check_servers_exist; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Servers Exist"]="PASS"
    else
        ((total_checks++))
        verification_results["Servers Exist"]="FAIL"
    fi

    if check_server_connectivity; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["SSH Connectivity"]="PASS"
    else
        ((total_checks++))
        verification_results["SSH Connectivity"]="FAIL"
    fi

    if check_installed_services; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Services Installation"]="PASS"
    else
        ((total_checks++))
        verification_results["Services Installation"]="FAIL"
    fi

    if check_web_server; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Web Server Functionality"]="PASS"
    else
        ((total_checks++))
        verification_results["Web Server Functionality"]="FAIL"
    fi

    if check_database_server; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Database Server Functionality"]="PASS"
    else
        ((total_checks++))
        verification_results["Database Server Functionality"]="FAIL"
    fi

    if check_network_connectivity; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Network Connectivity"]="PASS"
    else
        ((total_checks++))
        verification_results["Network Connectivity"]="FAIL"
    fi

    if check_development_tools; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Development Tools"]="PASS"
    else
        ((total_checks++))
        verification_results["Development Tools"]="FAIL"
    fi

    if check_resources; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Resource Usage"]="PASS"
    else
        ((total_checks++))
        verification_results["Resource Usage"]="FAIL"
    fi

    if run_functional_tests; then
        ((total_checks++))
        ((passed_checks++))
        verification_results["Functional Tests"]="PASS"
    else
        ((total_checks++))
        verification_results["Functional Tests"]="FAIL"
    fi

    local end_time=$(date +%s)
    local duration=$((end_time - start_time))

    # Show results
    show_verification_summary
    show_quick_start

    echo
    print_success "Verification completed in ${duration} seconds"

    # Return appropriate exit code
    if [ $passed_checks -eq $total_checks ]; then
        exit 0
    elif [ $passed_checks -gt $(( total_checks / 2 )) ]; then
        exit 1
    else
        exit 2
    fi
}

# Error handling for verification script
trap 'print_error "Verification script encountered an error"; exit 3' ERR

# Check if jq is available (needed for JSON parsing)
if ! command -v jq &> /dev/null; then
    print_warning "jq not found. Some verifications may be limited."
    print_status $BLUE "Install jq for better verification results: apt install jq / brew install jq"
fi

# Run main function
main "$@"