# Disaster Recovery Drills Example

Complete procedure for testing disaster recovery capabilities and validating RTO/RPO targets.

## Disaster Recovery Strategy

```nickel
{
  disaster_recovery = {
    # Recovery Time Objective - 15 minutes
    rto_minutes = 15,

    # Recovery Point Objective - 5 minutes
    rpo_minutes = 5,

    # Primary and secondary regions
    primary_region = "us-east-1",
    secondary_region = "eu-west-1",

    # Replication configuration
    replication = {
      database_replication = "synchronous",
      backup_frequency = "5m",
      log_shipping = "enabled"
    },

    # Failover automation
    failover = {
      automatic = true,
      health_check_interval = "10s",
      failure_threshold = 2,
      automation_level = "full"
    }
  }
}
```

## Drill 1: Database Failover Test

### Objective
Verify PostgreSQL streaming replication and failover within RTO target of 15 minutes.

### Pre-Drill Checklist

```nushell
def pre-drill-checklist [] {
    print "Pre-DR Drill Checklist:"

    # Verify primary database health
    let primary_health = (
        provisioning database health \
            --database production-primary \
            --region us-east-1
    )
    print $"Primary DB Health: ($primary_health.status)"

    # Verify secondary database health
    let secondary_health = (
        provisioning database health \
            --database production-secondary \
            --region eu-west-1
    )
    print $"Secondary DB Health: ($secondary_health.status)"

    # Verify replication lag
    let replication_lag = (
        provisioning database replication-lag \
            --primary production-primary \
            --replica production-secondary
    )
    print $"Replication Lag: ($replication_lag.lag_seconds)s (Target: <60s)"

    if $replication_lag.lag_seconds > 60 {
        print "✗ Replication lag exceeds target, abort drill"
        return 1
    }

    # Verify backup availability
    let latest_backup = (provisioning backup list --limit 1 --database production-primary)
    print $"Latest Backup: ($latest_backup.0.created_at)"

    print "✓ Pre-drill checks passed"
    return 0
}
```

### Drill Execution

```nushell
def drill-database-failover [] {
    let start_time = (date now)
    print $"[($start_time)] Starting database failover drill..."

    # Step 1: Simulate primary database failure
    print "[Step 1] Simulating primary database failure..."
    let failure_result = (
        provisioning database simulate-failure \
            --database production-primary \
            --region us-east-1 \
            --recovery-in-minutes 30  # Simulate 30-min recovery
    )

    # Step 2: Detect failure (automatic health check)
    print "[Step 2] Waiting for failure detection (target: <60s)..."
    let detection_time = 0
    let detected = false

    while $detection_time < 120 {
        let health = (
            provisioning database health \
                --database production-primary \
                --region us-east-1
        )

        if $health.status == "failed" {
            let elapsed = ((date now) - $start_time)
            print $"✓ Failure detected in ($elapsed.ms)ms (Target: <60s)"
            $detected = true
            break
        }

        sleep 5s
        $detection_time = $detection_time + 5
    }

    if not $detected {
        print "✗ Failure not detected within timeout"
        return 1
    }

    # Step 3: Promote secondary
    print "[Step 3] Promoting secondary to primary..."
    let promote_start = (date now)

    let promote_result = (
        provisioning database promote-replica \
            --replica production-secondary \
            --region eu-west-1
    )

    let promote_time = ((date now) - $promote_start)
    print $"✓ Secondary promoted in ($promote_time.ms)ms"

    # Step 4: Update application DNS
    print "[Step 4] Updating application DNS..."
    let dns_start = (date now)

    provisioning dns update \
        --record "db.production.internal" \
        --target "production-secondary.eu-west-1.rds.amazonaws.com" \
        --ttl 0  # Immediate DNS propagation

    let dns_time = ((date now) - $dns_start)
    print $"✓ DNS updated in ($dns_time.ms)ms"

    # Step 5: Verify application connectivity
    print "[Step 5] Verifying application connectivity..."
    let verify_start = (date now)

    let app_health = (
        provisioning app health-check \
            --service api \
            --retries 10 \
            --timeout 30
    )

    if $app_health.status != "healthy" {
        print "✗ Application unable to connect to new primary"
        return 1
    }

    let verify_time = ((date now) - $verify_start)

    # Step 6: Validate data consistency
    print "[Step 6] Validating data consistency..."
    let consistency_result = (
        provisioning database verify-consistency \
            --database production-secondary \
            --checksums full
    )

    if $consistency_result.status != "consistent" {
        print "✗ Data consistency check failed"
        return 1
    }

    # Step 7: Measure total RTO
    let total_rto = (date now) - $start_time
    print ""
    print "╔════════════════════════════════════════╗"
    print "║      DRILL RESULTS                     ║"
    print "╚════════════════════════════════════════╝"
    print $"Total RTO: ($total_rto.ms)ms"
    print $"Target RTO: 15 minutes (900,000ms)"
    print $"Status: PASSED ✓" | if $total_rto.ms < 900000 { . } else { $"Status: FAILED ✗" }

    print ""
    print "Detailed Timeline:"
    print $"  Failure Detection: 0ms - ($detection_time)ms"
    print $"  Replica Promotion: ($promote_time.ms)ms"
    print $"  DNS Update: ($dns_time.ms)ms"
    print $"  Application Reconnect: ($verify_time.ms)ms"

    # Step 8: Cleanup - restore original configuration
    print ""
    print "[Step 8] Cleanup: Restoring original configuration..."
    provisioning database recovery-complete \
        --database production-primary \
        --region us-east-1

    # Re-establish replication with new secondary
    provisioning database replicate \
        --primary production-secondary \
        --secondary production-primary \
        --replication-type streaming

    print "✓ Drill cleanup complete"

    return 0
}
```

## Drill 2: Complete Application Failover

### Objective
Test end-to-end application failover including database, cache, and load balancer reconfiguration.

```nushell
def drill-complete-failover [] {
    let start_time = (date now)
    print $"[($start_time)] Starting complete application failover drill..."

    # 1. Disable primary region (all services)
    print "Phase 1: Disabling primary region (us-east-1)..."
    provisioning region disable --region us-east-1 --simulate-failure true

    # 2. Monitor failover automation
    print "Phase 2: Monitoring automatic failover..."
    let failover_steps = [
        "health_check_failure",
        "database_promotion",
        "cache_invalidation",
        "dns_update",
        "load_balancer_reconfiguration",
        "application_health_verification"
    ]

    for step in $failover_steps {
        let step_result = (
            provisioning failover monitor \
                --step $step \
                --timeout 60
        )

        let elapsed = ((date now) - $start_time)
        print $"✓ ($step): ($step_result.duration_ms)ms [Total: ($elapsed.ms)ms]"
    }

    # 3. Verify application in secondary region
    print "Phase 3: Verifying application in secondary region (eu-west-1)..."
    let app_tests = [
        "authentication",
        "data_retrieval",
        "write_operations",
        "cache_operations",
        "external_api_calls"
    ]

    for test in $app_tests {
        let result = (
            provisioning test run --test-name $test --environment production
        )

        if $result.status == "passed" {
            print $"✓ ($test): PASSED"
        } else {
            print $"✗ ($test): FAILED"
            return 1
        }
    }

    # 4. Measure metrics
    print "Phase 4: Performance metrics in secondary region..."
    let metrics = (
        provisioning metrics get \
            --region eu-west-1 \
            --metrics ["latency_p99", "error_rate", "throughput"] \
            --window 5m
    )

    print $"Latency (P99): ($metrics.latency_p99)ms (Target: <200ms)"
    print $"Error Rate: ($metrics.error_rate)% (Target: <0.1%)"
    print $"Throughput: ($metrics.throughput) req/s (Target: >1000)"

    # 5. Complete timeline
    let total_time = (date now) - $start_time
    print ""
    print $"✓ Complete Failover RTO: ($total_time.ms)ms"
    print $"✓ All systems operational in secondary region"

    # 6. Cleanup
    print ""
    print "Cleanup: Re-enabling primary region..."
    provisioning region enable --region us-east-1
    provisioning replication resync \
        --primary eu-west-1 \
        --secondary us-east-1

    print "✓ Drill complete, original configuration restored"
}
```

## Drill 3: Backup Recovery Test

### Objective
Verify backup integrity and restoration capability within RPO target.

```nushell
def drill-backup-recovery [] {
    print "Starting backup recovery drill..."

    # Get oldest backup within RPO window (5 minutes)
    let target_time = ((date now) | date add -5m)
    let backups = (
        provisioning backup list \
            --database production-primary \
            --before $target_time \
            --limit 5
    )

    # Pick random backup for recovery test
    let backup_to_test = ($backups | shuffle | first)
    print $"Testing backup from: ($backup_to_test.created_at)"

    # Create test database from backup
    print "Restoring backup to test environment..."
    let restore_result = (
        provisioning backup restore \
            --backup-id $backup_to_test.id \
            --target-database test-recovery \
            --environment test
    )

    # Verify data integrity
    print "Verifying data integrity..."
    let verification = (
        provisioning database verify \
            --database test-recovery \
            --checksums full \
            --sampling-rate 0.1  # Check 10% of data
    )

    if $verification.inconsistencies == 0 {
        print "✓ Backup integrity verified (0 inconsistencies)"
    } else {
        print $"✗ Backup corruption detected: ($verification.inconsistencies) issues"
        return 1
    }

    # Cleanup
    provisioning database delete --database test-recovery --environment test
    print "✓ Backup recovery drill passed"
}
```

## Scheduled Drill Calendar

```nushell
# Schedule regular DR drills
def schedule-dr-drills [] {
    provisioning schedule create \
        --name "Monthly Database Failover Drill" \
        --schedule "0 2 15 * *" \  # 2 AM on 15th of each month
        --command "drill-database-failover" \
        --notifications {
            slack = "#infrastructure",
            email = "[ops@example.com](mailto:ops@example.com)"
        }

    provisioning schedule create \
        --name "Quarterly Complete Failover Drill" \
        --schedule "0 3 1 1,4,7,10 *" \  # 3 AM on first of each quarter
        --command "drill-complete-failover" \
        --duration-minutes = 120

    provisioning schedule create \
        --name "Weekly Backup Recovery Test" \
        --schedule "0 4 * * 0" \  # 4 AM every Sunday
        --command "drill-backup-recovery"
}
```

## Drill Report Template

```text
╔═══════════════════════════════════════════════════════════════╗
║          DISASTER RECOVERY DRILL REPORT                       ║
╚═══════════════════════════════════════════════════════════════╝

Drill Date: 2025-01-16
Drill Type: Complete Application Failover
Primary Region: us-east-1 → Secondary Region: eu-west-1

OBJECTIVES & RESULTS:
✓ Verify RTO < 15 minutes
✓ Verify RPO < 5 minutes
✓ Validate data consistency
✓ Test application failover automation

RTO METRICS:
Failure Detection: 23 seconds (Target: < 60s)
Database Promotion: 45 seconds (Target: < 120s)
DNS Update: 8 seconds (Target: < 30s)
Total RTO: 5 minutes 12 seconds (Target: < 15 minutes)

RPO METRICS:
Replication Lag at Failover: 3.2 seconds (Target: < 5 minutes)
Data Loss: 0 records (Target: 0)
Backup Freshness: 47 seconds old (Target: < 5 minutes)

APPLICATION TESTS:
✓ Authentication: PASSED
✓ Data Retrieval: PASSED
✓ Write Operations: PASSED
✓ Cache Operations: PASSED
✓ External API Integration: PASSED

PERFORMANCE:
Latency (P99): 145ms (Target: < 200ms)
Error Rate: 0.03% (Target: < 0.1%)
Throughput: 1,234 req/s (Target: > 1000)

ISSUES IDENTIFIED:
None

RECOMMENDATIONS:
- Latency in secondary region excellent, no optimization needed
- Consider quarterly instead of monthly database failover drills
- All automation working as expected

SIGN-OFF:
Conducted By: Infrastructure Team
Date: 2025-01-16
Status: PASSED ✓
```

## See Also

- [Disaster Recovery Guide](../guides/disaster-recovery.md)
- [Backup Strategies](../operations/backup-strategies.md)
- [Monitoring and Alerting](../operations/monitoring.md)
