Vapora/provisioning/.github/workflows/rollback.yml

name: Rollback Deployment

on:
  workflow_dispatch:
    inputs:
      target:
        description: 'Rollback target'
        required: true
        type: choice
        options:
          - kubernetes
          - docker
      environment:
        description: 'Target environment'
        required: true
        type: choice
        options:
          - staging
          - production
      deployment:
        description: 'Deployment to rollback (or "all")'
        required: false
        default: 'all'
        type: string
      revision:
        description: 'Specific revision to rollback to (0 = previous)'
        required: false
        default: '0'
        type: string

concurrency:
  group: rollback-${{ github.ref }}-${{ inputs.environment }}
  cancel-in-progress: false

jobs:
  pre-rollback-checks:
    name: Pre-Rollback Safety Checks
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Verify environment protection
        run: |
          echo "🔒 Verifying environment: ${{ inputs.environment }}"
          echo "Target: ${{ inputs.target }}"
          echo "Deployment: ${{ inputs.deployment }}"
          echo ""
          echo "⚠️  This action will rollback production systems!"
          echo "   Ensure this is intentional and approved."

      - name: Create pre-rollback snapshot
        run: |
          mkdir -p rollback-data
          echo "Rollback initiated at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" > rollback-data/rollback-snapshot.txt
          echo "Target: ${{ inputs.target }}" >> rollback-data/rollback-snapshot.txt
          echo "Environment: ${{ inputs.environment }}" >> rollback-data/rollback-snapshot.txt
          echo "Deployment: ${{ inputs.deployment }}" >> rollback-data/rollback-snapshot.txt
          echo "Requested By: ${{ github.actor }}" >> rollback-data/rollback-snapshot.txt
          echo "Workflow Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> rollback-data/rollback-snapshot.txt
          cat rollback-data/rollback-snapshot.txt

      - name: Upload pre-rollback snapshot
        uses: actions/upload-artifact@v4
        with:
          name: rollback-snapshot-${{ github.run_id }}
          path: rollback-data/
          retention-days: 90

  rollback-kubernetes:
    name: Rollback Kubernetes
    needs: pre-rollback-checks
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    if: ${{ inputs.target == 'kubernetes' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Nushell
        run: |
          cargo install nu --locked
          nu --version

      - name: Install kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'latest'

      - name: Configure kubeconfig (staging)
        run: |
          mkdir -p ~/.kube
          echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > ~/.kube/config
          chmod 600 ~/.kube/config
          kubectl cluster-info
        if: ${{ inputs.environment == 'staging' }}

      - name: Configure kubeconfig (production)
        run: |
          mkdir -p ~/.kube
          echo "${{ secrets.KUBE_CONFIG_PRODUCTION }}" | base64 -d > ~/.kube/config
          chmod 600 ~/.kube/config
          kubectl cluster-info
        if: ${{ inputs.environment == 'production' }}

      - name: Store deployment history
        run: |
          mkdir -p rollback-data
          echo "=== Deployment History ===" > rollback-data/pre-rollback-status.txt

          for deployment in vapora-backend vapora-agents vapora-llm-router; do
            if [ "${{ inputs.deployment }}" == "all" ] || [ "${{ inputs.deployment }}" == "$deployment" ]; then
              echo "Deployment: $deployment" >> rollback-data/pre-rollback-status.txt
              kubectl rollout history deployment/$deployment -n vapora >> rollback-data/pre-rollback-status.txt 2>&1
              kubectl get deployment $deployment -n vapora -o yaml >> rollback-data/pre-rollback-status.txt 2>&1
              echo "---" >> rollback-data/pre-rollback-status.txt
            fi
          done

          cat rollback-data/pre-rollback-status.txt

      - name: Perform Kubernetes rollback
        run: |
          cd provisioning
          nu scripts/rollback.nu \
            --target kubernetes \
            --deployment "${{ inputs.deployment }}" \
            --revision ${{ inputs.revision }} \
            2>&1 | tee ../rollback-data/rollback-output.log

      - name: Verify rollback status
        run: |
          echo "=== Post-Rollback Status ===" > rollback-data/post-rollback-status.txt

          for deployment in vapora-backend vapora-agents vapora-llm-router; do
            if [ "${{ inputs.deployment }}" == "all" ] || [ "${{ inputs.deployment }}" == "$deployment" ]; then
              echo "Deployment: $deployment" >> rollback-data/post-rollback-status.txt
              kubectl get deployment $deployment -n vapora -o wide >> rollback-data/post-rollback-status.txt 2>&1
              kubectl rollout status deployment/$deployment -n vapora --timeout=5m >> rollback-data/post-rollback-status.txt 2>&1 || true
              echo "---" >> rollback-data/post-rollback-status.txt
            fi
          done

          cat rollback-data/post-rollback-status.txt

      - name: Check pod health after rollback
        run: |
          echo "Pod Status After Rollback:" >> rollback-data/post-rollback-status.txt
          kubectl get pods -n vapora -o wide >> rollback-data/post-rollback-status.txt 2>&1
          echo "" >> rollback-data/post-rollback-status.txt
          echo "Recent Events:" >> rollback-data/post-rollback-status.txt
          kubectl get events -n vapora --sort-by='.lastTimestamp' | tail -20 >> rollback-data/post-rollback-status.txt 2>&1

      - name: Upload rollback logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: k8s-rollback-logs-${{ github.run_id }}
          path: rollback-data/
          retention-days: 90

  rollback-docker:
    name: Rollback Docker
    needs: pre-rollback-checks
    runs-on: ubuntu-latest
    if: ${{ inputs.target == 'docker' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Nushell
        run: |
          cargo install nu --locked
          nu --version

      - name: Show rollback options
        run: |
          mkdir -p rollback-data

          cat > rollback-data/docker-rollback-guide.md << 'EOF'
          # Docker Rollback Guide

          Docker Compose rollback requires manual steps:

          ## Option 1: Revert to previous compose file
          ```bash
          cd deploy/docker
          docker compose down
          # Restore previous docker-compose.yml
          git checkout HEAD~1 docker-compose.yml
          docker compose up -d
          ```

          ## Option 2: Stop and restart with older images
          ```bash
          docker compose -f docker-compose.yml.backup up -d
          ```

          ## Option 3: Remove containers and redeploy
          ```bash
          docker compose down
          docker system prune -f
          docker compose up -d
          ```

          ## Verification
          ```bash
          docker compose ps
          docker compose logs -f backend
          curl http://localhost:8001/health
          ```

          EOF

          cat rollback-data/docker-rollback-guide.md

      - name: Store Docker compose file
        run: |
          mkdir -p rollback-data
          if [ -f "deploy/docker/docker-compose.yml" ]; then
            cp deploy/docker/docker-compose.yml rollback-data/current-docker-compose.yml
            echo "Current docker-compose.yml backed up"
          fi

      - name: List available backups
        run: |
          echo "Looking for docker-compose backups..." >> rollback-data/available-backups.txt
          find . -name "docker-compose*.yml*" -type f 2>/dev/null | head -20 >> rollback-data/available-backups.txt 2>&1 || echo "No backups found"
          cat rollback-data/available-backups.txt

      - name: Upload rollback guide
        uses: actions/upload-artifact@v4
        with:
          name: docker-rollback-guide-${{ github.run_id }}
          path: rollback-data/
          retention-days: 90

  post-rollback-verification:
    name: Post-Rollback Verification
    needs: [pre-rollback-checks]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Create rollback report
        run: |
          mkdir -p rollback-reports

          cat > rollback-reports/ROLLBACK_REPORT.md << 'EOF'
          # Rollback Execution Report

          **Rollback Time**: $(date -u +'%Y-%m-%dT%H:%M:%SZ')
          **Target**: ${{ inputs.target }}
          **Environment**: ${{ inputs.environment }}
          **Deployment**: ${{ inputs.deployment }}
          **Revision**: ${{ inputs.revision }}
          **Initiated By**: ${{ github.actor }}
          **Workflow Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

          ## Status

          - **Pre-rollback Checks**: ✅ Passed
          - **Rollback Execution**: ${{ job.status == 'success' && '✅' || '⚠️' }}

          ## Artifacts

          Check the following artifacts for detailed information:
          - `rollback-snapshot-${{ github.run_id }}` - Initial snapshot
          - `k8s-rollback-logs-${{ github.run_id }}` - Kubernetes rollback logs (if K8s)
          - `docker-rollback-guide-${{ github.run_id }}` - Docker rollback guide (if Docker)

          ## Next Steps

          1. Verify service health
          2. Review application logs
          3. Monitor metrics
          4. Investigate root cause of previous deployment
          5. Plan corrected deployment

          ## Rollback Verification

          ### For Kubernetes
          ```bash
          kubectl get deployments -n vapora
          kubectl logs -f deployment/vapora-backend -n vapora
          kubectl rollout history deployment/vapora-backend -n vapora
          ```

          ### For Docker
          ```bash
          docker compose ps
          docker compose logs -f
          ```

          EOF
          cat rollback-reports/ROLLBACK_REPORT.md

      - name: Upload rollback report
        uses: actions/upload-artifact@v4
        with:
          name: rollback-report-${{ github.run_id }}
          path: rollback-reports/
          retention-days: 90

      - name: Post rollback notification
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: `🔙 Deployment Rollback Executed - ${{ inputs.environment }}`,
              body: `**Rollback Summary**\n\n- **Target**: ${{ inputs.target }}\n- **Environment**: ${{ inputs.environment }}\n- **Deployment**: ${{ inputs.deployment }}\n- **Revision**: ${{ inputs.revision }}\n- **Executed By**: ${{ github.actor }}\n- **Time**: ${new Date().toISOString()}\n\n**Artifacts**:\n- rollback-snapshot-${{ github.run_id }}\n- rollback-report-${{ github.run_id }}\n\n**Action Required**: Verify service health and investigate root cause.`,
              labels: ['deployment', 'rollback', 'incident']
            });

      - name: Notify Slack - Rollback
        uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          text: |
            🔙 VAPORA Rollback Executed
            Target: ${{ inputs.target }}
            Environment: ${{ inputs.environment }}
            Deployment: ${{ inputs.deployment }}
            Executed By: ${{ github.actor }}
          webhook_url: ${{ secrets.SLACK_WEBHOOK_ALERTS }}
          fields: repo,message,commit,author
        continue-on-error: true