name: Rollback Deployment on: workflow_dispatch: inputs: target: description: 'Rollback target' required: true type: choice options: - kubernetes - docker environment: description: 'Target environment' required: true type: choice options: - staging - production deployment: description: 'Deployment to rollback (or "all")' required: false default: 'all' type: string revision: description: 'Specific revision to rollback to (0 = previous)' required: false default: '0' type: string concurrency: group: rollback-${{ github.ref }}-${{ inputs.environment }} cancel-in-progress: false jobs: pre-rollback-checks: name: Pre-Rollback Safety Checks runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Verify environment protection run: | echo "🔒 Verifying environment: ${{ inputs.environment }}" echo "Target: ${{ inputs.target }}" echo "Deployment: ${{ inputs.deployment }}" echo "" echo "⚠️ This action will rollback production systems!" echo " Ensure this is intentional and approved." - name: Create pre-rollback snapshot run: | mkdir -p rollback-data echo "Rollback initiated at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" > rollback-data/rollback-snapshot.txt echo "Target: ${{ inputs.target }}" >> rollback-data/rollback-snapshot.txt echo "Environment: ${{ inputs.environment }}" >> rollback-data/rollback-snapshot.txt echo "Deployment: ${{ inputs.deployment }}" >> rollback-data/rollback-snapshot.txt echo "Requested By: ${{ github.actor }}" >> rollback-data/rollback-snapshot.txt echo "Workflow Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> rollback-data/rollback-snapshot.txt cat rollback-data/rollback-snapshot.txt - name: Upload pre-rollback snapshot uses: actions/upload-artifact@v4 with: name: rollback-snapshot-${{ github.run_id }} path: rollback-data/ retention-days: 90 rollback-kubernetes: name: Rollback Kubernetes needs: pre-rollback-checks runs-on: ubuntu-latest environment: ${{ inputs.environment }} if: ${{ inputs.target == 'kubernetes' }} steps: - name: Checkout code uses: actions/checkout@v4 - name: Install Nushell run: | cargo install nu --locked nu --version - name: Install kubectl uses: azure/setup-kubectl@v3 with: version: 'latest' - name: Configure kubeconfig (staging) run: | mkdir -p ~/.kube echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > ~/.kube/config chmod 600 ~/.kube/config kubectl cluster-info if: ${{ inputs.environment == 'staging' }} - name: Configure kubeconfig (production) run: | mkdir -p ~/.kube echo "${{ secrets.KUBE_CONFIG_PRODUCTION }}" | base64 -d > ~/.kube/config chmod 600 ~/.kube/config kubectl cluster-info if: ${{ inputs.environment == 'production' }} - name: Store deployment history run: | mkdir -p rollback-data echo "=== Deployment History ===" > rollback-data/pre-rollback-status.txt for deployment in vapora-backend vapora-agents vapora-llm-router; do if [ "${{ inputs.deployment }}" == "all" ] || [ "${{ inputs.deployment }}" == "$deployment" ]; then echo "Deployment: $deployment" >> rollback-data/pre-rollback-status.txt kubectl rollout history deployment/$deployment -n vapora >> rollback-data/pre-rollback-status.txt 2>&1 kubectl get deployment $deployment -n vapora -o yaml >> rollback-data/pre-rollback-status.txt 2>&1 echo "---" >> rollback-data/pre-rollback-status.txt fi done cat rollback-data/pre-rollback-status.txt - name: Perform Kubernetes rollback run: | cd provisioning nu scripts/rollback.nu \ --target kubernetes \ --deployment "${{ inputs.deployment }}" \ --revision ${{ inputs.revision }} \ 2>&1 | tee ../rollback-data/rollback-output.log - name: Verify rollback status run: | echo "=== Post-Rollback Status ===" > rollback-data/post-rollback-status.txt for deployment in vapora-backend vapora-agents vapora-llm-router; do if [ "${{ inputs.deployment }}" == "all" ] || [ "${{ inputs.deployment }}" == "$deployment" ]; then echo "Deployment: $deployment" >> rollback-data/post-rollback-status.txt kubectl get deployment $deployment -n vapora -o wide >> rollback-data/post-rollback-status.txt 2>&1 kubectl rollout status deployment/$deployment -n vapora --timeout=5m >> rollback-data/post-rollback-status.txt 2>&1 || true echo "---" >> rollback-data/post-rollback-status.txt fi done cat rollback-data/post-rollback-status.txt - name: Check pod health after rollback run: | echo "Pod Status After Rollback:" >> rollback-data/post-rollback-status.txt kubectl get pods -n vapora -o wide >> rollback-data/post-rollback-status.txt 2>&1 echo "" >> rollback-data/post-rollback-status.txt echo "Recent Events:" >> rollback-data/post-rollback-status.txt kubectl get events -n vapora --sort-by='.lastTimestamp' | tail -20 >> rollback-data/post-rollback-status.txt 2>&1 - name: Upload rollback logs if: always() uses: actions/upload-artifact@v4 with: name: k8s-rollback-logs-${{ github.run_id }} path: rollback-data/ retention-days: 90 rollback-docker: name: Rollback Docker needs: pre-rollback-checks runs-on: ubuntu-latest if: ${{ inputs.target == 'docker' }} steps: - name: Checkout code uses: actions/checkout@v4 - name: Install Nushell run: | cargo install nu --locked nu --version - name: Show rollback options run: | mkdir -p rollback-data cat > rollback-data/docker-rollback-guide.md << 'EOF' # Docker Rollback Guide Docker Compose rollback requires manual steps: ## Option 1: Revert to previous compose file ```bash cd deploy/docker docker compose down # Restore previous docker-compose.yml git checkout HEAD~1 docker-compose.yml docker compose up -d ``` ## Option 2: Stop and restart with older images ```bash docker compose -f docker-compose.yml.backup up -d ``` ## Option 3: Remove containers and redeploy ```bash docker compose down docker system prune -f docker compose up -d ``` ## Verification ```bash docker compose ps docker compose logs -f backend curl http://localhost:8001/health ``` EOF cat rollback-data/docker-rollback-guide.md - name: Store Docker compose file run: | mkdir -p rollback-data if [ -f "deploy/docker/docker-compose.yml" ]; then cp deploy/docker/docker-compose.yml rollback-data/current-docker-compose.yml echo "Current docker-compose.yml backed up" fi - name: List available backups run: | echo "Looking for docker-compose backups..." >> rollback-data/available-backups.txt find . -name "docker-compose*.yml*" -type f 2>/dev/null | head -20 >> rollback-data/available-backups.txt 2>&1 || echo "No backups found" cat rollback-data/available-backups.txt - name: Upload rollback guide uses: actions/upload-artifact@v4 with: name: docker-rollback-guide-${{ github.run_id }} path: rollback-data/ retention-days: 90 post-rollback-verification: name: Post-Rollback Verification needs: [pre-rollback-checks] runs-on: ubuntu-latest if: always() steps: - name: Checkout code uses: actions/checkout@v4 - name: Create rollback report run: | mkdir -p rollback-reports cat > rollback-reports/ROLLBACK_REPORT.md << 'EOF' # Rollback Execution Report **Rollback Time**: $(date -u +'%Y-%m-%dT%H:%M:%SZ') **Target**: ${{ inputs.target }} **Environment**: ${{ inputs.environment }} **Deployment**: ${{ inputs.deployment }} **Revision**: ${{ inputs.revision }} **Initiated By**: ${{ github.actor }} **Workflow Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ## Status - **Pre-rollback Checks**: ✅ Passed - **Rollback Execution**: ${{ job.status == 'success' && '✅' || '⚠️' }} ## Artifacts Check the following artifacts for detailed information: - `rollback-snapshot-${{ github.run_id }}` - Initial snapshot - `k8s-rollback-logs-${{ github.run_id }}` - Kubernetes rollback logs (if K8s) - `docker-rollback-guide-${{ github.run_id }}` - Docker rollback guide (if Docker) ## Next Steps 1. Verify service health 2. Review application logs 3. Monitor metrics 4. Investigate root cause of previous deployment 5. Plan corrected deployment ## Rollback Verification ### For Kubernetes ```bash kubectl get deployments -n vapora kubectl logs -f deployment/vapora-backend -n vapora kubectl rollout history deployment/vapora-backend -n vapora ``` ### For Docker ```bash docker compose ps docker compose logs -f ``` EOF cat rollback-reports/ROLLBACK_REPORT.md - name: Upload rollback report uses: actions/upload-artifact@v4 with: name: rollback-report-${{ github.run_id }} path: rollback-reports/ retention-days: 90 - name: Post rollback notification uses: actions/github-script@v7 with: script: | github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: `🔙 Deployment Rollback Executed - ${{ inputs.environment }}`, body: `**Rollback Summary**\n\n- **Target**: ${{ inputs.target }}\n- **Environment**: ${{ inputs.environment }}\n- **Deployment**: ${{ inputs.deployment }}\n- **Revision**: ${{ inputs.revision }}\n- **Executed By**: ${{ github.actor }}\n- **Time**: ${new Date().toISOString()}\n\n**Artifacts**:\n- rollback-snapshot-${{ github.run_id }}\n- rollback-report-${{ github.run_id }}\n\n**Action Required**: Verify service health and investigate root cause.`, labels: ['deployment', 'rollback', 'incident'] }); - name: Notify Slack - Rollback uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} text: | 🔙 VAPORA Rollback Executed Target: ${{ inputs.target }} Environment: ${{ inputs.environment }} Deployment: ${{ inputs.deployment }} Executed By: ${{ github.actor }} webhook_url: ${{ secrets.SLACK_WEBHOOK_ALERTS }} fields: repo,message,commit,author continue-on-error: true