352 lines
12 KiB
YAML
352 lines
12 KiB
YAML
|
# VAPORA Woodpecker Pipeline - Rollback Deployment
|
||
|
|
# Safe deployment rollback with verification and pre-checks
|
||
|
|
# Triggers on: manual promotion only (safety feature)
|
||
|
|
|
||
|
|
trigger:
|
||
|
|
event: [promote]
|
||
|
|
branch: [main, develop]
|
||
|
|
|
||
|
|
variables:
|
||
|
|
ARTIFACTS_DIR: provisioning/artifacts
|
||
|
|
LOGS_DIR: provisioning/logs
|
||
|
|
VAPORA_NAMESPACE: vapora
|
||
|
|
|
||
|
|
stages:
|
||
|
|
pre_rollback_checks:
|
||
|
|
steps:
|
||
|
|
- name: verify_environment
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
echo "🔒 Pre-Rollback Safety Checks"
|
||
|
|
echo "---"
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
{
|
||
|
|
echo "Rollback initiated at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||
|
|
echo "Commit: ${CI_COMMIT_SHA:0:8}"
|
||
|
|
echo "Branch: ${CI_COMMIT_BRANCH}"
|
||
|
|
echo "Pipeline: ${CI_BUILD_LINK}"
|
||
|
|
echo ""
|
||
|
|
echo "⚠️ This action will rollback production systems!"
|
||
|
|
echo " Ensure this is intentional and approved."
|
||
|
|
} | tee ${LOGS_DIR}/rollback/pre-rollback-snapshot.txt
|
||
|
|
|
||
|
|
install_dependencies:
|
||
|
|
depends_on: [pre_rollback_checks]
|
||
|
|
steps:
|
||
|
|
- name: install_tools
|
||
|
|
image: rust:latest
|
||
|
|
commands:
|
||
|
|
- apt-get update && apt-get install -y curl jq yq
|
||
|
|
- cargo install nu --locked
|
||
|
|
- pip install jinja2-cli
|
||
|
|
- curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
||
|
|
- chmod +x kubectl && mv kubectl /usr/local/bin/
|
||
|
|
- nu --version
|
||
|
|
- kubectl version --client
|
||
|
|
- yq --version
|
||
|
|
|
||
|
|
configure_kubernetes:
|
||
|
|
depends_on: [install_dependencies]
|
||
|
|
steps:
|
||
|
|
- name: setup_kubeconfig
|
||
|
|
image: alpine:latest
|
||
|
|
environment:
|
||
|
|
KUBE_CONFIG_STAGING: ${KUBE_CONFIG_STAGING}
|
||
|
|
KUBE_CONFIG_PRODUCTION: ${KUBE_CONFIG_PRODUCTION}
|
||
|
|
commands:
|
||
|
|
- mkdir -p ~/.kube
|
||
|
|
- |
|
||
|
|
if [ "${Rollback_Environment}" = "production" ]; then
|
||
|
|
echo "$KUBE_CONFIG_PRODUCTION" | base64 -d > ~/.kube/config
|
||
|
|
echo "✓ Production kubeconfig configured"
|
||
|
|
else
|
||
|
|
echo "$KUBE_CONFIG_STAGING" | base64 -d > ~/.kube/config
|
||
|
|
echo "✓ Staging kubeconfig configured"
|
||
|
|
fi
|
||
|
|
- chmod 600 ~/.kube/config
|
||
|
|
- kubectl cluster-info
|
||
|
|
- kubectl get nodes
|
||
|
|
|
||
|
|
store_deployment_history:
|
||
|
|
depends_on: [configure_kubernetes]
|
||
|
|
steps:
|
||
|
|
- name: snapshot_current_state
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "📸 Storing current deployment history..."
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
{
|
||
|
|
echo "=== Current Deployment State ===" | tee ${LOGS_DIR}/rollback/pre-rollback-status.txt
|
||
|
|
echo ""
|
||
|
|
echo "Deployments:"
|
||
|
|
kubectl get deployments -n ${VAPORA_NAMESPACE} -o yaml | tee -a ${LOGS_DIR}/rollback/pre-rollback-status.txt
|
||
|
|
echo ""
|
||
|
|
echo "Rollout History:"
|
||
|
|
for deployment in vapora-backend vapora-agents vapora-llm-router; do
|
||
|
|
echo "--- $deployment ---" | tee -a ${LOGS_DIR}/rollback/pre-rollback-status.txt
|
||
|
|
kubectl rollout history deployment/$deployment -n ${VAPORA_NAMESPACE} 2>&1 | tee -a ${LOGS_DIR}/rollback/pre-rollback-status.txt
|
||
|
|
done
|
||
|
|
}
|
||
|
|
|
||
|
|
kubernetes_rollback:
|
||
|
|
depends_on: [store_deployment_history]
|
||
|
|
steps:
|
||
|
|
- name: perform_rollback
|
||
|
|
image: rust:latest
|
||
|
|
environment:
|
||
|
|
RUST_LOG: warn
|
||
|
|
commands:
|
||
|
|
- apt-get update && apt-get install -y curl jq
|
||
|
|
- |
|
||
|
|
echo "🔙 Performing Kubernetes Rollback..."
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
cd provisioning
|
||
|
|
nu scripts/rollback.nu \
|
||
|
|
--target kubernetes \
|
||
|
|
--deployment "${Rollback_Deployment:-all}" \
|
||
|
|
--revision ${Rollback_Revision:-0} \
|
||
|
|
2>&1 | tee ../${LOGS_DIR}/rollback/rollback-output.log
|
||
|
|
|
||
|
|
- name: verify_rollback
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "✓ Verifying rollback status..."
|
||
|
|
{
|
||
|
|
echo "=== Post-Rollback Deployment State ===" | tee ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
echo ""
|
||
|
|
echo "Deployments:"
|
||
|
|
kubectl get deployments -n ${VAPORA_NAMESPACE} -o wide | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
echo ""
|
||
|
|
echo "Rollout Status:"
|
||
|
|
for deployment in vapora-backend vapora-agents vapora-llm-router; do
|
||
|
|
echo "--- $deployment ---" | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
kubectl rollout status deployment/$deployment -n ${VAPORA_NAMESPACE} --timeout=5m 2>&1 | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
done
|
||
|
|
}
|
||
|
|
|
||
|
|
- name: check_pod_health
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "Pod Status After Rollback:" | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
kubectl get pods -n ${VAPORA_NAMESPACE} -o wide | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
echo "" | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
echo "Recent Events:" | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
kubectl get events -n ${VAPORA_NAMESPACE} --sort-by='.lastTimestamp' | tail -20 | tee -a ${LOGS_DIR}/rollback/post-rollback-status.txt
|
||
|
|
|
||
|
|
docker_rollback_guide:
|
||
|
|
depends_on: [store_deployment_history]
|
||
|
|
steps:
|
||
|
|
- name: generate_docker_guide
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
echo "📝 Generating Docker rollback guide..."
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
cat > ${LOGS_DIR}/rollback/DOCKER_ROLLBACK_GUIDE.md << 'EOF'
|
||
|
|
# Docker Rollback Guide
|
||
|
|
|
||
|
|
Docker Compose rollback requires manual steps:
|
||
|
|
|
||
|
|
## Option 1: Revert to previous compose file
|
||
|
|
|
||
|
|
```bash
|
||
|
|
cd deploy/docker
|
||
|
|
docker compose down
|
||
|
|
git checkout HEAD~1 docker-compose.yml
|
||
|
|
docker compose up -d
|
||
|
|
```
|
||
|
|
|
||
|
|
## Option 2: Stop and restart with older images
|
||
|
|
|
||
|
|
```bash
|
||
|
|
docker compose -f docker-compose.yml.backup up -d
|
||
|
|
```
|
||
|
|
|
||
|
|
## Option 3: Remove containers and redeploy from previous artifacts
|
||
|
|
|
||
|
|
```bash
|
||
|
|
docker compose down
|
||
|
|
docker system prune -f
|
||
|
|
docker compose up -d
|
||
|
|
```
|
||
|
|
|
||
|
|
## Verification
|
||
|
|
|
||
|
|
After rollback, verify services are running:
|
||
|
|
|
||
|
|
```bash
|
||
|
|
docker compose ps
|
||
|
|
docker compose logs -f backend
|
||
|
|
curl http://localhost:8001/health
|
||
|
|
```
|
||
|
|
|
||
|
|
## Checking Compose File Backups
|
||
|
|
|
||
|
|
```bash
|
||
|
|
find . -name "docker-compose*.yml*" -type f | sort
|
||
|
|
```
|
||
|
|
|
||
|
|
## Restoring from Backup
|
||
|
|
|
||
|
|
```bash
|
||
|
|
# If you have a timestamped backup
|
||
|
|
cp docker-compose.yml.$(date +%s) docker-compose.yml
|
||
|
|
docker compose up -d
|
||
|
|
```
|
||
|
|
|
||
|
|
EOF
|
||
|
|
cat ${LOGS_DIR}/rollback/DOCKER_ROLLBACK_GUIDE.md
|
||
|
|
|
||
|
|
- name: store_docker_state
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
echo "📋 Storing Docker Compose state..."
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
if [ -f "deploy/docker/docker-compose.yml" ]; then
|
||
|
|
cp deploy/docker/docker-compose.yml ${LOGS_DIR}/rollback/current-docker-compose.yml
|
||
|
|
echo "✓ Current docker-compose.yml backed up"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "Looking for available backups..."
|
||
|
|
find . -name "docker-compose*.yml*" -type f 2>/dev/null | head -20 | tee ${LOGS_DIR}/rollback/available-backups.txt
|
||
|
|
|
||
|
|
post_rollback_verification:
|
||
|
|
depends_on: [kubernetes_rollback, docker_rollback_guide]
|
||
|
|
steps:
|
||
|
|
- name: generate_rollback_report
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
mkdir -p ${LOGS_DIR}/rollback
|
||
|
|
cat > ${LOGS_DIR}/rollback/ROLLBACK_REPORT.md << 'EOF'
|
||
|
|
# Rollback Execution Report
|
||
|
|
|
||
|
|
**Rollback Time**: $(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||
|
|
**Target**: ${Rollback_Target:-kubernetes}
|
||
|
|
**Environment**: ${Rollback_Environment:-staging}
|
||
|
|
**Deployment**: ${Rollback_Deployment:-all}
|
||
|
|
**Revision**: ${Rollback_Revision:-0 (previous)}
|
||
|
|
**Pipeline**: ${CI_BUILD_LINK}
|
||
|
|
|
||
|
|
## Status
|
||
|
|
|
||
|
|
- **Pre-rollback Checks**: ✅ Passed
|
||
|
|
- **Rollback Execution**: In Progress
|
||
|
|
- **Post-rollback Verification**: Pending
|
||
|
|
|
||
|
|
## Artifacts
|
||
|
|
|
||
|
|
Check the following for detailed information:
|
||
|
|
|
||
|
|
- `pre-rollback-snapshot.txt` - Initial state snapshot
|
||
|
|
- `pre-rollback-status.txt` - Pre-rollback deployments
|
||
|
|
- `post-rollback-status.txt` - Post-rollback status
|
||
|
|
- `rollback-output.log` - Rollback script output
|
||
|
|
- `DOCKER_ROLLBACK_GUIDE.md` - Docker rollback instructions (if applicable)
|
||
|
|
|
||
|
|
## Next Steps
|
||
|
|
|
||
|
|
1. Verify all services are running
|
||
|
|
2. Check application logs for errors
|
||
|
|
3. Run health checks
|
||
|
|
4. Monitor metrics and alerts
|
||
|
|
5. Investigate root cause of previous deployment failure
|
||
|
|
6. Plan corrected deployment
|
||
|
|
|
||
|
|
## Rollback Verification Commands
|
||
|
|
|
||
|
|
### For Kubernetes
|
||
|
|
|
||
|
|
```bash
|
||
|
|
# Check current deployments
|
||
|
|
kubectl get deployments -n ${VAPORA_NAMESPACE}
|
||
|
|
kubectl get pods -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# View logs
|
||
|
|
kubectl logs -f deployment/vapora-backend -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# Check rollout history
|
||
|
|
kubectl rollout history deployment/vapora-backend -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# View recent events
|
||
|
|
kubectl get events -n ${VAPORA_NAMESPACE} --sort-by='.lastTimestamp'
|
||
|
|
```
|
||
|
|
|
||
|
|
### For Docker
|
||
|
|
|
||
|
|
```bash
|
||
|
|
# Check container status
|
||
|
|
docker compose ps
|
||
|
|
|
||
|
|
# View logs
|
||
|
|
docker compose logs -f
|
||
|
|
|
||
|
|
# Check service health
|
||
|
|
curl http://localhost:8001/health
|
||
|
|
```
|
||
|
|
|
||
|
|
EOF
|
||
|
|
cat ${LOGS_DIR}/rollback/ROLLBACK_REPORT.md
|
||
|
|
|
||
|
|
publish:
|
||
|
|
depends_on: [post_rollback_verification]
|
||
|
|
steps:
|
||
|
|
- name: publish_rollback_artifacts
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- echo "📦 Rollback artifacts published"
|
||
|
|
- echo ""
|
||
|
|
- ls -lah ${LOGS_DIR}/rollback/
|
||
|
|
- echo ""
|
||
|
|
- du -sh ${LOGS_DIR}/rollback/
|
||
|
|
|
||
|
|
- name: notify_slack
|
||
|
|
image: alpine:latest
|
||
|
|
environment:
|
||
|
|
SLACK_WEBHOOK: ${SLACK_WEBHOOK_ALERTS}
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
if [ -n "$SLACK_WEBHOOK" ]; then
|
||
|
|
apk add --no-cache curl jq
|
||
|
|
curl -X POST $SLACK_WEBHOOK \
|
||
|
|
-H 'Content-Type: application/json' \
|
||
|
|
-d '{
|
||
|
|
"text": "🔙 VAPORA Rollback Executed",
|
||
|
|
"blocks": [
|
||
|
|
{
|
||
|
|
"type": "section",
|
||
|
|
"text": {
|
||
|
|
"type": "mrkdwn",
|
||
|
|
"text": "🔙 **VAPORA Rollback Executed**\n\n*Rollback Details:*\n• Target: ${Rollback_Target:-kubernetes}\n• Environment: ${Rollback_Environment:-staging}\n• Deployment: ${Rollback_Deployment:-all}"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "section",
|
||
|
|
"text": {
|
||
|
|
"type": "mrkdwn",
|
||
|
|
"text": "*⚠️ Action Required:*\n1. Verify service health\n2. Review application logs\n3. Investigate root cause\n4. Plan corrected deployment"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "context",
|
||
|
|
"elements": [
|
||
|
|
{
|
||
|
|
"type": "mrkdwn",
|
||
|
|
"text": "*Reports*: Check rollback artifacts in logs\n*Commit*: '"${CI_COMMIT_SHA:0:8}"'\n*Branch*: '"${CI_COMMIT_BRANCH}"'"
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}'
|
||
|
|
else
|
||
|
|
echo "⚠️ Slack webhook not configured"
|
||
|
|
fi
|