353 lines
12 KiB
YAML
353 lines
12 KiB
YAML
|
# VAPORA Woodpecker Pipeline - Deploy to Kubernetes
|
||
|
|
# Deploys VAPORA to Kubernetes cluster with dry-run and verification
|
||
|
|
# Triggers on: manual promotion
|
||
|
|
|
||
|
|
trigger:
|
||
|
|
event: [promote]
|
||
|
|
branch: [main, develop]
|
||
|
|
|
||
|
|
variables:
|
||
|
|
ARTIFACTS_DIR: provisioning/artifacts
|
||
|
|
LOGS_DIR: provisioning/logs
|
||
|
|
VAPORA_NAMESPACE: vapora
|
||
|
|
|
||
|
|
stages:
|
||
|
|
setup:
|
||
|
|
steps:
|
||
|
|
- name: prepare
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- mkdir -p ${ARTIFACTS_DIR} ${LOGS_DIR}
|
||
|
|
- echo "☸️ VAPORA Kubernetes Deployment Pipeline"
|
||
|
|
- echo "Commit: ${CI_COMMIT_SHA:0:8}"
|
||
|
|
- echo "Branch: ${CI_COMMIT_BRANCH}"
|
||
|
|
- echo "Event: ${CI_PIPELINE_EVENT}"
|
||
|
|
|
||
|
|
install_dependencies:
|
||
|
|
steps:
|
||
|
|
- name: install_tools
|
||
|
|
image: rust:latest
|
||
|
|
commands:
|
||
|
|
- apt-get update && apt-get install -y curl jq yq
|
||
|
|
- cargo install nu --locked
|
||
|
|
- pip install jinja2-cli
|
||
|
|
- curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
||
|
|
- chmod +x kubectl && mv kubectl /usr/local/bin/
|
||
|
|
- nu --version
|
||
|
|
- kubectl version --client
|
||
|
|
- jinja2 --version
|
||
|
|
- yq --version
|
||
|
|
|
||
|
|
configure_kubernetes:
|
||
|
|
depends_on: [install_dependencies]
|
||
|
|
steps:
|
||
|
|
- name: setup_kubeconfig_staging
|
||
|
|
image: alpine:latest
|
||
|
|
environment:
|
||
|
|
KUBE_CONFIG_STAGING: ${KUBE_CONFIG_STAGING}
|
||
|
|
commands:
|
||
|
|
- mkdir -p ~/.kube
|
||
|
|
- echo "$KUBE_CONFIG_STAGING" | base64 -d > ~/.kube/config
|
||
|
|
- chmod 600 ~/.kube/config
|
||
|
|
- echo "✓ Kubeconfig configured for staging"
|
||
|
|
when:
|
||
|
|
evaluate: 'return build.Deploy_Environment == "staging"'
|
||
|
|
|
||
|
|
- name: setup_kubeconfig_production
|
||
|
|
image: alpine:latest
|
||
|
|
environment:
|
||
|
|
KUBE_CONFIG_PRODUCTION: ${KUBE_CONFIG_PRODUCTION}
|
||
|
|
commands:
|
||
|
|
- mkdir -p ~/.kube
|
||
|
|
- echo "$KUBE_CONFIG_PRODUCTION" | base64 -d > ~/.kube/config
|
||
|
|
- chmod 600 ~/.kube/config
|
||
|
|
- echo "✓ Kubeconfig configured for production"
|
||
|
|
when:
|
||
|
|
evaluate: 'return build.Deploy_Environment == "production"'
|
||
|
|
|
||
|
|
- name: verify_cluster
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- kubectl cluster-info
|
||
|
|
- kubectl get nodes
|
||
|
|
- echo "✓ Kubernetes cluster accessible"
|
||
|
|
|
||
|
|
validate_manifests:
|
||
|
|
depends_on: [configure_kubernetes]
|
||
|
|
steps:
|
||
|
|
- name: validate_kubernetes_manifests
|
||
|
|
image: rust:latest
|
||
|
|
environment:
|
||
|
|
RUST_LOG: warn
|
||
|
|
commands:
|
||
|
|
- apt-get update && apt-get install -y curl jq yq
|
||
|
|
- cargo install nu --locked > /dev/null 2>&1
|
||
|
|
- cargo install nickel --locked > /dev/null 2>&1
|
||
|
|
- pip install jinja2-cli > /dev/null 2>&1
|
||
|
|
- |
|
||
|
|
echo "Validating Kubernetes manifests..."
|
||
|
|
if [ -f "${ARTIFACTS_DIR}/deployment.yaml" ]; then
|
||
|
|
yq eval '.' "${ARTIFACTS_DIR}/deployment.yaml" > /dev/null && echo "✓ Deployment manifest valid"
|
||
|
|
yq eval '.' "${ARTIFACTS_DIR}/configmap.yaml" > /dev/null && echo "✓ ConfigMap manifest valid"
|
||
|
|
else
|
||
|
|
echo "⚠️ Manifests not found, generating from Nickel"
|
||
|
|
cd provisioning
|
||
|
|
nu scripts/ci-pipeline.nu --artifact-dir ../${ARTIFACTS_DIR} --mode multiuser 2>&1 | tee ../${LOGS_DIR}/k8s-generation.log
|
||
|
|
fi
|
||
|
|
|
||
|
|
- name: dry_run_validation
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "🔍 Performing dry-run validation..."
|
||
|
|
kubectl apply -f ${ARTIFACTS_DIR}/deployment.yaml --dry-run=server -n ${VAPORA_NAMESPACE} --record 2>&1 | tee ${LOGS_DIR}/dry-run-validation.log
|
||
|
|
if [ $? -eq 0 ]; then
|
||
|
|
echo "✓ Dry-run validation passed"
|
||
|
|
else
|
||
|
|
echo "❌ Dry-run validation failed"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
create_namespace:
|
||
|
|
depends_on: [validate_manifests]
|
||
|
|
steps:
|
||
|
|
- name: ensure_namespace
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "📁 Creating/verifying vapora namespace..."
|
||
|
|
kubectl get namespace ${VAPORA_NAMESPACE} > /dev/null 2>&1 || kubectl create namespace ${VAPORA_NAMESPACE}
|
||
|
|
echo "✓ Namespace ready"
|
||
|
|
|
||
|
|
- name: setup_rbac
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "🔐 Setting up RBAC..."
|
||
|
|
# Default service account has basic access
|
||
|
|
kubectl get serviceaccount default -n ${VAPORA_NAMESPACE} > /dev/null 2>&1 || {
|
||
|
|
echo "Creating default service account"
|
||
|
|
kubectl create serviceaccount default -n ${VAPORA_NAMESPACE}
|
||
|
|
}
|
||
|
|
echo "✓ RBAC configured"
|
||
|
|
|
||
|
|
deploy_configmap:
|
||
|
|
depends_on: [create_namespace]
|
||
|
|
steps:
|
||
|
|
- name: apply_configmap
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "⚙️ Applying ConfigMap..."
|
||
|
|
kubectl apply -f ${ARTIFACTS_DIR}/configmap.yaml -n ${VAPORA_NAMESPACE} --record
|
||
|
|
echo "✓ ConfigMap applied"
|
||
|
|
|
||
|
|
- name: verify_configmap
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "✓ ConfigMap contents:"
|
||
|
|
kubectl get configmap -n ${VAPORA_NAMESPACE} -o yaml | head -50
|
||
|
|
|
||
|
|
deploy_services:
|
||
|
|
depends_on: [deploy_configmap]
|
||
|
|
steps:
|
||
|
|
- name: apply_deployments
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "🚀 Applying Kubernetes Deployments..."
|
||
|
|
kubectl apply -f ${ARTIFACTS_DIR}/deployment.yaml -n ${VAPORA_NAMESPACE} --record
|
||
|
|
echo "✓ Deployments applied"
|
||
|
|
|
||
|
|
- name: monitor_rollout_backend
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "⏳ Waiting for backend rollout..."
|
||
|
|
kubectl rollout status deployment/vapora-backend -n ${VAPORA_NAMESPACE} --timeout=5m
|
||
|
|
echo "✓ Backend deployment ready"
|
||
|
|
|
||
|
|
- name: monitor_rollout_agents
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "⏳ Waiting for agents rollout..."
|
||
|
|
kubectl rollout status deployment/vapora-agents -n ${VAPORA_NAMESPACE} --timeout=5m
|
||
|
|
echo "✓ Agents deployment ready"
|
||
|
|
|
||
|
|
- name: monitor_rollout_llm_router
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "⏳ Waiting for LLM router rollout..."
|
||
|
|
kubectl rollout status deployment/vapora-llm-router -n ${VAPORA_NAMESPACE} --timeout=5m
|
||
|
|
echo "✓ LLM router deployment ready"
|
||
|
|
|
||
|
|
verify_deployment:
|
||
|
|
depends_on: [deploy_services]
|
||
|
|
steps:
|
||
|
|
- name: check_pods
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "🔍 Verifying pod status..."
|
||
|
|
kubectl get pods -n ${VAPORA_NAMESPACE} -o wide
|
||
|
|
echo ""
|
||
|
|
echo "Checking pod readiness..."
|
||
|
|
kubectl get pods -n ${VAPORA_NAMESPACE} -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
|
||
|
|
|
||
|
|
- name: check_services
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "🔍 Verifying services..."
|
||
|
|
kubectl get services -n ${VAPORA_NAMESPACE} -o wide
|
||
|
|
echo ""
|
||
|
|
echo "Service endpoints:"
|
||
|
|
kubectl get endpoints -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
- name: collect_logs
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "📋 Collecting deployment logs..."
|
||
|
|
mkdir -p ${LOGS_DIR}/kubernetes
|
||
|
|
kubectl get events -n ${VAPORA_NAMESPACE} --sort-by='.lastTimestamp' > ${LOGS_DIR}/kubernetes/events.log 2>&1
|
||
|
|
kubectl logs -n ${VAPORA_NAMESPACE} deployment/vapora-backend --tail=100 > ${LOGS_DIR}/kubernetes/backend.log 2>&1
|
||
|
|
kubectl logs -n ${VAPORA_NAMESPACE} deployment/vapora-agents --tail=100 > ${LOGS_DIR}/kubernetes/agents.log 2>&1
|
||
|
|
kubectl logs -n ${VAPORA_NAMESPACE} deployment/vapora-llm-router --tail=100 > ${LOGS_DIR}/kubernetes/llm-router.log 2>&1
|
||
|
|
|
||
|
|
- name: annotate_deployment
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- apk add --no-cache curl
|
||
|
|
- |
|
||
|
|
echo "📝 Annotating deployments..."
|
||
|
|
kubectl annotate deployment vapora-backend -n ${VAPORA_NAMESPACE} \
|
||
|
|
deployment.vapora/timestamp="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
|
||
|
|
deployment.vapora/commit="${CI_COMMIT_SHA:0:8}" \
|
||
|
|
deployment.vapora/branch="${CI_COMMIT_BRANCH}" \
|
||
|
|
--overwrite
|
||
|
|
|
||
|
|
generate_report:
|
||
|
|
depends_on: [verify_deployment]
|
||
|
|
steps:
|
||
|
|
- name: create_deployment_report
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
mkdir -p ${LOGS_DIR}
|
||
|
|
cat > ${LOGS_DIR}/KUBERNETES_DEPLOYMENT_REPORT.md << 'EOF'
|
||
|
|
# Kubernetes Deployment Report
|
||
|
|
|
||
|
|
**Deployment Time**: $(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||
|
|
**Commit**: ${CI_COMMIT_SHA}
|
||
|
|
**Branch**: ${CI_COMMIT_BRANCH}
|
||
|
|
**Namespace**: ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
## Status
|
||
|
|
|
||
|
|
✅ Kubernetes deployment successful
|
||
|
|
|
||
|
|
## Deployments
|
||
|
|
|
||
|
|
- **vapora-backend**: Running with configured replicas
|
||
|
|
- **vapora-agents**: Running with configured replicas
|
||
|
|
- **vapora-llm-router**: Running with configured replicas
|
||
|
|
|
||
|
|
## Verification Commands
|
||
|
|
|
||
|
|
```bash
|
||
|
|
# Check deployments
|
||
|
|
kubectl get deployments -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# View pods
|
||
|
|
kubectl get pods -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# Check logs
|
||
|
|
kubectl logs -f deployment/vapora-backend -n ${VAPORA_NAMESPACE}
|
||
|
|
|
||
|
|
# Port forward for local testing
|
||
|
|
kubectl port-forward -n ${VAPORA_NAMESPACE} svc/vapora-backend 8001:8001
|
||
|
|
|
||
|
|
# View events
|
||
|
|
kubectl get events -n ${VAPORA_NAMESPACE} --sort-by='.lastTimestamp'
|
||
|
|
|
||
|
|
# Check rollout status
|
||
|
|
kubectl rollout history deployment/vapora-backend -n ${VAPORA_NAMESPACE}
|
||
|
|
```
|
||
|
|
|
||
|
|
## Next Steps
|
||
|
|
|
||
|
|
1. Run health checks to verify all services
|
||
|
|
2. Monitor logs for any errors
|
||
|
|
3. Test API endpoints
|
||
|
|
4. Set up monitoring and alerts
|
||
|
|
5. Plan rollout to next environment
|
||
|
|
|
||
|
|
EOF
|
||
|
|
cat ${LOGS_DIR}/KUBERNETES_DEPLOYMENT_REPORT.md
|
||
|
|
|
||
|
|
publish:
|
||
|
|
depends_on: [generate_report]
|
||
|
|
steps:
|
||
|
|
- name: publish_results
|
||
|
|
image: alpine:latest
|
||
|
|
commands:
|
||
|
|
- echo "📦 Kubernetes deployment complete"
|
||
|
|
- echo ""
|
||
|
|
- echo "Logs:"
|
||
|
|
- ls -lah ${LOGS_DIR}/kubernetes/
|
||
|
|
- echo ""
|
||
|
|
- echo "Report:"
|
||
|
|
- cat ${LOGS_DIR}/KUBERNETES_DEPLOYMENT_REPORT.md
|
||
|
|
|
||
|
|
- name: notify_slack
|
||
|
|
image: alpine:latest
|
||
|
|
environment:
|
||
|
|
SLACK_WEBHOOK: ${SLACK_WEBHOOK_ALERTS}
|
||
|
|
commands:
|
||
|
|
- |
|
||
|
|
if [ -n "$SLACK_WEBHOOK" ]; then
|
||
|
|
apk add --no-cache curl jq
|
||
|
|
curl -X POST $SLACK_WEBHOOK \
|
||
|
|
-H 'Content-Type: application/json' \
|
||
|
|
-d '{
|
||
|
|
"text": "✅ VAPORA Kubernetes deployment successful!",
|
||
|
|
"blocks": [
|
||
|
|
{
|
||
|
|
"type": "section",
|
||
|
|
"text": {
|
||
|
|
"type": "mrkdwn",
|
||
|
|
"text": "✅ **VAPORA Kubernetes Deployment Successful**\n\n*Deployments Ready:*\n• backend (vapora-backend)\n• agents (vapora-agents)\n• llm-router (vapora-llm-router)"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "context",
|
||
|
|
"elements": [
|
||
|
|
{
|
||
|
|
"type": "mrkdwn",
|
||
|
|
"text": "*Commit*: '"${CI_COMMIT_SHA:0:8}"'\n*Branch*: '"${CI_COMMIT_BRANCH}"'\n*Namespace*: '"${VAPORA_NAMESPACE}"'\n*Triggered By*: '"${CI_COMMIT_AUTHOR}"'"
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}'
|
||
|
|
else
|
||
|
|
echo "⚠️ Slack webhook not configured"
|
||
|
|
fi
|