#!/bin/bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" # shellcheck source=/dev/null [ -f "${SCRIPT_DIR}/fleet_daemon-lib.sh" ] && source "${SCRIPT_DIR}/fleet_daemon-lib.sh" # shellcheck source=/dev/null [ -f "${SCRIPT_DIR}/diagnostics.sh" ] && source "${SCRIPT_DIR}/diagnostics.sh" # shellcheck source=/dev/null [ -f "${SCRIPT_DIR}/env-fleet_daemon" ] && source "${SCRIPT_DIR}/env-fleet_daemon" # shellcheck source=/dev/null [ -f "${SCRIPT_DIR}/_credentials.env" ] && source "${SCRIPT_DIR}/_credentials.env" # ─── config (env-overridable, with sane defaults) ──────────────────────── FLEET_DAEMON_NAME="${FLEET_DAEMON_NAME:-fleet-daemon}" FLEET_DAEMON_NAMESPACE="${FLEET_DAEMON_NAMESPACE:-fleet-system}" FLEET_DAEMON_IMAGE="${FLEET_DAEMON_IMAGE:-daoreg.librecloud.online/libre-forge/fleet-daemon:0.1.0}" FLEET_DAEMON_REPLICAS="${FLEET_DAEMON_REPLICAS:-1}" FLEET_DAEMON_SERVICE_PORT="${FLEET_DAEMON_SERVICE_PORT:-19012}" FLEET_DAEMON_TLS_SECRET="${FLEET_DAEMON_TLS_SECRET:-fleet-daemon-tls}" # NodePort for the agent WebSocket channel (legacy). 0 = disabled. # Superseded by FLEET_DAEMON_CHANNEL_LB_IP when set. FLEET_DAEMON_CHANNEL_NODEPORT="${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" # Cilium LB-IPAM IP for the agent WebSocket channel. When non-empty the Service # is typed as LoadBalancer and annotated with io.cilium/lb-ipam-ips. This is the # required exposure for off-cluster agents whose packets arrive via WireGuard NAT # (externalTrafficPolicy: Local on NodePort would drop them). Takes precedence # over FLEET_DAEMON_CHANNEL_NODEPORT. FLEET_DAEMON_CHANNEL_LB_IP="${FLEET_DAEMON_CHANNEL_LB_IP:-}" FLEET_DAEMON_CPU_REQ="${FLEET_DAEMON_CPU_REQ:-100m}" FLEET_DAEMON_MEM_REQ="${FLEET_DAEMON_MEM_REQ:-128Mi}" FLEET_DAEMON_CPU_LIM="${FLEET_DAEMON_CPU_LIM:-500m}" FLEET_DAEMON_MEM_LIM="${FLEET_DAEMON_MEM_LIM:-512Mi}" NATS_URL="${NATS_URL:-nats://nats.core-service.svc:4222}" NATS_SUBJECT_PREFIX="${NATS_SUBJECT_PREFIX:-fleet.libre-forge}" FLEET_ID="${FLEET_ID:-lian-builders}" FLEET_NODE_ID="${FLEET_NODE_ID:-lian-01}" FLEET_NODE_IP="${FLEET_NODE_IP:-10.0.10.17}" FLEET_POOL_ID="${FLEET_POOL_ID:-arm64-standard}" FLEET_NODE_ARCH="${FLEET_NODE_ARCH:-arm64}" FLEET_NODE_TIER="${FLEET_NODE_TIER:-standard}" FLEET_NODE_CPU="${FLEET_NODE_CPU:-4}" FLEET_NODE_RAM_GB="${FLEET_NODE_RAM_GB:-8}" FLEET_NODE_DISK_GB="${FLEET_NODE_DISK_GB:-80}" FLEET_BUILDKIT_PORT="${FLEET_BUILDKIT_PORT:-1234}" # baseline == peak disables physical resize (AlwaysOnResize fixed-capacity mode). FLEET_BASELINE_SERVER_TYPE="${FLEET_BASELINE_SERVER_TYPE:-cax21}" FLEET_PEAK_SERVER_TYPE="${FLEET_PEAK_SERVER_TYPE:-cax21}" # Comma-separated node IDs that should also receive install_tls propagation. # Derived from nodes[] excluding the primary node. Example: "lian-02" FLEET_EXTRA_NODE_IDS="${FLEET_EXTRA_NODE_IDS:-}" # TOML [node_overrides.*] sections for non-primary nodes that differ in arch/pool. # Rendered by the workspace recipe from NCL nodes[] (skip 1). May be empty. FLEET_NODE_OVERRIDES_TOML="${FLEET_NODE_OVERRIDES_TOML:-}" # NOTE: bash parameter expansion does brace-matching inside ${VAR:-DEFAULT}; # inlining a JSON object as DEFAULT causes the trailing `}` of the JSON to be # appended OUTSIDE the expansion (root cause of the "trailing characters at # line 14 column 2" CrashLoop in earlier iterations). Keep the default in its # own variable to avoid the mismatch. _DEFAULT_ACCESS_POLICY_JSON='{"claims_accepted_from":[],"max_concurrent_claims_per_workspace":[],"priority_order":[]}' ACCESS_POLICY_JSON="${ACCESS_POLICY_JSON:-$_DEFAULT_ACCESS_POLICY_JSON}" TICK_INTERVAL_SECONDS="${TICK_INTERVAL_SECONDS:-30}" SNAPSHOT_INTERVAL_SECONDS="${SNAPSHOT_INTERVAL_SECONDS:-300}" SNAPSHOT_RETAIN_SECONDS="${SNAPSHOT_RETAIN_SECONDS:-86400}" SNAPSHOT_LOCAL_PATH="${SNAPSHOT_LOCAL_PATH:-/var/lib/fleet-snapshots}" SNAPSHOT_PVC_SIZE="${SNAPSHOT_PVC_SIZE:-5Gi}" SNAPSHOT_PVC_STORAGE_CLASS="${SNAPSHOT_PVC_STORAGE_CLASS:-local-path}" SNAPSHOT_UPLOAD_SCHEDULE="${SNAPSHOT_UPLOAD_SCHEDULE:-*/30 * * * *}" SNAPSHOT_UPLOAD_IMAGE="${SNAPSHOT_UPLOAD_IMAGE:-ghcr.io/oras-project/oras:v1.3.0}" SNAPSHOT_UPLOAD_REGISTRY="${SNAPSHOT_UPLOAD_REGISTRY:-daoreg.librecloud.online/libre-forge/fleet-state}" SNAPSHOT_UPLOAD_ARTIFACT_TYPE="${SNAPSHOT_UPLOAD_ARTIFACT_TYPE:-application/vnd.libre-forge.fleet-state.v1+json}" PKI_ENABLED="${PKI_ENABLED:-true}" PKI_CA_DURATION="${PKI_CA_DURATION:-43800h}" PKI_CA_RENEW_BEFORE="${PKI_CA_RENEW_BEFORE:-720h}" PKI_CA_PARENT_ISSUER="${PKI_CA_PARENT_ISSUER:-buildkit-selfsigned-issuer}" PKI_CA_COMMON_NAME="${PKI_CA_COMMON_NAME:-Libre Daoshi Fleet CA}" PKI_CERT_DURATION="${PKI_CERT_DURATION:-2160h}" PKI_CERT_RENEW_BEFORE="${PKI_CERT_RENEW_BEFORE:-360h}" L2_ANNOUNCEMENT_ENABLED="${L2_ANNOUNCEMENT_ENABLED:-false}" L2_ANNOUNCEMENT_INTERFACES="${L2_ANNOUNCEMENT_INTERFACES:-enp7s0}" NETPOL_ENABLED="${NETPOL_ENABLED:-true}" NETPOL_OPERATOR_INGRESS_CIDRS="${NETPOL_OPERATOR_INGRESS_CIDRS:-10.200.0.0/24}" NETPOL_CLUSTER_INTERNAL_CIDRS="${NETPOL_CLUSTER_INTERNAL_CIDRS:-10.0.0.0/16}" NETPOL_EXTERNAL_EGRESS_HTTPS="${NETPOL_EXTERNAL_EGRESS_HTTPS:-true}" NETPOL_NATS_NAMESPACE="${NETPOL_NATS_NAMESPACE:-core-service}" PROMETHEUS_RULE_ENABLED="${PROMETHEUS_RULE_ENABLED:-true}" PROMETHEUS_RULE_SELECTOR="${PROMETHEUS_RULE_SELECTOR:-kube-prometheus}" CMD_TSK="${CMD_TSK:-${1:-install}}" # Resolve PRVNG_/PROVISIONING_ aliases (when diagnostics.sh isn't present, # fall back to a local truthy check). if command -v _diag_is_force >/dev/null 2>&1; then _diag_is_force && FORCE=true || FORCE=false _diag_is_debug && DEBUG=true || DEBUG=false _diag_is_verbose && VERBOSE=true || VERBOSE=false else case "$(printf '%s' "${PROVISIONING_FORCE:-${PRVNG_FORCE:-}}" | tr '[:upper:]' '[:lower:]')" in 1|true|yes) FORCE=true ;; *) FORCE=false ;; esac case "$(printf '%s' "${PROVISIONING_DEBUG:-${PRVNG_DEBUG:-}}" | tr '[:upper:]' '[:lower:]')" in 1|true|yes) DEBUG=true ;; *) DEBUG=false ;; esac case "$(printf '%s' "${PROVISIONING_VERBOSE:-${PRVNG_VERBOSE:-}}" | tr '[:upper:]' '[:lower:]')" in 1|true|yes) VERBOSE=true ;; *) VERBOSE=false ;; esac fi [ "$FORCE" = "true" ] && echo " [force] enabled — Deployment will be deleted before apply" [ "$DEBUG" = "true" ] && echo " [debug] enabled — extra logging, /tmp bundle preserved on failure" [ "$VERBOSE" = "true" ] && echo " [verbose] enabled — full kubectl apply output" # _log: progress markers, ALWAYS shown (one concise line per phase). They give # essential "where am I" feedback even in quiet mode and are cheap (10 lines # for a full install vs ~30 lines of kubectl noise that VERBOSE adds). _log() { printf ' → %s\n' "$*" } # Convert a comma-separated list of strings to a TOML inline array of quoted strings. # "" → [] "lian-02" → ["lian-02"] "lian-02,lian-03" → ["lian-02","lian-03"] _toml_string_array_from_csv() { local csv="$1" if [ -z "$csv" ]; then echo "[]" return fi local out='[' local first=1 local IFS=',' for item in $csv; do [ $first -eq 0 ] && out+=',' out+='"'"$item"'"' first=0 done echo "${out}]" } # _apply_stdin: read a manifest from stdin and apply via kubectl, suppressing # routine output ("unchanged"/"configured"/"created") when not VERBOSE. Errors # are never hidden — non-zero exit always prints stderr. _apply_stdin() { if [ "$VERBOSE" = "true" ]; then _kubectl apply -f - return $? fi local out rc out=$(_kubectl apply -f - 2>&1) rc=$? if [ $rc -ne 0 ]; then printf '%s\n' "$out" >&2 fi return $rc } _resolve_kubeconfig # Pre-register the workload so the ERR trap can dump diagnostics for ANY # failing kubectl apply (not only the final rollout wait). if command -v _diag_register_cluster_workload >/dev/null 2>&1; then _diag_register_cluster_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" \ "app.kubernetes.io/name=${FLEET_DAEMON_NAME}" _diag_install_err_trap fi # ─── secret materialisation (stdin-only, never to disk) ────────────────── _apply_namespace_and_sa() { _apply_stdin <&2 fi if [ -n "${DAOREG_PUSH_USERNAME:-}" ] && [ -n "${DAOREG_PUSH_PASSWORD:-}" ]; then _kubectl create secret docker-registry daoreg-push-secret \ --namespace "$FLEET_DAEMON_NAMESPACE" \ --docker-server=daoreg.librecloud.online \ --docker-username="${DAOREG_PUSH_USERNAME}" \ --docker-password="${DAOREG_PUSH_PASSWORD}" \ --dry-run=client -o yaml | _apply_stdin else echo " WARN: DAOREG_PUSH_USERNAME/PASSWORD not set — daoreg-push-secret not materialised (snapshot uploader will fail)" >&2 fi } _apply_pki() { [ "$PKI_ENABLED" = "true" ] || { echo " [skip] PKI disabled"; return 0; } if ! _crd_present certificates.cert-manager.io; then echo " WARN: cert-manager CRDs not installed — skipping PKI block" >&2 return 0 fi _apply_stdin </dev/null 2>&1; then local jq_err if ! jq_err=$(printf '%s' "$ACCESS_POLICY_JSON" | jq -e . 2>&1 >/dev/null); then echo "ERROR: access-policy JSON failed local validation — refusing to apply ConfigMap." >&2 echo " jq: ${jq_err}" >&2 if [ "$DEBUG" = "true" ]; then echo " raw value (DEBUG):" >&2 printf ' %s\n' "$ACCESS_POLICY_JSON" >&2 else echo " (re-run with PRVNG_DEBUG=true to see the raw value)" >&2 fi exit 1 fi fi # Serialise as a YAML double-quoted scalar via kubectl create --dry-run, which # handles all escaping correctly regardless of newlines/quotes in the JSON. _kubectl create configmap fleet-daemon-access-policy \ --namespace "${FLEET_DAEMON_NAMESPACE}" \ --from-literal=access-policy.json="${ACCESS_POLICY_JSON}" \ --dry-run=client -o yaml \ | _kubectl label --local --dry-run=client -o yaml -f - \ app.kubernetes.io/name="${FLEET_DAEMON_NAME}" \ app.kubernetes.io/part-of=libre-forge-fleet \ app.kubernetes.io/component=access-policy \ --overwrite \ | _apply_stdin } _apply_snapshot_pvc() { _apply_stdin </dev/null 2>&1; then _diag_force_delete_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" 30 else _kubectl delete deployment "$FLEET_DAEMON_NAME" \ --namespace "$FLEET_DAEMON_NAMESPACE" \ --wait=true --timeout=30s --ignore-not-found 2>&1 || true fi fi _apply_stdin </dev/null || true uploaded=0 skipped=0 for f in snap-*.json; do [ -f "\$f" ] || continue tag="\${f#snap-}" tag="\${tag%.json}" if oras push \\ "${SNAPSHOT_UPLOAD_REGISTRY}:\${tag}" \\ --artifact-type ${SNAPSHOT_UPLOAD_ARTIFACT_TYPE} \\ "\${f}:application/json" 2>/dev/null; then rm -f "\${f}" uploaded=\$((uploaded + 1)) else echo "WARN: failed to push \${f}; leaving on disk for next cycle" >&2 skipped=\$((skipped + 1)) fi done echo "uploaded=\${uploaded} skipped=\${skipped}" volumeMounts: - name: snapshots mountPath: ${SNAPSHOT_LOCAL_PATH} - name: docker-config mountPath: /root/.docker readOnly: true env: - name: DOCKER_CONFIG value: /root/.docker resources: requests: cpu: 50m memory: 64Mi limits: cpu: 200m memory: 256Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: false capabilities: drop: ["ALL"] volumes: - name: snapshots persistentVolumeClaim: claimName: fleet-snapshots - name: docker-config secret: secretName: daoreg-push-secret items: - key: .dockerconfigjson path: config.json MANIFEST } _apply_prometheus_rule() { [ "$PROMETHEUS_RULE_ENABLED" = "true" ] || { echo " [skip] PrometheusRule disabled"; return 0; } if ! _crd_present prometheusrules.monitoring.coreos.com; then echo " WARN: PrometheusRule CRD not installed — skipping alert rules (re-run after installing prometheus-operator)" >&2 return 0 fi _apply_stdin < 0 for: 10m labels: severity: warning workspace: libre-forge annotations: summary: "Invariant {{ \$labels.invariant }} blocked decisions for 10m" - alert: FleetProviderRelayFailing expr: | sum by (op) ( rate(fleet_provider_api_calls_total{namespace="${FLEET_DAEMON_NAMESPACE}",result!~"ok|dry_run"}[5m]) ) > 0.1 for: 10m labels: severity: critical workspace: libre-forge annotations: summary: "Provider relay verb {{ \$labels.op }} failing at > 0.1/s" - alert: FleetApplyFailuresRising expr: | sum by (kind) ( rate(fleet_apply_failures_total{namespace="${FLEET_DAEMON_NAMESPACE}"}[5m]) ) > 0.05 for: 15m labels: severity: warning workspace: libre-forge annotations: summary: "Decision apply path {{ \$labels.kind }} failing" - name: fleet-daemon-capacity interval: 60s rules: - alert: FleetForecastEmpty expr: | fleet_forecast_n{namespace="${FLEET_DAEMON_NAMESPACE}"} == 0 for: 30m labels: severity: warning workspace: libre-forge annotations: summary: "Forecast N=0 for pool {{ \$labels.pool }}" - alert: FleetSnapshotUploaderStalled expr: | time() - kube_cronjob_status_last_successful_time{ namespace="${FLEET_DAEMON_NAMESPACE}", cronjob="fleet-snapshot-uploader" } > 7200 for: 30m labels: severity: warning workspace: libre-forge annotations: summary: "fleet-snapshot-uploader hasn't completed successfully in 2h" MANIFEST } # ─── ops ───────────────────────────────────────────────────────────────── _do_install() { _log "namespace + service account" _apply_namespace_and_sa _log "secrets (nkey + age + registry pull/push)" _apply_secrets _log "PKI (cert-manager CA + Issuer + daemon TLS)" _apply_pki _log "config ConfigMap (TOML, rendered from workspace NCL)" _apply_config_map _log "access policy ConfigMap" _apply_access_policy _log "snapshot PVC" _apply_snapshot_pvc _log "LB-IPAM pool" _apply_lb_pool _log "L2 announcement policy" _apply_l2_announcement _log "deployment + service" _apply_deployment_and_service _log "NetworkPolicy" _apply_networkpolicy _log "snapshot uploader CronJob" _apply_snapshot_uploader _log "PrometheusRule" _apply_prometheus_rule _log "waiting for rollout" _wait_for_deployment 300 echo "fleet_daemon installed in namespace ${FLEET_DAEMON_NAMESPACE}" echo " service : ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc:${FLEET_DAEMON_SERVICE_PORT}" if [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ]; then echo " channel (LB) : ws://${FLEET_DAEMON_CHANNEL_LB_IP}:${FLEET_DAEMON_SERVICE_PORT}/channel (fleet-agent daemon_url)" elif [ "${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" != "0" ]; then echo " channel (NP) : ws://:${FLEET_DAEMON_CHANNEL_NODEPORT}/channel (fleet-agent daemon_url, legacy)" fi echo " nats : ${NATS_URL} (prefix=${NATS_SUBJECT_PREFIX})" echo " image : ${FLEET_DAEMON_IMAGE}" } _do_update() { _apply_pki _apply_config_map _apply_access_policy _apply_lb_pool _apply_l2_announcement _apply_networkpolicy _apply_deployment_and_service _kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" _wait_for_deployment 300 echo "fleet_daemon updated" } _do_delete() { _kubectl delete deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete service/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete cronjob/fleet-snapshot-uploader --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete configmap/fleet-daemon-access-policy --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete configmap/fleet-daemon-config --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete networkpolicy default-deny-all fleet-daemon-egress fleet-daemon-ingress \ --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete ciliumloadbalancerippool fleet-daemon-pool --ignore-not-found _kubectl delete ciliuml2announcementpolicy "${FLEET_DAEMON_NAME}-l2" --ignore-not-found echo "fleet_daemon workload removed (PVC + secrets + namespace preserved)" } _do_purge() { if [ "${1:-}" != "--confirm" ]; then echo "ERROR: purge requires --confirm" >&2 exit 1 fi _do_delete _kubectl delete pvc/fleet-snapshots --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete secret/forge-control-nkey-creds secret/fleet-daemon-age-key \ secret/daoreg-pull-secret secret/daoreg-push-secret \ --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found if _crd_present certificates.cert-manager.io; then _kubectl delete certificate/fleet-mtls-ca certificate/"$FLEET_DAEMON_TLS_SECRET" issuer/fleet-ca-issuer \ --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found _kubectl delete secret/fleet-mtls-ca secret/"$FLEET_DAEMON_TLS_SECRET" \ --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found fi if _crd_present prometheusrules.monitoring.coreos.com; then _kubectl delete prometheusrule/fleet-daemon-alerts \ --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found fi _kubectl delete namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found echo "fleet_daemon purged" } _do_health() { if ! _deployment_exists; then echo "UNHEALTHY: deployment ${FLEET_DAEMON_NAME} missing in ${FLEET_DAEMON_NAMESPACE}" >&2 exit 1 fi local ready_replicas desired_replicas ready_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \ -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo 0) desired_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \ -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0) if [ "${ready_replicas:-0}" = "${desired_replicas:-0}" ] && [ "${ready_replicas:-0}" -gt 0 ]; then echo "HEALTHY: ${ready_replicas}/${desired_replicas} replicas ready" else echo "UNHEALTHY: ${ready_replicas:-0}/${desired_replicas:-0} replicas ready" >&2 _kubectl get pods --namespace "$FLEET_DAEMON_NAMESPACE" -l app.kubernetes.io/name="$FLEET_DAEMON_NAME" >&2 exit 1 fi } _do_restart() { _kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" _wait_for_deployment 300 echo "fleet_daemon restarted" } case "$CMD_TSK" in install) _do_install ;; update) _do_update ;; delete) _do_delete ;; purge) shift; _do_purge "${1:-}" ;; health) _do_health ;; restart) _do_restart ;; *) echo "ERROR: unknown CMD_TSK='$CMD_TSK'. Valid: install|update|delete|purge|health|restart" >&2 exit 1 ;; esac