provisioning-catalog/components/fleet_daemon/cluster/install-fleet_daemon.sh

#!/bin/bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

# shellcheck source=/dev/null
[ -f "${SCRIPT_DIR}/fleet_daemon-lib.sh" ] && source "${SCRIPT_DIR}/fleet_daemon-lib.sh"
# shellcheck source=/dev/null
[ -f "${SCRIPT_DIR}/diagnostics.sh" ]      && source "${SCRIPT_DIR}/diagnostics.sh"
# shellcheck source=/dev/null
[ -f "${SCRIPT_DIR}/env-fleet_daemon" ]    && source "${SCRIPT_DIR}/env-fleet_daemon"
# shellcheck source=/dev/null
[ -f "${SCRIPT_DIR}/_credentials.env" ]    && source "${SCRIPT_DIR}/_credentials.env"

# ─── config (env-overridable, with sane defaults) ────────────────────────
FLEET_DAEMON_NAME="${FLEET_DAEMON_NAME:-fleet-daemon}"
FLEET_DAEMON_NAMESPACE="${FLEET_DAEMON_NAMESPACE:-fleet-system}"
FLEET_DAEMON_IMAGE="${FLEET_DAEMON_IMAGE:-daoreg.librecloud.online/libre-forge/fleet-daemon:0.1.0}"
FLEET_DAEMON_REPLICAS="${FLEET_DAEMON_REPLICAS:-1}"
FLEET_DAEMON_SERVICE_PORT="${FLEET_DAEMON_SERVICE_PORT:-19012}"
FLEET_DAEMON_TLS_SECRET="${FLEET_DAEMON_TLS_SECRET:-fleet-daemon-tls}"
# NodePort for the agent WebSocket channel (legacy). 0 = disabled.
# Superseded by FLEET_DAEMON_CHANNEL_LB_IP when set.
FLEET_DAEMON_CHANNEL_NODEPORT="${FLEET_DAEMON_CHANNEL_NODEPORT:-0}"
# Cilium LB-IPAM IP for the agent WebSocket channel. When non-empty the Service
# is typed as LoadBalancer and annotated with io.cilium/lb-ipam-ips. This is the
# required exposure for off-cluster agents whose packets arrive via WireGuard NAT
# (externalTrafficPolicy: Local on NodePort would drop them). Takes precedence
# over FLEET_DAEMON_CHANNEL_NODEPORT.
FLEET_DAEMON_CHANNEL_LB_IP="${FLEET_DAEMON_CHANNEL_LB_IP:-}"

FLEET_DAEMON_CPU_REQ="${FLEET_DAEMON_CPU_REQ:-100m}"
FLEET_DAEMON_MEM_REQ="${FLEET_DAEMON_MEM_REQ:-128Mi}"
FLEET_DAEMON_CPU_LIM="${FLEET_DAEMON_CPU_LIM:-500m}"
FLEET_DAEMON_MEM_LIM="${FLEET_DAEMON_MEM_LIM:-512Mi}"

NATS_URL="${NATS_URL:-nats://nats.core-service.svc:4222}"
NATS_SUBJECT_PREFIX="${NATS_SUBJECT_PREFIX:-fleet.libre-forge}"

FLEET_ID="${FLEET_ID:-lian-builders}"
FLEET_NODE_ID="${FLEET_NODE_ID:-lian-01}"
FLEET_NODE_IP="${FLEET_NODE_IP:-10.0.10.17}"
FLEET_POOL_ID="${FLEET_POOL_ID:-arm64-standard}"
FLEET_NODE_ARCH="${FLEET_NODE_ARCH:-arm64}"
FLEET_NODE_TIER="${FLEET_NODE_TIER:-standard}"
FLEET_NODE_CPU="${FLEET_NODE_CPU:-4}"
FLEET_NODE_RAM_GB="${FLEET_NODE_RAM_GB:-8}"
FLEET_NODE_DISK_GB="${FLEET_NODE_DISK_GB:-80}"
FLEET_BUILDKIT_PORT="${FLEET_BUILDKIT_PORT:-1234}"
# baseline == peak disables physical resize (AlwaysOnResize fixed-capacity mode).
FLEET_BASELINE_SERVER_TYPE="${FLEET_BASELINE_SERVER_TYPE:-cax21}"
FLEET_PEAK_SERVER_TYPE="${FLEET_PEAK_SERVER_TYPE:-cax21}"
# Comma-separated node IDs that should also receive install_tls propagation.
# Derived from nodes[] excluding the primary node. Example: "lian-02"
FLEET_EXTRA_NODE_IDS="${FLEET_EXTRA_NODE_IDS:-}"
# TOML [node_overrides.*] sections for non-primary nodes that differ in arch/pool.
# Rendered by the workspace recipe from NCL nodes[] (skip 1). May be empty.
FLEET_NODE_OVERRIDES_TOML="${FLEET_NODE_OVERRIDES_TOML:-}"

# NOTE: bash parameter expansion does brace-matching inside ${VAR:-DEFAULT};
# inlining a JSON object as DEFAULT causes the trailing `}` of the JSON to be
# appended OUTSIDE the expansion (root cause of the "trailing characters at
# line 14 column 2" CrashLoop in earlier iterations). Keep the default in its
# own variable to avoid the mismatch.
_DEFAULT_ACCESS_POLICY_JSON='{"claims_accepted_from":[],"max_concurrent_claims_per_workspace":[],"priority_order":[]}'
ACCESS_POLICY_JSON="${ACCESS_POLICY_JSON:-$_DEFAULT_ACCESS_POLICY_JSON}"

TICK_INTERVAL_SECONDS="${TICK_INTERVAL_SECONDS:-30}"
SNAPSHOT_INTERVAL_SECONDS="${SNAPSHOT_INTERVAL_SECONDS:-300}"
SNAPSHOT_RETAIN_SECONDS="${SNAPSHOT_RETAIN_SECONDS:-86400}"
SNAPSHOT_LOCAL_PATH="${SNAPSHOT_LOCAL_PATH:-/var/lib/fleet-snapshots}"
SNAPSHOT_PVC_SIZE="${SNAPSHOT_PVC_SIZE:-5Gi}"
SNAPSHOT_PVC_STORAGE_CLASS="${SNAPSHOT_PVC_STORAGE_CLASS:-local-path}"
SNAPSHOT_UPLOAD_SCHEDULE="${SNAPSHOT_UPLOAD_SCHEDULE:-*/30 * * * *}"
SNAPSHOT_UPLOAD_IMAGE="${SNAPSHOT_UPLOAD_IMAGE:-ghcr.io/oras-project/oras:v1.3.0}"
SNAPSHOT_UPLOAD_REGISTRY="${SNAPSHOT_UPLOAD_REGISTRY:-daoreg.librecloud.online/libre-forge/fleet-state}"
SNAPSHOT_UPLOAD_ARTIFACT_TYPE="${SNAPSHOT_UPLOAD_ARTIFACT_TYPE:-application/vnd.libre-forge.fleet-state.v1+json}"

PKI_ENABLED="${PKI_ENABLED:-true}"
PKI_CA_DURATION="${PKI_CA_DURATION:-43800h}"
PKI_CA_RENEW_BEFORE="${PKI_CA_RENEW_BEFORE:-720h}"
PKI_CA_PARENT_ISSUER="${PKI_CA_PARENT_ISSUER:-buildkit-selfsigned-issuer}"
PKI_CA_COMMON_NAME="${PKI_CA_COMMON_NAME:-Libre Daoshi Fleet CA}"
PKI_CERT_DURATION="${PKI_CERT_DURATION:-2160h}"
PKI_CERT_RENEW_BEFORE="${PKI_CERT_RENEW_BEFORE:-360h}"

L2_ANNOUNCEMENT_ENABLED="${L2_ANNOUNCEMENT_ENABLED:-false}"
L2_ANNOUNCEMENT_INTERFACES="${L2_ANNOUNCEMENT_INTERFACES:-enp7s0}"

NETPOL_ENABLED="${NETPOL_ENABLED:-true}"
NETPOL_OPERATOR_INGRESS_CIDRS="${NETPOL_OPERATOR_INGRESS_CIDRS:-10.200.0.0/24}"
NETPOL_CLUSTER_INTERNAL_CIDRS="${NETPOL_CLUSTER_INTERNAL_CIDRS:-10.0.0.0/16}"
NETPOL_EXTERNAL_EGRESS_HTTPS="${NETPOL_EXTERNAL_EGRESS_HTTPS:-true}"
NETPOL_NATS_NAMESPACE="${NETPOL_NATS_NAMESPACE:-core-service}"

PROMETHEUS_RULE_ENABLED="${PROMETHEUS_RULE_ENABLED:-true}"
PROMETHEUS_RULE_SELECTOR="${PROMETHEUS_RULE_SELECTOR:-kube-prometheus}"

CMD_TSK="${CMD_TSK:-${1:-install}}"

# Resolve PRVNG_/PROVISIONING_ aliases (when diagnostics.sh isn't present,
# fall back to a local truthy check).
if command -v _diag_is_force >/dev/null 2>&1; then
    _diag_is_force   && FORCE=true   || FORCE=false
    _diag_is_debug   && DEBUG=true   || DEBUG=false
    _diag_is_verbose && VERBOSE=true || VERBOSE=false
else
    case "$(printf '%s' "${PROVISIONING_FORCE:-${PRVNG_FORCE:-}}" | tr '[:upper:]' '[:lower:]')" in
        1|true|yes) FORCE=true ;; *) FORCE=false ;;
    esac
    case "$(printf '%s' "${PROVISIONING_DEBUG:-${PRVNG_DEBUG:-}}" | tr '[:upper:]' '[:lower:]')" in
        1|true|yes) DEBUG=true ;; *) DEBUG=false ;;
    esac
    case "$(printf '%s' "${PROVISIONING_VERBOSE:-${PRVNG_VERBOSE:-}}" | tr '[:upper:]' '[:lower:]')" in
        1|true|yes) VERBOSE=true ;; *) VERBOSE=false ;;
    esac
fi
[ "$FORCE" = "true" ]   && echo "  [force] enabled — Deployment will be deleted before apply"
[ "$DEBUG" = "true" ]   && echo "  [debug] enabled — extra logging, /tmp bundle preserved on failure"
[ "$VERBOSE" = "true" ] && echo "  [verbose] enabled — full kubectl apply output"

# _log: progress markers, ALWAYS shown (one concise line per phase). They give
# essential "where am I" feedback even in quiet mode and are cheap (10 lines
# for a full install vs ~30 lines of kubectl noise that VERBOSE adds).
_log() {
  printf '  → %s\n' "$*"
}
# Convert a comma-separated list of strings to a TOML inline array of quoted strings.
# "" → []   "lian-02" → ["lian-02"]   "lian-02,lian-03" → ["lian-02","lian-03"]
_toml_string_array_from_csv() {
  local csv="$1"
  if [ -z "$csv" ]; then
    echo "[]"
    return
  fi
  local out='['
  local first=1
  local IFS=','
  for item in $csv; do
    [ $first -eq 0 ] && out+=','
    out+='"'"$item"'"'
    first=0
  done
  echo "${out}]"
}
# _apply_stdin: read a manifest from stdin and apply via kubectl, suppressing
# routine output ("unchanged"/"configured"/"created") when not VERBOSE. Errors
# are never hidden — non-zero exit always prints stderr.
_apply_stdin() {
  if [ "$VERBOSE" = "true" ]; then
    _kubectl apply -f -
    return $?
  fi
  local out rc
  out=$(_kubectl apply -f - 2>&1)
  rc=$?
  if [ $rc -ne 0 ]; then
    printf '%s\n' "$out" >&2
  fi
  return $rc
}

_resolve_kubeconfig

# Pre-register the workload so the ERR trap can dump diagnostics for ANY
# failing kubectl apply (not only the final rollout wait).
if command -v _diag_register_cluster_workload >/dev/null 2>&1; then
    _diag_register_cluster_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" \
        "app.kubernetes.io/name=${FLEET_DAEMON_NAME}"
    _diag_install_err_trap
fi

# ─── secret materialisation (stdin-only, never to disk) ──────────────────

_apply_namespace_and_sa() {
  _apply_stdin <<MANIFEST
apiVersion: v1
kind: Namespace
metadata:
  name: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/managed-by: provisioning
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: ${FLEET_DAEMON_NAME}
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fleet-snapshot-uploader
  namespace: ${FLEET_DAEMON_NAMESPACE}
automountServiceAccountToken: false
MANIFEST
}

_apply_secrets() {
  _require_env FORGE_CONTROL_SEED
  _require_env FLEET_AGE_KEY

  _kubectl create secret generic forge-control-nkey-creds \
    --namespace "$FLEET_DAEMON_NAMESPACE" \
    --from-literal=seed="${FORGE_CONTROL_SEED}" \
    --dry-run=client -o yaml | _apply_stdin

  _kubectl create secret generic fleet-daemon-age-key \
    --namespace "$FLEET_DAEMON_NAMESPACE" \
    --from-literal=age.key="${FLEET_AGE_KEY}" \
    --dry-run=client -o yaml | _apply_stdin

  if [ -n "${DAOREG_PULL_USERNAME:-}" ] && [ -n "${DAOREG_PULL_PASSWORD:-}" ]; then
    _kubectl create secret docker-registry daoreg-pull-secret \
      --namespace "$FLEET_DAEMON_NAMESPACE" \
      --docker-server=daoreg.librecloud.online \
      --docker-username="${DAOREG_PULL_USERNAME}" \
      --docker-password="${DAOREG_PULL_PASSWORD}" \
      --dry-run=client -o yaml | _apply_stdin
  else
    echo "  WARN: DAOREG_PULL_USERNAME/PASSWORD not set — daoreg-pull-secret not materialised (image pulls may fail)" >&2
  fi

  if [ -n "${DAOREG_PUSH_USERNAME:-}" ] && [ -n "${DAOREG_PUSH_PASSWORD:-}" ]; then
    _kubectl create secret docker-registry daoreg-push-secret \
      --namespace "$FLEET_DAEMON_NAMESPACE" \
      --docker-server=daoreg.librecloud.online \
      --docker-username="${DAOREG_PUSH_USERNAME}" \
      --docker-password="${DAOREG_PUSH_PASSWORD}" \
      --dry-run=client -o yaml | _apply_stdin
  else
    echo "  WARN: DAOREG_PUSH_USERNAME/PASSWORD not set — daoreg-push-secret not materialised (snapshot uploader will fail)" >&2
  fi
}

_apply_pki() {
  [ "$PKI_ENABLED" = "true" ] || { echo "  [skip] PKI disabled"; return 0; }

  if ! _crd_present certificates.cert-manager.io; then
    echo "  WARN: cert-manager CRDs not installed — skipping PKI block" >&2
    return 0
  fi

  _apply_stdin <<MANIFEST
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: fleet-mtls-ca
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  isCA: true
  commonName: ${PKI_CA_COMMON_NAME}
  duration: ${PKI_CA_DURATION}
  renewBefore: ${PKI_CA_RENEW_BEFORE}
  secretName: fleet-mtls-ca
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: ${PKI_CA_PARENT_ISSUER}
    kind: ClusterIssuer
    group: cert-manager.io
  usages:
    - cert sign
    - crl sign
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: fleet-ca-issuer
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  ca:
    secretName: fleet-mtls-ca
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: ${FLEET_DAEMON_TLS_SECRET}
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  secretName: ${FLEET_DAEMON_TLS_SECRET}
  duration: ${PKI_CERT_DURATION}
  renewBefore: ${PKI_CERT_RENEW_BEFORE}
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: fleet-ca-issuer
    kind: Issuer
    group: cert-manager.io
  commonName: ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc
  dnsNames:
    - ${FLEET_DAEMON_NAME}
    - ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}
    - ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc
    - ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc.cluster.local
  usages:
    - server auth
    - digital signature
    - key encipherment
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: fleet-buildkit-tls
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  secretName: fleet-buildkit-tls
  duration: ${PKI_CERT_DURATION}
  renewBefore: ${PKI_CERT_RENEW_BEFORE}
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: fleet-ca-issuer
    kind: Issuer
    group: cert-manager.io
  commonName: fleet-buildkit
  usages:
    - server auth
    - client auth
    - digital signature
    - key encipherment
MANIFEST
}

_apply_config_map() {
  # Render config.toml from env vars (which themselves come from NCL via the
  # workspace recipe) + apply as ConfigMap `fleet-daemon-config`. The pod
  # mounts it at /etc/fleet-daemon/config.toml; the binary reads at startup.
  local toml
  toml=$(cat <<TOML
# Auto-generated by install-fleet_daemon.sh from workspace NCL.
fleet_id              = "${FLEET_ID}"
claim_ttl_seconds     = ${FLEET_CLAIM_TTL_SECONDS:-3600}
http_bind_addr        = "0.0.0.0:${FLEET_DAEMON_SERVICE_PORT}"
tick_interval_seconds = ${TICK_INTERVAL_SECONDS}
dry_run               = ${FLEET_DRY_RUN:-false}
access_policy_path    = "/etc/fleet-daemon/access-policy.json"
nkey_seed_file        = "/var/run/secrets/fleet-daemon-nkey/seed"
channel_token_path    = "/var/run/secrets/channel-token/token"
extra_node_ids        = $(_toml_string_array_from_csv "${FLEET_EXTRA_NODE_IDS}")

[nats]
url = "${NATS_URL}"

[node]
node_id             = "${FLEET_NODE_ID}"
private_ip          = "${FLEET_NODE_IP}"
buildkit_port           = ${FLEET_BUILDKIT_PORT}
buildkit_probe_enabled  = ${FLEET_BUILDKIT_PROBE_ENABLED}
pool_id             = "${FLEET_POOL_ID}"
arch                = "${FLEET_NODE_ARCH}"
tier                = "${FLEET_NODE_TIER}"
declared_cpu        = ${FLEET_NODE_CPU}
declared_ram_gb     = ${FLEET_NODE_RAM_GB}
declared_disk_gb    = ${FLEET_NODE_DISK_GB}
mtls_server_subject      = "CN=fleet-buildkit"
baseline_server_type     = "${FLEET_BASELINE_SERVER_TYPE}"
peak_server_type         = "${FLEET_PEAK_SERVER_TYPE}"

[snapshot]
interval_seconds = ${SNAPSHOT_INTERVAL_SECONDS}
local_path       = "${SNAPSHOT_LOCAL_PATH}"
retain_seconds   = ${SNAPSHOT_RETAIN_SECONDS}
TOML
)
  if [ -n "${FLEET_NODE_OVERRIDES_TOML}" ]; then
    toml="${toml}
${FLEET_NODE_OVERRIDES_TOML}"
  fi
  _kubectl create configmap fleet-daemon-config \
    --namespace "${FLEET_DAEMON_NAMESPACE}" \
    --from-literal=config.toml="${toml}" \
    --dry-run=client -o yaml | _apply_stdin
}

_apply_access_policy() {
  # Validate JSON shape locally before sending to the cluster. Catches malformed
  # ACCESS_POLICY_JSON at the bash boundary instead of inside the daemon at
  # startup (which manifests as CrashLoopBackOff with serde_json parse error).
  if command -v jq >/dev/null 2>&1; then
    local jq_err
    if ! jq_err=$(printf '%s' "$ACCESS_POLICY_JSON" | jq -e . 2>&1 >/dev/null); then
      echo "ERROR: access-policy JSON failed local validation — refusing to apply ConfigMap." >&2
      echo "  jq: ${jq_err}" >&2
      if [ "$DEBUG" = "true" ]; then
        echo "  raw value (DEBUG):" >&2
        printf '    %s\n' "$ACCESS_POLICY_JSON" >&2
      else
        echo "  (re-run with PRVNG_DEBUG=true to see the raw value)" >&2
      fi
      exit 1
    fi
  fi
  # Serialise as a YAML double-quoted scalar via kubectl create --dry-run, which
  # handles all escaping correctly regardless of newlines/quotes in the JSON.
  _kubectl create configmap fleet-daemon-access-policy \
    --namespace "${FLEET_DAEMON_NAMESPACE}" \
    --from-literal=access-policy.json="${ACCESS_POLICY_JSON}" \
    --dry-run=client -o yaml \
  | _kubectl label --local --dry-run=client -o yaml -f - \
      app.kubernetes.io/name="${FLEET_DAEMON_NAME}" \
      app.kubernetes.io/part-of=libre-forge-fleet \
      app.kubernetes.io/component=access-policy \
      --overwrite \
  | _apply_stdin
}

_apply_snapshot_pvc() {
  _apply_stdin <<MANIFEST
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: fleet-snapshots
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/component: snapshots
spec:
  accessModes: ["ReadWriteOnce"]
  storageClassName: ${SNAPSHOT_PVC_STORAGE_CLASS}
  resources:
    requests:
      storage: ${SNAPSHOT_PVC_SIZE}
MANIFEST
}

_apply_deployment_and_service() {
  # Pre-compute Service type for the channel endpoint:
  #   FLEET_DAEMON_CHANNEL_LB_IP set → LoadBalancer (Cilium LB-IPAM, preferred)
  #   FLEET_DAEMON_CHANNEL_NODEPORT != 0 → NodePort (legacy; fails for WireGuard-NATted agents)
  #   Neither → ClusterIP (cluster-internal only)
  local svc_type="ClusterIP"
  local svc_nodeport_line=""
  local svc_etp_line=""
  local svc_annotations_block=""
  local svc_lb_ip_spec=""
  if [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ]; then
    svc_type="LoadBalancer"
    svc_annotations_block="  annotations:
    io.cilium/lb-ipam-ips: \"${FLEET_DAEMON_CHANNEL_LB_IP}\""
    svc_lb_ip_spec="  loadBalancerIP: ${FLEET_DAEMON_CHANNEL_LB_IP}"
  elif [ "${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" != "0" ]; then
    svc_type="NodePort"
    svc_nodeport_line="      nodePort: ${FLEET_DAEMON_CHANNEL_NODEPORT}"
    svc_etp_line="  externalTrafficPolicy: Local"
  fi
  if [ "$FORCE" = "true" ]; then
    if command -v _diag_force_delete_workload >/dev/null 2>&1; then
      _diag_force_delete_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" 30
    else
      _kubectl delete deployment "$FLEET_DAEMON_NAME" \
        --namespace "$FLEET_DAEMON_NAMESPACE" \
        --wait=true --timeout=30s --ignore-not-found 2>&1 || true
    fi
  fi
  _apply_stdin <<MANIFEST
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${FLEET_DAEMON_NAME}
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/version: "${FLEET_DAEMON_IMAGE##*:}"
spec:
  replicas: ${FLEET_DAEMON_REPLICAS}
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
        app.kubernetes.io/part-of: libre-forge-fleet
        app.kubernetes.io/version: "${FLEET_DAEMON_IMAGE##*:}"
    spec:
      serviceAccountName: ${FLEET_DAEMON_NAME}
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 65532
        runAsGroup: 65532
        fsGroup: 65532
        # OnRootMismatch (the default) only chgrp's when the volume's root dir
        # already has the right group. local-path-provisioner creates dirs as
        # uid:gid 1000:1000, so OnRootMismatch correctly detects the mismatch
        # but in practice some kubelet versions still skip the recursive walk
        # (observed: snapshot writes fail with "permission denied" on a fresh
        # PVC). Always = unconditional recursive chgrp/chmod on every mount —
        # slower for huge volumes, correct for small ones like our snapshots PVC.
        fsGroupChangePolicy: Always
        seccompProfile:
          type: RuntimeDefault
      imagePullSecrets:
        - name: daoreg-pull-secret
      containers:
        - name: ${FLEET_DAEMON_NAME}
          image: ${FLEET_DAEMON_IMAGE}
          # Always re-pull so a re-pushed tag (same :0.1.0) takes effect on
          # rolling restart. Cost = one image fetch per restart; benefit =
          # zero risk of running cached older binaries from local containerd.
          imagePullPolicy: Always
          ports:
            - name: http
              containerPort: ${FLEET_DAEMON_SERVICE_PORT}
              protocol: TCP
          env:
            - name: RUST_LOG
              value: "fleet_daemon=info,async_nats=warn"
            - name: FLEET_DAEMON_CONFIG
              value: "/etc/fleet-daemon/config.toml"
            - name: SOPS_AGE_KEY_FILE
              value: "/var/run/secrets/fleet-daemon-age-key/age.key"
          volumeMounts:
            - name: config
              mountPath: /etc/fleet-daemon/config.toml
              subPath: config.toml
              readOnly: true
            - name: access-policy
              mountPath: /etc/fleet-daemon/access-policy.json
              subPath: access-policy.json
              readOnly: true
            - name: nkey-creds
              mountPath: /var/run/secrets/fleet-daemon-nkey
              readOnly: true
            - name: age-key
              mountPath: /var/run/secrets/fleet-daemon-age-key
              readOnly: true
            - name: tls
              mountPath: /etc/fleet-daemon/buildkit-tls
              readOnly: true
            - name: channel-token
              mountPath: /var/run/secrets/channel-token
              readOnly: true
            - name: snapshots
              mountPath: ${SNAPSHOT_LOCAL_PATH}
          readinessProbe:
            httpGet:
              path: /readyz
              port: http
              scheme: HTTP
            initialDelaySeconds: 3
            periodSeconds: 5
            timeoutSeconds: 1
            failureThreshold: 3
          livenessProbe:
            httpGet:
              path: /healthz
              port: http
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 15
            timeoutSeconds: 1
            failureThreshold: 5
          resources:
            requests:
              cpu: ${FLEET_DAEMON_CPU_REQ}
              memory: ${FLEET_DAEMON_MEM_REQ}
            limits:
              cpu: ${FLEET_DAEMON_CPU_LIM}
              memory: ${FLEET_DAEMON_MEM_LIM}
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: config
          configMap:
            name: fleet-daemon-config
            items:
              - key: config.toml
                path: config.toml
        - name: access-policy
          configMap:
            name: fleet-daemon-access-policy
            items:
              - key: access-policy.json
                path: access-policy.json
        - name: nkey-creds
          secret:
            secretName: forge-control-nkey-creds
            defaultMode: 0400
            items:
              - key: seed
                path: seed
        - name: age-key
          secret:
            secretName: fleet-daemon-age-key
            defaultMode: 0400
            items:
              - key: age.key
                path: age.key
        - name: tls
          secret:
            secretName: fleet-buildkit-tls
            defaultMode: 0400
            optional: true
        - name: channel-token
          secret:
            secretName: fleet-channel-token
            defaultMode: 0400
            items:
              - key: token
                path: token
        - name: snapshots
          persistentVolumeClaim:
            claimName: fleet-snapshots
---
apiVersion: v1
kind: Service
metadata:
  name: ${FLEET_DAEMON_NAME}
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
${svc_annotations_block}
spec:
  type: ${svc_type}
${svc_lb_ip_spec}
${svc_etp_line}
  selector:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
  ports:
    - name: http
      port: ${FLEET_DAEMON_SERVICE_PORT}
      targetPort: http
      protocol: TCP
${svc_nodeport_line}
MANIFEST
}

_apply_lb_pool() {
  [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ] || { echo "  [skip] LB pool: no channel_lb_ip set"; return 0; }
  _apply_stdin <<POOL
apiVersion: cilium.io/v2
kind: CiliumLoadBalancerIPPool
metadata:
  name: fleet-daemon-pool
  labels:
    app.kubernetes.io/name: fleet-daemon
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/managed-by: provisioning
spec:
  blocks:
  - cidr: "${FLEET_DAEMON_CHANNEL_LB_IP}/32"
  serviceSelector:
    matchLabels:
      app.kubernetes.io/name: fleet-daemon
POOL
}

_apply_l2_announcement() {
  [ "$L2_ANNOUNCEMENT_ENABLED" = "true" ] || { echo "  [skip] L2 announcement: disabled (set L2_ANNOUNCEMENT_ENABLED=true when Cilium l2announcements.enabled=true)"; return 0; }
  local iface_entries=""
  for iface in ${L2_ANNOUNCEMENT_INTERFACES}; do
    iface_entries="${iface_entries}  - \"${iface}\"\n"
  done
  _apply_stdin <<L2ANN
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
  name: ${FLEET_DAEMON_NAME}-l2
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/managed-by: provisioning
spec:
  serviceSelector:
    matchLabels:
      app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
  nodeSelector:
    matchExpressions:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
  interfaces:
$(printf "%s" "${iface_entries}" | sed 's/\\n/\n/g')
  externalIPs: false
  loadBalancerIPs: true
L2ANN
}

_apply_networkpolicy() {
  [ "$NETPOL_ENABLED" = "true" ] || { echo "  [skip] NetworkPolicy disabled"; return 0; }

  local op_ingress cluster_internal external_block
  # shellcheck disable=SC2086
  op_ingress=$(_cidrs_yaml_array "        " ${NETPOL_OPERATOR_INGRESS_CIDRS})
  # shellcheck disable=SC2086
  cluster_internal=$(_cidrs_yaml_array "        " ${NETPOL_CLUSTER_INTERNAL_CIDRS})

  if [ "$NETPOL_EXTERNAL_EGRESS_HTTPS" = "true" ]; then
    external_block=$(cat <<'BLK'
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
            except: ["169.254.0.0/16", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
      ports:
        - port: 443
          protocol: TCP
BLK
)
  else
    external_block=""
  fi

  _apply_stdin <<MANIFEST
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  podSelector: {}
  policyTypes: [Ingress, Egress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: fleet-daemon-egress
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
  policyTypes: [Egress]
  egress:
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: kube-system
          podSelector:
            matchLabels:
              k8s-app: kube-dns
      ports:
        - port: 53
          protocol: UDP
        - port: 53
          protocol: TCP
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: ${NETPOL_NATS_NAMESPACE}
          podSelector:
            matchLabels:
              app.kubernetes.io/name: nats
      ports:
        - port: 4222
          protocol: TCP
    - to:
${cluster_internal}
      ports:
        - port: 443
          protocol: TCP
    - to:
${cluster_internal}
      ports:
        - port: ${FLEET_BUILDKIT_PORT}
          protocol: TCP
${external_block}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: fleet-daemon-ingress
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
    app.kubernetes.io/part-of: libre-forge-fleet
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
  policyTypes: [Ingress]
  ingress:
    - from:
${op_ingress}
      ports:
        - port: ${FLEET_DAEMON_SERVICE_PORT}
          protocol: TCP
MANIFEST
}

_apply_snapshot_uploader() {
  _apply_stdin <<MANIFEST
apiVersion: batch/v1
kind: CronJob
metadata:
  name: fleet-snapshot-uploader
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
    app.kubernetes.io/component: snapshots
spec:
  schedule: "${SNAPSHOT_UPLOAD_SCHEDULE}"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      ttlSecondsAfterFinished: 1800
      template:
        metadata:
          labels:
            app.kubernetes.io/name: fleet-snapshot-uploader
            app.kubernetes.io/part-of: libre-forge-fleet
        spec:
          serviceAccountName: fleet-snapshot-uploader
          automountServiceAccountToken: false
          restartPolicy: OnFailure
          securityContext:
            runAsNonRoot: true
            runAsUser: 1000
            runAsGroup: 1000
            fsGroup: 1000
            seccompProfile:
              type: RuntimeDefault
          imagePullSecrets:
            - name: daoreg-pull-secret
          containers:
            - name: uploader
              image: ${SNAPSHOT_UPLOAD_IMAGE}
              imagePullPolicy: IfNotPresent
              command:
                - /bin/sh
                - -c
                - |
                  set -eu
                  cd ${SNAPSHOT_LOCAL_PATH}
                  shopt -s nullglob 2>/dev/null || true
                  uploaded=0
                  skipped=0
                  for f in snap-*.json; do
                    [ -f "\$f" ] || continue
                    tag="\${f#snap-}"
                    tag="\${tag%.json}"
                    if oras push \\
                         "${SNAPSHOT_UPLOAD_REGISTRY}:\${tag}" \\
                         --artifact-type ${SNAPSHOT_UPLOAD_ARTIFACT_TYPE} \\
                         "\${f}:application/json" 2>/dev/null; then
                      rm -f "\${f}"
                      uploaded=\$((uploaded + 1))
                    else
                      echo "WARN: failed to push \${f}; leaving on disk for next cycle" >&2
                      skipped=\$((skipped + 1))
                    fi
                  done
                  echo "uploaded=\${uploaded} skipped=\${skipped}"
              volumeMounts:
                - name: snapshots
                  mountPath: ${SNAPSHOT_LOCAL_PATH}
                - name: docker-config
                  mountPath: /root/.docker
                  readOnly: true
              env:
                - name: DOCKER_CONFIG
                  value: /root/.docker
              resources:
                requests:
                  cpu: 50m
                  memory: 64Mi
                limits:
                  cpu: 200m
                  memory: 256Mi
              securityContext:
                allowPrivilegeEscalation: false
                readOnlyRootFilesystem: false
                capabilities:
                  drop: ["ALL"]
          volumes:
            - name: snapshots
              persistentVolumeClaim:
                claimName: fleet-snapshots
            - name: docker-config
              secret:
                secretName: daoreg-push-secret
                items:
                  - key: .dockerconfigjson
                    path: config.json
MANIFEST
}

_apply_prometheus_rule() {
  [ "$PROMETHEUS_RULE_ENABLED" = "true" ] || { echo "  [skip] PrometheusRule disabled"; return 0; }

  if ! _crd_present prometheusrules.monitoring.coreos.com; then
    echo "  WARN: PrometheusRule CRD not installed — skipping alert rules (re-run after installing prometheus-operator)" >&2
    return 0
  fi

  _apply_stdin <<MANIFEST
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: fleet-daemon-alerts
  namespace: ${FLEET_DAEMON_NAMESPACE}
  labels:
    app.kubernetes.io/part-of: libre-forge-fleet
    prometheus: ${PROMETHEUS_RULE_SELECTOR}
    role: alert-rules
spec:
  groups:
    - name: fleet-daemon-health
      interval: 30s
      rules:
        - alert: FleetDaemonDown
          expr: |
            absent(fleet_daemon_up{namespace="${FLEET_DAEMON_NAMESPACE}"})
              or fleet_daemon_up{namespace="${FLEET_DAEMON_NAMESPACE}"} == 0
          for: 2m
          labels:
            severity: critical
            workspace: libre-forge
          annotations:
            summary: "fleet-daemon down or unreachable for 2m"
    - name: fleet-daemon-safety
      interval: 30s
      rules:
        - alert: FleetInvariantViolationsRising
          expr: |
            sum by (invariant) (
              rate(fleet_invariant_violations_total{namespace="${FLEET_DAEMON_NAMESPACE}"}[5m])
            ) > 0
          for: 10m
          labels:
            severity: warning
            workspace: libre-forge
          annotations:
            summary: "Invariant {{ \$labels.invariant }} blocked decisions for 10m"
        - alert: FleetProviderRelayFailing
          expr: |
            sum by (op) (
              rate(fleet_provider_api_calls_total{namespace="${FLEET_DAEMON_NAMESPACE}",result!~"ok|dry_run"}[5m])
            ) > 0.1
          for: 10m
          labels:
            severity: critical
            workspace: libre-forge
          annotations:
            summary: "Provider relay verb {{ \$labels.op }} failing at > 0.1/s"
        - alert: FleetApplyFailuresRising
          expr: |
            sum by (kind) (
              rate(fleet_apply_failures_total{namespace="${FLEET_DAEMON_NAMESPACE}"}[5m])
            ) > 0.05
          for: 15m
          labels:
            severity: warning
            workspace: libre-forge
          annotations:
            summary: "Decision apply path {{ \$labels.kind }} failing"
    - name: fleet-daemon-capacity
      interval: 60s
      rules:
        - alert: FleetForecastEmpty
          expr: |
            fleet_forecast_n{namespace="${FLEET_DAEMON_NAMESPACE}"} == 0
          for: 30m
          labels:
            severity: warning
            workspace: libre-forge
          annotations:
            summary: "Forecast N=0 for pool {{ \$labels.pool }}"
        - alert: FleetSnapshotUploaderStalled
          expr: |
            time() - kube_cronjob_status_last_successful_time{
              namespace="${FLEET_DAEMON_NAMESPACE}",
              cronjob="fleet-snapshot-uploader"
            } > 7200
          for: 30m
          labels:
            severity: warning
            workspace: libre-forge
          annotations:
            summary: "fleet-snapshot-uploader hasn't completed successfully in 2h"
MANIFEST
}

# ─── ops ─────────────────────────────────────────────────────────────────

_do_install() {
  _log "namespace + service account"
  _apply_namespace_and_sa
  _log "secrets (nkey + age + registry pull/push)"
  _apply_secrets
  _log "PKI (cert-manager CA + Issuer + daemon TLS)"
  _apply_pki
  _log "config ConfigMap (TOML, rendered from workspace NCL)"
  _apply_config_map
  _log "access policy ConfigMap"
  _apply_access_policy
  _log "snapshot PVC"
  _apply_snapshot_pvc
  _log "LB-IPAM pool"
  _apply_lb_pool
  _log "L2 announcement policy"
  _apply_l2_announcement
  _log "deployment + service"
  _apply_deployment_and_service
  _log "NetworkPolicy"
  _apply_networkpolicy
  _log "snapshot uploader CronJob"
  _apply_snapshot_uploader
  _log "PrometheusRule"
  _apply_prometheus_rule

  _log "waiting for rollout"
  _wait_for_deployment 300

  echo "fleet_daemon installed in namespace ${FLEET_DAEMON_NAMESPACE}"
  echo "  service        : ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc:${FLEET_DAEMON_SERVICE_PORT}"
  if [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ]; then
    echo "  channel (LB)   : ws://${FLEET_DAEMON_CHANNEL_LB_IP}:${FLEET_DAEMON_SERVICE_PORT}/channel  (fleet-agent daemon_url)"
  elif [ "${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" != "0" ]; then
    echo "  channel (NP)   : ws://<node-ip>:${FLEET_DAEMON_CHANNEL_NODEPORT}/channel  (fleet-agent daemon_url, legacy)"
  fi
  echo "  nats           : ${NATS_URL} (prefix=${NATS_SUBJECT_PREFIX})"
  echo "  image          : ${FLEET_DAEMON_IMAGE}"
}

_do_update() {
  _apply_pki
  _apply_config_map
  _apply_access_policy
  _apply_lb_pool
  _apply_l2_announcement
  _apply_networkpolicy
  _apply_deployment_and_service
  _kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE"
  _wait_for_deployment 300
  echo "fleet_daemon updated"
}

_do_delete() {
  _kubectl delete deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete service/"$FLEET_DAEMON_NAME"    --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete cronjob/fleet-snapshot-uploader --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete configmap/fleet-daemon-access-policy --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete configmap/fleet-daemon-config        --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete networkpolicy default-deny-all fleet-daemon-egress fleet-daemon-ingress \
    --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete ciliumloadbalancerippool fleet-daemon-pool --ignore-not-found
  _kubectl delete ciliuml2announcementpolicy "${FLEET_DAEMON_NAME}-l2" --ignore-not-found
  echo "fleet_daemon workload removed (PVC + secrets + namespace preserved)"
}

_do_purge() {
  if [ "${1:-}" != "--confirm" ]; then
    echo "ERROR: purge requires --confirm" >&2
    exit 1
  fi
  _do_delete
  _kubectl delete pvc/fleet-snapshots --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  _kubectl delete secret/forge-control-nkey-creds secret/fleet-daemon-age-key \
                   secret/daoreg-pull-secret secret/daoreg-push-secret \
    --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  if _crd_present certificates.cert-manager.io; then
    _kubectl delete certificate/fleet-mtls-ca certificate/"$FLEET_DAEMON_TLS_SECRET" issuer/fleet-ca-issuer \
      --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
    _kubectl delete secret/fleet-mtls-ca secret/"$FLEET_DAEMON_TLS_SECRET" \
      --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  fi
  if _crd_present prometheusrules.monitoring.coreos.com; then
    _kubectl delete prometheusrule/fleet-daemon-alerts \
      --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  fi
  _kubectl delete namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
  echo "fleet_daemon purged"
}

_do_health() {
  if ! _deployment_exists; then
    echo "UNHEALTHY: deployment ${FLEET_DAEMON_NAME} missing in ${FLEET_DAEMON_NAMESPACE}" >&2
    exit 1
  fi
  local ready_replicas desired_replicas
  ready_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \
    -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo 0)
  desired_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \
    -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0)
  if [ "${ready_replicas:-0}" = "${desired_replicas:-0}" ] && [ "${ready_replicas:-0}" -gt 0 ]; then
    echo "HEALTHY: ${ready_replicas}/${desired_replicas} replicas ready"
  else
    echo "UNHEALTHY: ${ready_replicas:-0}/${desired_replicas:-0} replicas ready" >&2
    _kubectl get pods --namespace "$FLEET_DAEMON_NAMESPACE" -l app.kubernetes.io/name="$FLEET_DAEMON_NAME" >&2
    exit 1
  fi
}

_do_restart() {
  _kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE"
  _wait_for_deployment 300
  echo "fleet_daemon restarted"
}

case "$CMD_TSK" in
  install) _do_install ;;
  update)  _do_update  ;;
  delete)  _do_delete  ;;
  purge)   shift; _do_purge "${1:-}" ;;
  health)  _do_health  ;;
  restart) _do_restart ;;
  *)
    echo "ERROR: unknown CMD_TSK='$CMD_TSK'. Valid: install|update|delete|purge|health|restart" >&2
    exit 1
    ;;
esac