1138 lines
39 KiB
Bash
Executable file
1138 lines
39 KiB
Bash
Executable file
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# shellcheck source=/dev/null
|
|
[ -f "${SCRIPT_DIR}/fleet_daemon-lib.sh" ] && source "${SCRIPT_DIR}/fleet_daemon-lib.sh"
|
|
# shellcheck source=/dev/null
|
|
[ -f "${SCRIPT_DIR}/diagnostics.sh" ] && source "${SCRIPT_DIR}/diagnostics.sh"
|
|
# shellcheck source=/dev/null
|
|
[ -f "${SCRIPT_DIR}/env-fleet_daemon" ] && source "${SCRIPT_DIR}/env-fleet_daemon"
|
|
# shellcheck source=/dev/null
|
|
[ -f "${SCRIPT_DIR}/_credentials.env" ] && source "${SCRIPT_DIR}/_credentials.env"
|
|
|
|
# ─── config (env-overridable, with sane defaults) ────────────────────────
|
|
FLEET_DAEMON_NAME="${FLEET_DAEMON_NAME:-fleet-daemon}"
|
|
FLEET_DAEMON_NAMESPACE="${FLEET_DAEMON_NAMESPACE:-fleet-system}"
|
|
FLEET_DAEMON_IMAGE="${FLEET_DAEMON_IMAGE:-daoreg.librecloud.online/libre-forge/fleet-daemon:0.1.0}"
|
|
FLEET_DAEMON_REPLICAS="${FLEET_DAEMON_REPLICAS:-1}"
|
|
FLEET_DAEMON_SERVICE_PORT="${FLEET_DAEMON_SERVICE_PORT:-19012}"
|
|
FLEET_DAEMON_TLS_SECRET="${FLEET_DAEMON_TLS_SECRET:-fleet-daemon-tls}"
|
|
# NodePort for the agent WebSocket channel (legacy). 0 = disabled.
|
|
# Superseded by FLEET_DAEMON_CHANNEL_LB_IP when set.
|
|
FLEET_DAEMON_CHANNEL_NODEPORT="${FLEET_DAEMON_CHANNEL_NODEPORT:-0}"
|
|
# Cilium LB-IPAM IP for the agent WebSocket channel. When non-empty the Service
|
|
# is typed as LoadBalancer and annotated with io.cilium/lb-ipam-ips. This is the
|
|
# required exposure for off-cluster agents whose packets arrive via WireGuard NAT
|
|
# (externalTrafficPolicy: Local on NodePort would drop them). Takes precedence
|
|
# over FLEET_DAEMON_CHANNEL_NODEPORT.
|
|
FLEET_DAEMON_CHANNEL_LB_IP="${FLEET_DAEMON_CHANNEL_LB_IP:-}"
|
|
|
|
FLEET_DAEMON_CPU_REQ="${FLEET_DAEMON_CPU_REQ:-100m}"
|
|
FLEET_DAEMON_MEM_REQ="${FLEET_DAEMON_MEM_REQ:-128Mi}"
|
|
FLEET_DAEMON_CPU_LIM="${FLEET_DAEMON_CPU_LIM:-500m}"
|
|
FLEET_DAEMON_MEM_LIM="${FLEET_DAEMON_MEM_LIM:-512Mi}"
|
|
|
|
NATS_URL="${NATS_URL:-nats://nats.core-service.svc:4222}"
|
|
NATS_SUBJECT_PREFIX="${NATS_SUBJECT_PREFIX:-fleet.libre-forge}"
|
|
|
|
FLEET_ID="${FLEET_ID:-lian-builders}"
|
|
FLEET_NODE_ID="${FLEET_NODE_ID:-lian-01}"
|
|
FLEET_NODE_IP="${FLEET_NODE_IP:-10.0.10.17}"
|
|
FLEET_POOL_ID="${FLEET_POOL_ID:-arm64-standard}"
|
|
FLEET_NODE_ARCH="${FLEET_NODE_ARCH:-arm64}"
|
|
FLEET_NODE_TIER="${FLEET_NODE_TIER:-standard}"
|
|
FLEET_NODE_CPU="${FLEET_NODE_CPU:-4}"
|
|
FLEET_NODE_RAM_GB="${FLEET_NODE_RAM_GB:-8}"
|
|
FLEET_NODE_DISK_GB="${FLEET_NODE_DISK_GB:-80}"
|
|
FLEET_BUILDKIT_PORT="${FLEET_BUILDKIT_PORT:-1234}"
|
|
# baseline == peak disables physical resize (AlwaysOnResize fixed-capacity mode).
|
|
FLEET_BASELINE_SERVER_TYPE="${FLEET_BASELINE_SERVER_TYPE:-cax21}"
|
|
FLEET_PEAK_SERVER_TYPE="${FLEET_PEAK_SERVER_TYPE:-cax21}"
|
|
# Comma-separated node IDs that should also receive install_tls propagation.
|
|
# Derived from nodes[] excluding the primary node. Example: "lian-02"
|
|
FLEET_EXTRA_NODE_IDS="${FLEET_EXTRA_NODE_IDS:-}"
|
|
# TOML [node_overrides.*] sections for non-primary nodes that differ in arch/pool.
|
|
# Rendered by the workspace recipe from NCL nodes[] (skip 1). May be empty.
|
|
FLEET_NODE_OVERRIDES_TOML="${FLEET_NODE_OVERRIDES_TOML:-}"
|
|
|
|
# NOTE: bash parameter expansion does brace-matching inside ${VAR:-DEFAULT};
|
|
# inlining a JSON object as DEFAULT causes the trailing `}` of the JSON to be
|
|
# appended OUTSIDE the expansion (root cause of the "trailing characters at
|
|
# line 14 column 2" CrashLoop in earlier iterations). Keep the default in its
|
|
# own variable to avoid the mismatch.
|
|
_DEFAULT_ACCESS_POLICY_JSON='{"claims_accepted_from":[],"max_concurrent_claims_per_workspace":[],"priority_order":[]}'
|
|
ACCESS_POLICY_JSON="${ACCESS_POLICY_JSON:-$_DEFAULT_ACCESS_POLICY_JSON}"
|
|
|
|
TICK_INTERVAL_SECONDS="${TICK_INTERVAL_SECONDS:-30}"
|
|
SNAPSHOT_INTERVAL_SECONDS="${SNAPSHOT_INTERVAL_SECONDS:-300}"
|
|
SNAPSHOT_RETAIN_SECONDS="${SNAPSHOT_RETAIN_SECONDS:-86400}"
|
|
SNAPSHOT_LOCAL_PATH="${SNAPSHOT_LOCAL_PATH:-/var/lib/fleet-snapshots}"
|
|
SNAPSHOT_PVC_SIZE="${SNAPSHOT_PVC_SIZE:-5Gi}"
|
|
SNAPSHOT_PVC_STORAGE_CLASS="${SNAPSHOT_PVC_STORAGE_CLASS:-local-path}"
|
|
SNAPSHOT_UPLOAD_SCHEDULE="${SNAPSHOT_UPLOAD_SCHEDULE:-*/30 * * * *}"
|
|
SNAPSHOT_UPLOAD_IMAGE="${SNAPSHOT_UPLOAD_IMAGE:-ghcr.io/oras-project/oras:v1.3.0}"
|
|
SNAPSHOT_UPLOAD_REGISTRY="${SNAPSHOT_UPLOAD_REGISTRY:-daoreg.librecloud.online/libre-forge/fleet-state}"
|
|
SNAPSHOT_UPLOAD_ARTIFACT_TYPE="${SNAPSHOT_UPLOAD_ARTIFACT_TYPE:-application/vnd.libre-forge.fleet-state.v1+json}"
|
|
|
|
PKI_ENABLED="${PKI_ENABLED:-true}"
|
|
PKI_CA_DURATION="${PKI_CA_DURATION:-43800h}"
|
|
PKI_CA_RENEW_BEFORE="${PKI_CA_RENEW_BEFORE:-720h}"
|
|
PKI_CA_PARENT_ISSUER="${PKI_CA_PARENT_ISSUER:-buildkit-selfsigned-issuer}"
|
|
PKI_CA_COMMON_NAME="${PKI_CA_COMMON_NAME:-Libre Daoshi Fleet CA}"
|
|
PKI_CERT_DURATION="${PKI_CERT_DURATION:-2160h}"
|
|
PKI_CERT_RENEW_BEFORE="${PKI_CERT_RENEW_BEFORE:-360h}"
|
|
|
|
L2_ANNOUNCEMENT_ENABLED="${L2_ANNOUNCEMENT_ENABLED:-false}"
|
|
L2_ANNOUNCEMENT_INTERFACES="${L2_ANNOUNCEMENT_INTERFACES:-enp7s0}"
|
|
|
|
NETPOL_ENABLED="${NETPOL_ENABLED:-true}"
|
|
NETPOL_OPERATOR_INGRESS_CIDRS="${NETPOL_OPERATOR_INGRESS_CIDRS:-10.200.0.0/24}"
|
|
NETPOL_CLUSTER_INTERNAL_CIDRS="${NETPOL_CLUSTER_INTERNAL_CIDRS:-10.0.0.0/16}"
|
|
NETPOL_EXTERNAL_EGRESS_HTTPS="${NETPOL_EXTERNAL_EGRESS_HTTPS:-true}"
|
|
NETPOL_NATS_NAMESPACE="${NETPOL_NATS_NAMESPACE:-core-service}"
|
|
|
|
PROMETHEUS_RULE_ENABLED="${PROMETHEUS_RULE_ENABLED:-true}"
|
|
PROMETHEUS_RULE_SELECTOR="${PROMETHEUS_RULE_SELECTOR:-kube-prometheus}"
|
|
|
|
CMD_TSK="${CMD_TSK:-${1:-install}}"
|
|
|
|
# Resolve PRVNG_/PROVISIONING_ aliases (when diagnostics.sh isn't present,
|
|
# fall back to a local truthy check).
|
|
if command -v _diag_is_force >/dev/null 2>&1; then
|
|
_diag_is_force && FORCE=true || FORCE=false
|
|
_diag_is_debug && DEBUG=true || DEBUG=false
|
|
_diag_is_verbose && VERBOSE=true || VERBOSE=false
|
|
else
|
|
case "$(printf '%s' "${PROVISIONING_FORCE:-${PRVNG_FORCE:-}}" | tr '[:upper:]' '[:lower:]')" in
|
|
1|true|yes) FORCE=true ;; *) FORCE=false ;;
|
|
esac
|
|
case "$(printf '%s' "${PROVISIONING_DEBUG:-${PRVNG_DEBUG:-}}" | tr '[:upper:]' '[:lower:]')" in
|
|
1|true|yes) DEBUG=true ;; *) DEBUG=false ;;
|
|
esac
|
|
case "$(printf '%s' "${PROVISIONING_VERBOSE:-${PRVNG_VERBOSE:-}}" | tr '[:upper:]' '[:lower:]')" in
|
|
1|true|yes) VERBOSE=true ;; *) VERBOSE=false ;;
|
|
esac
|
|
fi
|
|
[ "$FORCE" = "true" ] && echo " [force] enabled — Deployment will be deleted before apply"
|
|
[ "$DEBUG" = "true" ] && echo " [debug] enabled — extra logging, /tmp bundle preserved on failure"
|
|
[ "$VERBOSE" = "true" ] && echo " [verbose] enabled — full kubectl apply output"
|
|
|
|
# _log: progress markers, ALWAYS shown (one concise line per phase). They give
|
|
# essential "where am I" feedback even in quiet mode and are cheap (10 lines
|
|
# for a full install vs ~30 lines of kubectl noise that VERBOSE adds).
|
|
_log() {
|
|
printf ' → %s\n' "$*"
|
|
}
|
|
# Convert a comma-separated list of strings to a TOML inline array of quoted strings.
|
|
# "" → [] "lian-02" → ["lian-02"] "lian-02,lian-03" → ["lian-02","lian-03"]
|
|
_toml_string_array_from_csv() {
|
|
local csv="$1"
|
|
if [ -z "$csv" ]; then
|
|
echo "[]"
|
|
return
|
|
fi
|
|
local out='['
|
|
local first=1
|
|
local IFS=','
|
|
for item in $csv; do
|
|
[ $first -eq 0 ] && out+=','
|
|
out+='"'"$item"'"'
|
|
first=0
|
|
done
|
|
echo "${out}]"
|
|
}
|
|
# _apply_stdin: read a manifest from stdin and apply via kubectl, suppressing
|
|
# routine output ("unchanged"/"configured"/"created") when not VERBOSE. Errors
|
|
# are never hidden — non-zero exit always prints stderr.
|
|
_apply_stdin() {
|
|
if [ "$VERBOSE" = "true" ]; then
|
|
_kubectl apply -f -
|
|
return $?
|
|
fi
|
|
local out rc
|
|
out=$(_kubectl apply -f - 2>&1)
|
|
rc=$?
|
|
if [ $rc -ne 0 ]; then
|
|
printf '%s\n' "$out" >&2
|
|
fi
|
|
return $rc
|
|
}
|
|
|
|
_resolve_kubeconfig
|
|
|
|
# Pre-register the workload so the ERR trap can dump diagnostics for ANY
|
|
# failing kubectl apply (not only the final rollout wait).
|
|
if command -v _diag_register_cluster_workload >/dev/null 2>&1; then
|
|
_diag_register_cluster_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" \
|
|
"app.kubernetes.io/name=${FLEET_DAEMON_NAME}"
|
|
_diag_install_err_trap
|
|
fi
|
|
|
|
# ─── secret materialisation (stdin-only, never to disk) ──────────────────
|
|
|
|
_apply_namespace_and_sa() {
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/managed-by: provisioning
|
|
pod-security.kubernetes.io/enforce: restricted
|
|
pod-security.kubernetes.io/audit: restricted
|
|
pod-security.kubernetes.io/warn: restricted
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: ${FLEET_DAEMON_NAME}
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
automountServiceAccountToken: false
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: fleet-snapshot-uploader
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
automountServiceAccountToken: false
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_secrets() {
|
|
_require_env FORGE_CONTROL_SEED
|
|
_require_env FLEET_AGE_KEY
|
|
|
|
_kubectl create secret generic forge-control-nkey-creds \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
--from-literal=seed="${FORGE_CONTROL_SEED}" \
|
|
--dry-run=client -o yaml | _apply_stdin
|
|
|
|
_kubectl create secret generic fleet-daemon-age-key \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
--from-literal=age.key="${FLEET_AGE_KEY}" \
|
|
--dry-run=client -o yaml | _apply_stdin
|
|
|
|
if [ -n "${DAOREG_PULL_USERNAME:-}" ] && [ -n "${DAOREG_PULL_PASSWORD:-}" ]; then
|
|
_kubectl create secret docker-registry daoreg-pull-secret \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
--docker-server=daoreg.librecloud.online \
|
|
--docker-username="${DAOREG_PULL_USERNAME}" \
|
|
--docker-password="${DAOREG_PULL_PASSWORD}" \
|
|
--dry-run=client -o yaml | _apply_stdin
|
|
else
|
|
echo " WARN: DAOREG_PULL_USERNAME/PASSWORD not set — daoreg-pull-secret not materialised (image pulls may fail)" >&2
|
|
fi
|
|
|
|
if [ -n "${DAOREG_PUSH_USERNAME:-}" ] && [ -n "${DAOREG_PUSH_PASSWORD:-}" ]; then
|
|
_kubectl create secret docker-registry daoreg-push-secret \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
--docker-server=daoreg.librecloud.online \
|
|
--docker-username="${DAOREG_PUSH_USERNAME}" \
|
|
--docker-password="${DAOREG_PUSH_PASSWORD}" \
|
|
--dry-run=client -o yaml | _apply_stdin
|
|
else
|
|
echo " WARN: DAOREG_PUSH_USERNAME/PASSWORD not set — daoreg-push-secret not materialised (snapshot uploader will fail)" >&2
|
|
fi
|
|
}
|
|
|
|
_apply_pki() {
|
|
[ "$PKI_ENABLED" = "true" ] || { echo " [skip] PKI disabled"; return 0; }
|
|
|
|
if ! _crd_present certificates.cert-manager.io; then
|
|
echo " WARN: cert-manager CRDs not installed — skipping PKI block" >&2
|
|
return 0
|
|
fi
|
|
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: fleet-mtls-ca
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
isCA: true
|
|
commonName: ${PKI_CA_COMMON_NAME}
|
|
duration: ${PKI_CA_DURATION}
|
|
renewBefore: ${PKI_CA_RENEW_BEFORE}
|
|
secretName: fleet-mtls-ca
|
|
privateKey:
|
|
algorithm: ECDSA
|
|
size: 256
|
|
issuerRef:
|
|
name: ${PKI_CA_PARENT_ISSUER}
|
|
kind: ClusterIssuer
|
|
group: cert-manager.io
|
|
usages:
|
|
- cert sign
|
|
- crl sign
|
|
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Issuer
|
|
metadata:
|
|
name: fleet-ca-issuer
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
ca:
|
|
secretName: fleet-mtls-ca
|
|
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: ${FLEET_DAEMON_TLS_SECRET}
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
secretName: ${FLEET_DAEMON_TLS_SECRET}
|
|
duration: ${PKI_CERT_DURATION}
|
|
renewBefore: ${PKI_CERT_RENEW_BEFORE}
|
|
privateKey:
|
|
algorithm: ECDSA
|
|
size: 256
|
|
issuerRef:
|
|
name: fleet-ca-issuer
|
|
kind: Issuer
|
|
group: cert-manager.io
|
|
commonName: ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc
|
|
dnsNames:
|
|
- ${FLEET_DAEMON_NAME}
|
|
- ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}
|
|
- ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc
|
|
- ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc.cluster.local
|
|
usages:
|
|
- server auth
|
|
- digital signature
|
|
- key encipherment
|
|
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: fleet-buildkit-tls
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
secretName: fleet-buildkit-tls
|
|
duration: ${PKI_CERT_DURATION}
|
|
renewBefore: ${PKI_CERT_RENEW_BEFORE}
|
|
privateKey:
|
|
algorithm: ECDSA
|
|
size: 256
|
|
issuerRef:
|
|
name: fleet-ca-issuer
|
|
kind: Issuer
|
|
group: cert-manager.io
|
|
commonName: fleet-buildkit
|
|
usages:
|
|
- server auth
|
|
- client auth
|
|
- digital signature
|
|
- key encipherment
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_config_map() {
|
|
# Render config.toml from env vars (which themselves come from NCL via the
|
|
# workspace recipe) + apply as ConfigMap `fleet-daemon-config`. The pod
|
|
# mounts it at /etc/fleet-daemon/config.toml; the binary reads at startup.
|
|
local toml
|
|
toml=$(cat <<TOML
|
|
# Auto-generated by install-fleet_daemon.sh from workspace NCL.
|
|
fleet_id = "${FLEET_ID}"
|
|
claim_ttl_seconds = ${FLEET_CLAIM_TTL_SECONDS:-3600}
|
|
http_bind_addr = "0.0.0.0:${FLEET_DAEMON_SERVICE_PORT}"
|
|
tick_interval_seconds = ${TICK_INTERVAL_SECONDS}
|
|
dry_run = ${FLEET_DRY_RUN:-false}
|
|
access_policy_path = "/etc/fleet-daemon/access-policy.json"
|
|
nkey_seed_file = "/var/run/secrets/fleet-daemon-nkey/seed"
|
|
channel_token_path = "/var/run/secrets/channel-token/token"
|
|
extra_node_ids = $(_toml_string_array_from_csv "${FLEET_EXTRA_NODE_IDS}")
|
|
|
|
[nats]
|
|
url = "${NATS_URL}"
|
|
|
|
[node]
|
|
node_id = "${FLEET_NODE_ID}"
|
|
private_ip = "${FLEET_NODE_IP}"
|
|
buildkit_port = ${FLEET_BUILDKIT_PORT}
|
|
buildkit_probe_enabled = ${FLEET_BUILDKIT_PROBE_ENABLED}
|
|
pool_id = "${FLEET_POOL_ID}"
|
|
arch = "${FLEET_NODE_ARCH}"
|
|
tier = "${FLEET_NODE_TIER}"
|
|
declared_cpu = ${FLEET_NODE_CPU}
|
|
declared_ram_gb = ${FLEET_NODE_RAM_GB}
|
|
declared_disk_gb = ${FLEET_NODE_DISK_GB}
|
|
mtls_server_subject = "CN=fleet-buildkit"
|
|
baseline_server_type = "${FLEET_BASELINE_SERVER_TYPE}"
|
|
peak_server_type = "${FLEET_PEAK_SERVER_TYPE}"
|
|
|
|
[snapshot]
|
|
interval_seconds = ${SNAPSHOT_INTERVAL_SECONDS}
|
|
local_path = "${SNAPSHOT_LOCAL_PATH}"
|
|
retain_seconds = ${SNAPSHOT_RETAIN_SECONDS}
|
|
TOML
|
|
)
|
|
if [ -n "${FLEET_NODE_OVERRIDES_TOML}" ]; then
|
|
toml="${toml}
|
|
${FLEET_NODE_OVERRIDES_TOML}"
|
|
fi
|
|
_kubectl create configmap fleet-daemon-config \
|
|
--namespace "${FLEET_DAEMON_NAMESPACE}" \
|
|
--from-literal=config.toml="${toml}" \
|
|
--dry-run=client -o yaml | _apply_stdin
|
|
}
|
|
|
|
_apply_access_policy() {
|
|
# Validate JSON shape locally before sending to the cluster. Catches malformed
|
|
# ACCESS_POLICY_JSON at the bash boundary instead of inside the daemon at
|
|
# startup (which manifests as CrashLoopBackOff with serde_json parse error).
|
|
if command -v jq >/dev/null 2>&1; then
|
|
local jq_err
|
|
if ! jq_err=$(printf '%s' "$ACCESS_POLICY_JSON" | jq -e . 2>&1 >/dev/null); then
|
|
echo "ERROR: access-policy JSON failed local validation — refusing to apply ConfigMap." >&2
|
|
echo " jq: ${jq_err}" >&2
|
|
if [ "$DEBUG" = "true" ]; then
|
|
echo " raw value (DEBUG):" >&2
|
|
printf ' %s\n' "$ACCESS_POLICY_JSON" >&2
|
|
else
|
|
echo " (re-run with PRVNG_DEBUG=true to see the raw value)" >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
# Serialise as a YAML double-quoted scalar via kubectl create --dry-run, which
|
|
# handles all escaping correctly regardless of newlines/quotes in the JSON.
|
|
_kubectl create configmap fleet-daemon-access-policy \
|
|
--namespace "${FLEET_DAEMON_NAMESPACE}" \
|
|
--from-literal=access-policy.json="${ACCESS_POLICY_JSON}" \
|
|
--dry-run=client -o yaml \
|
|
| _kubectl label --local --dry-run=client -o yaml -f - \
|
|
app.kubernetes.io/name="${FLEET_DAEMON_NAME}" \
|
|
app.kubernetes.io/part-of=libre-forge-fleet \
|
|
app.kubernetes.io/component=access-policy \
|
|
--overwrite \
|
|
| _apply_stdin
|
|
}
|
|
|
|
_apply_snapshot_pvc() {
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: fleet-snapshots
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/component: snapshots
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
storageClassName: ${SNAPSHOT_PVC_STORAGE_CLASS}
|
|
resources:
|
|
requests:
|
|
storage: ${SNAPSHOT_PVC_SIZE}
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_deployment_and_service() {
|
|
# Pre-compute Service type for the channel endpoint:
|
|
# FLEET_DAEMON_CHANNEL_LB_IP set → LoadBalancer (Cilium LB-IPAM, preferred)
|
|
# FLEET_DAEMON_CHANNEL_NODEPORT != 0 → NodePort (legacy; fails for WireGuard-NATted agents)
|
|
# Neither → ClusterIP (cluster-internal only)
|
|
local svc_type="ClusterIP"
|
|
local svc_nodeport_line=""
|
|
local svc_etp_line=""
|
|
local svc_annotations_block=""
|
|
local svc_lb_ip_spec=""
|
|
if [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ]; then
|
|
svc_type="LoadBalancer"
|
|
svc_annotations_block=" annotations:
|
|
io.cilium/lb-ipam-ips: \"${FLEET_DAEMON_CHANNEL_LB_IP}\""
|
|
svc_lb_ip_spec=" loadBalancerIP: ${FLEET_DAEMON_CHANNEL_LB_IP}"
|
|
elif [ "${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" != "0" ]; then
|
|
svc_type="NodePort"
|
|
svc_nodeport_line=" nodePort: ${FLEET_DAEMON_CHANNEL_NODEPORT}"
|
|
svc_etp_line=" externalTrafficPolicy: Local"
|
|
fi
|
|
if [ "$FORCE" = "true" ]; then
|
|
if command -v _diag_force_delete_workload >/dev/null 2>&1; then
|
|
_diag_force_delete_workload deployment "$FLEET_DAEMON_NAME" "$FLEET_DAEMON_NAMESPACE" 30
|
|
else
|
|
_kubectl delete deployment "$FLEET_DAEMON_NAME" \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
--wait=true --timeout=30s --ignore-not-found 2>&1 || true
|
|
fi
|
|
fi
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: ${FLEET_DAEMON_NAME}
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/version: "${FLEET_DAEMON_IMAGE##*:}"
|
|
spec:
|
|
replicas: ${FLEET_DAEMON_REPLICAS}
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/version: "${FLEET_DAEMON_IMAGE##*:}"
|
|
spec:
|
|
serviceAccountName: ${FLEET_DAEMON_NAME}
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
runAsGroup: 65532
|
|
fsGroup: 65532
|
|
# OnRootMismatch (the default) only chgrp's when the volume's root dir
|
|
# already has the right group. local-path-provisioner creates dirs as
|
|
# uid:gid 1000:1000, so OnRootMismatch correctly detects the mismatch
|
|
# but in practice some kubelet versions still skip the recursive walk
|
|
# (observed: snapshot writes fail with "permission denied" on a fresh
|
|
# PVC). Always = unconditional recursive chgrp/chmod on every mount —
|
|
# slower for huge volumes, correct for small ones like our snapshots PVC.
|
|
fsGroupChangePolicy: Always
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
imagePullSecrets:
|
|
- name: daoreg-pull-secret
|
|
containers:
|
|
- name: ${FLEET_DAEMON_NAME}
|
|
image: ${FLEET_DAEMON_IMAGE}
|
|
# Always re-pull so a re-pushed tag (same :0.1.0) takes effect on
|
|
# rolling restart. Cost = one image fetch per restart; benefit =
|
|
# zero risk of running cached older binaries from local containerd.
|
|
imagePullPolicy: Always
|
|
ports:
|
|
- name: http
|
|
containerPort: ${FLEET_DAEMON_SERVICE_PORT}
|
|
protocol: TCP
|
|
env:
|
|
- name: RUST_LOG
|
|
value: "fleet_daemon=info,async_nats=warn"
|
|
- name: FLEET_DAEMON_CONFIG
|
|
value: "/etc/fleet-daemon/config.toml"
|
|
- name: SOPS_AGE_KEY_FILE
|
|
value: "/var/run/secrets/fleet-daemon-age-key/age.key"
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/fleet-daemon/config.toml
|
|
subPath: config.toml
|
|
readOnly: true
|
|
- name: access-policy
|
|
mountPath: /etc/fleet-daemon/access-policy.json
|
|
subPath: access-policy.json
|
|
readOnly: true
|
|
- name: nkey-creds
|
|
mountPath: /var/run/secrets/fleet-daemon-nkey
|
|
readOnly: true
|
|
- name: age-key
|
|
mountPath: /var/run/secrets/fleet-daemon-age-key
|
|
readOnly: true
|
|
- name: tls
|
|
mountPath: /etc/fleet-daemon/buildkit-tls
|
|
readOnly: true
|
|
- name: channel-token
|
|
mountPath: /var/run/secrets/channel-token
|
|
readOnly: true
|
|
- name: snapshots
|
|
mountPath: ${SNAPSHOT_LOCAL_PATH}
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /readyz
|
|
port: http
|
|
scheme: HTTP
|
|
initialDelaySeconds: 3
|
|
periodSeconds: 5
|
|
timeoutSeconds: 1
|
|
failureThreshold: 3
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: http
|
|
scheme: HTTP
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 15
|
|
timeoutSeconds: 1
|
|
failureThreshold: 5
|
|
resources:
|
|
requests:
|
|
cpu: ${FLEET_DAEMON_CPU_REQ}
|
|
memory: ${FLEET_DAEMON_MEM_REQ}
|
|
limits:
|
|
cpu: ${FLEET_DAEMON_CPU_LIM}
|
|
memory: ${FLEET_DAEMON_MEM_LIM}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: fleet-daemon-config
|
|
items:
|
|
- key: config.toml
|
|
path: config.toml
|
|
- name: access-policy
|
|
configMap:
|
|
name: fleet-daemon-access-policy
|
|
items:
|
|
- key: access-policy.json
|
|
path: access-policy.json
|
|
- name: nkey-creds
|
|
secret:
|
|
secretName: forge-control-nkey-creds
|
|
defaultMode: 0400
|
|
items:
|
|
- key: seed
|
|
path: seed
|
|
- name: age-key
|
|
secret:
|
|
secretName: fleet-daemon-age-key
|
|
defaultMode: 0400
|
|
items:
|
|
- key: age.key
|
|
path: age.key
|
|
- name: tls
|
|
secret:
|
|
secretName: fleet-buildkit-tls
|
|
defaultMode: 0400
|
|
optional: true
|
|
- name: channel-token
|
|
secret:
|
|
secretName: fleet-channel-token
|
|
defaultMode: 0400
|
|
items:
|
|
- key: token
|
|
path: token
|
|
- name: snapshots
|
|
persistentVolumeClaim:
|
|
claimName: fleet-snapshots
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: ${FLEET_DAEMON_NAME}
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
${svc_annotations_block}
|
|
spec:
|
|
type: ${svc_type}
|
|
${svc_lb_ip_spec}
|
|
${svc_etp_line}
|
|
selector:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
ports:
|
|
- name: http
|
|
port: ${FLEET_DAEMON_SERVICE_PORT}
|
|
targetPort: http
|
|
protocol: TCP
|
|
${svc_nodeport_line}
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_lb_pool() {
|
|
[ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ] || { echo " [skip] LB pool: no channel_lb_ip set"; return 0; }
|
|
_apply_stdin <<POOL
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumLoadBalancerIPPool
|
|
metadata:
|
|
name: fleet-daemon-pool
|
|
labels:
|
|
app.kubernetes.io/name: fleet-daemon
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/managed-by: provisioning
|
|
spec:
|
|
blocks:
|
|
- cidr: "${FLEET_DAEMON_CHANNEL_LB_IP}/32"
|
|
serviceSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: fleet-daemon
|
|
POOL
|
|
}
|
|
|
|
_apply_l2_announcement() {
|
|
[ "$L2_ANNOUNCEMENT_ENABLED" = "true" ] || { echo " [skip] L2 announcement: disabled (set L2_ANNOUNCEMENT_ENABLED=true when Cilium l2announcements.enabled=true)"; return 0; }
|
|
local iface_entries=""
|
|
for iface in ${L2_ANNOUNCEMENT_INTERFACES}; do
|
|
iface_entries="${iface_entries} - \"${iface}\"\n"
|
|
done
|
|
_apply_stdin <<L2ANN
|
|
apiVersion: cilium.io/v2alpha1
|
|
kind: CiliumL2AnnouncementPolicy
|
|
metadata:
|
|
name: ${FLEET_DAEMON_NAME}-l2
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/managed-by: provisioning
|
|
spec:
|
|
serviceSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
nodeSelector:
|
|
matchExpressions:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
interfaces:
|
|
$(printf "%s" "${iface_entries}" | sed 's/\\n/\n/g')
|
|
externalIPs: false
|
|
loadBalancerIPs: true
|
|
L2ANN
|
|
}
|
|
|
|
_apply_networkpolicy() {
|
|
[ "$NETPOL_ENABLED" = "true" ] || { echo " [skip] NetworkPolicy disabled"; return 0; }
|
|
|
|
local op_ingress cluster_internal external_block
|
|
# shellcheck disable=SC2086
|
|
op_ingress=$(_cidrs_yaml_array " " ${NETPOL_OPERATOR_INGRESS_CIDRS})
|
|
# shellcheck disable=SC2086
|
|
cluster_internal=$(_cidrs_yaml_array " " ${NETPOL_CLUSTER_INTERNAL_CIDRS})
|
|
|
|
if [ "$NETPOL_EXTERNAL_EGRESS_HTTPS" = "true" ]; then
|
|
external_block=$(cat <<'BLK'
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except: ["169.254.0.0/16", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
BLK
|
|
)
|
|
else
|
|
external_block=""
|
|
fi
|
|
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: default-deny-all
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes: [Ingress, Egress]
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: fleet-daemon-egress
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: kube-system
|
|
podSelector:
|
|
matchLabels:
|
|
k8s-app: kube-dns
|
|
ports:
|
|
- port: 53
|
|
protocol: UDP
|
|
- port: 53
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: ${NETPOL_NATS_NAMESPACE}
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: nats
|
|
ports:
|
|
- port: 4222
|
|
protocol: TCP
|
|
- to:
|
|
${cluster_internal}
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
- to:
|
|
${cluster_internal}
|
|
ports:
|
|
- port: ${FLEET_BUILDKIT_PORT}
|
|
protocol: TCP
|
|
${external_block}
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: fleet-daemon-ingress
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: ${FLEET_DAEMON_NAME}
|
|
policyTypes: [Ingress]
|
|
ingress:
|
|
- from:
|
|
${op_ingress}
|
|
ports:
|
|
- port: ${FLEET_DAEMON_SERVICE_PORT}
|
|
protocol: TCP
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_snapshot_uploader() {
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: fleet-snapshot-uploader
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
app.kubernetes.io/component: snapshots
|
|
spec:
|
|
schedule: "${SNAPSHOT_UPLOAD_SCHEDULE}"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 1
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
ttlSecondsAfterFinished: 1800
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: fleet-snapshot-uploader
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
spec:
|
|
serviceAccountName: fleet-snapshot-uploader
|
|
automountServiceAccountToken: false
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
fsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
imagePullSecrets:
|
|
- name: daoreg-pull-secret
|
|
containers:
|
|
- name: uploader
|
|
image: ${SNAPSHOT_UPLOAD_IMAGE}
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
cd ${SNAPSHOT_LOCAL_PATH}
|
|
shopt -s nullglob 2>/dev/null || true
|
|
uploaded=0
|
|
skipped=0
|
|
for f in snap-*.json; do
|
|
[ -f "\$f" ] || continue
|
|
tag="\${f#snap-}"
|
|
tag="\${tag%.json}"
|
|
if oras push \\
|
|
"${SNAPSHOT_UPLOAD_REGISTRY}:\${tag}" \\
|
|
--artifact-type ${SNAPSHOT_UPLOAD_ARTIFACT_TYPE} \\
|
|
"\${f}:application/json" 2>/dev/null; then
|
|
rm -f "\${f}"
|
|
uploaded=\$((uploaded + 1))
|
|
else
|
|
echo "WARN: failed to push \${f}; leaving on disk for next cycle" >&2
|
|
skipped=\$((skipped + 1))
|
|
fi
|
|
done
|
|
echo "uploaded=\${uploaded} skipped=\${skipped}"
|
|
volumeMounts:
|
|
- name: snapshots
|
|
mountPath: ${SNAPSHOT_LOCAL_PATH}
|
|
- name: docker-config
|
|
mountPath: /root/.docker
|
|
readOnly: true
|
|
env:
|
|
- name: DOCKER_CONFIG
|
|
value: /root/.docker
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumes:
|
|
- name: snapshots
|
|
persistentVolumeClaim:
|
|
claimName: fleet-snapshots
|
|
- name: docker-config
|
|
secret:
|
|
secretName: daoreg-push-secret
|
|
items:
|
|
- key: .dockerconfigjson
|
|
path: config.json
|
|
MANIFEST
|
|
}
|
|
|
|
_apply_prometheus_rule() {
|
|
[ "$PROMETHEUS_RULE_ENABLED" = "true" ] || { echo " [skip] PrometheusRule disabled"; return 0; }
|
|
|
|
if ! _crd_present prometheusrules.monitoring.coreos.com; then
|
|
echo " WARN: PrometheusRule CRD not installed — skipping alert rules (re-run after installing prometheus-operator)" >&2
|
|
return 0
|
|
fi
|
|
|
|
_apply_stdin <<MANIFEST
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: fleet-daemon-alerts
|
|
namespace: ${FLEET_DAEMON_NAMESPACE}
|
|
labels:
|
|
app.kubernetes.io/part-of: libre-forge-fleet
|
|
prometheus: ${PROMETHEUS_RULE_SELECTOR}
|
|
role: alert-rules
|
|
spec:
|
|
groups:
|
|
- name: fleet-daemon-health
|
|
interval: 30s
|
|
rules:
|
|
- alert: FleetDaemonDown
|
|
expr: |
|
|
absent(fleet_daemon_up{namespace="${FLEET_DAEMON_NAMESPACE}"})
|
|
or fleet_daemon_up{namespace="${FLEET_DAEMON_NAMESPACE}"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "fleet-daemon down or unreachable for 2m"
|
|
- name: fleet-daemon-safety
|
|
interval: 30s
|
|
rules:
|
|
- alert: FleetInvariantViolationsRising
|
|
expr: |
|
|
sum by (invariant) (
|
|
rate(fleet_invariant_violations_total{namespace="${FLEET_DAEMON_NAMESPACE}"}[5m])
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "Invariant {{ \$labels.invariant }} blocked decisions for 10m"
|
|
- alert: FleetProviderRelayFailing
|
|
expr: |
|
|
sum by (op) (
|
|
rate(fleet_provider_api_calls_total{namespace="${FLEET_DAEMON_NAMESPACE}",result!~"ok|dry_run"}[5m])
|
|
) > 0.1
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "Provider relay verb {{ \$labels.op }} failing at > 0.1/s"
|
|
- alert: FleetApplyFailuresRising
|
|
expr: |
|
|
sum by (kind) (
|
|
rate(fleet_apply_failures_total{namespace="${FLEET_DAEMON_NAMESPACE}"}[5m])
|
|
) > 0.05
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "Decision apply path {{ \$labels.kind }} failing"
|
|
- name: fleet-daemon-capacity
|
|
interval: 60s
|
|
rules:
|
|
- alert: FleetForecastEmpty
|
|
expr: |
|
|
fleet_forecast_n{namespace="${FLEET_DAEMON_NAMESPACE}"} == 0
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "Forecast N=0 for pool {{ \$labels.pool }}"
|
|
- alert: FleetSnapshotUploaderStalled
|
|
expr: |
|
|
time() - kube_cronjob_status_last_successful_time{
|
|
namespace="${FLEET_DAEMON_NAMESPACE}",
|
|
cronjob="fleet-snapshot-uploader"
|
|
} > 7200
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
workspace: libre-forge
|
|
annotations:
|
|
summary: "fleet-snapshot-uploader hasn't completed successfully in 2h"
|
|
MANIFEST
|
|
}
|
|
|
|
# ─── ops ─────────────────────────────────────────────────────────────────
|
|
|
|
_do_install() {
|
|
_log "namespace + service account"
|
|
_apply_namespace_and_sa
|
|
_log "secrets (nkey + age + registry pull/push)"
|
|
_apply_secrets
|
|
_log "PKI (cert-manager CA + Issuer + daemon TLS)"
|
|
_apply_pki
|
|
_log "config ConfigMap (TOML, rendered from workspace NCL)"
|
|
_apply_config_map
|
|
_log "access policy ConfigMap"
|
|
_apply_access_policy
|
|
_log "snapshot PVC"
|
|
_apply_snapshot_pvc
|
|
_log "LB-IPAM pool"
|
|
_apply_lb_pool
|
|
_log "L2 announcement policy"
|
|
_apply_l2_announcement
|
|
_log "deployment + service"
|
|
_apply_deployment_and_service
|
|
_log "NetworkPolicy"
|
|
_apply_networkpolicy
|
|
_log "snapshot uploader CronJob"
|
|
_apply_snapshot_uploader
|
|
_log "PrometheusRule"
|
|
_apply_prometheus_rule
|
|
|
|
_log "waiting for rollout"
|
|
_wait_for_deployment 300
|
|
|
|
echo "fleet_daemon installed in namespace ${FLEET_DAEMON_NAMESPACE}"
|
|
echo " service : ${FLEET_DAEMON_NAME}.${FLEET_DAEMON_NAMESPACE}.svc:${FLEET_DAEMON_SERVICE_PORT}"
|
|
if [ -n "${FLEET_DAEMON_CHANNEL_LB_IP:-}" ]; then
|
|
echo " channel (LB) : ws://${FLEET_DAEMON_CHANNEL_LB_IP}:${FLEET_DAEMON_SERVICE_PORT}/channel (fleet-agent daemon_url)"
|
|
elif [ "${FLEET_DAEMON_CHANNEL_NODEPORT:-0}" != "0" ]; then
|
|
echo " channel (NP) : ws://<node-ip>:${FLEET_DAEMON_CHANNEL_NODEPORT}/channel (fleet-agent daemon_url, legacy)"
|
|
fi
|
|
echo " nats : ${NATS_URL} (prefix=${NATS_SUBJECT_PREFIX})"
|
|
echo " image : ${FLEET_DAEMON_IMAGE}"
|
|
}
|
|
|
|
_do_update() {
|
|
_apply_pki
|
|
_apply_config_map
|
|
_apply_access_policy
|
|
_apply_lb_pool
|
|
_apply_l2_announcement
|
|
_apply_networkpolicy
|
|
_apply_deployment_and_service
|
|
_kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE"
|
|
_wait_for_deployment 300
|
|
echo "fleet_daemon updated"
|
|
}
|
|
|
|
_do_delete() {
|
|
_kubectl delete deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete service/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete cronjob/fleet-snapshot-uploader --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete configmap/fleet-daemon-access-policy --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete configmap/fleet-daemon-config --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete networkpolicy default-deny-all fleet-daemon-egress fleet-daemon-ingress \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete ciliumloadbalancerippool fleet-daemon-pool --ignore-not-found
|
|
_kubectl delete ciliuml2announcementpolicy "${FLEET_DAEMON_NAME}-l2" --ignore-not-found
|
|
echo "fleet_daemon workload removed (PVC + secrets + namespace preserved)"
|
|
}
|
|
|
|
_do_purge() {
|
|
if [ "${1:-}" != "--confirm" ]; then
|
|
echo "ERROR: purge requires --confirm" >&2
|
|
exit 1
|
|
fi
|
|
_do_delete
|
|
_kubectl delete pvc/fleet-snapshots --namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete secret/forge-control-nkey-creds secret/fleet-daemon-age-key \
|
|
secret/daoreg-pull-secret secret/daoreg-push-secret \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
if _crd_present certificates.cert-manager.io; then
|
|
_kubectl delete certificate/fleet-mtls-ca certificate/"$FLEET_DAEMON_TLS_SECRET" issuer/fleet-ca-issuer \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
_kubectl delete secret/fleet-mtls-ca secret/"$FLEET_DAEMON_TLS_SECRET" \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
fi
|
|
if _crd_present prometheusrules.monitoring.coreos.com; then
|
|
_kubectl delete prometheusrule/fleet-daemon-alerts \
|
|
--namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
fi
|
|
_kubectl delete namespace "$FLEET_DAEMON_NAMESPACE" --ignore-not-found
|
|
echo "fleet_daemon purged"
|
|
}
|
|
|
|
_do_health() {
|
|
if ! _deployment_exists; then
|
|
echo "UNHEALTHY: deployment ${FLEET_DAEMON_NAME} missing in ${FLEET_DAEMON_NAMESPACE}" >&2
|
|
exit 1
|
|
fi
|
|
local ready_replicas desired_replicas
|
|
ready_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
-o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo 0)
|
|
desired_replicas=$(_kubectl get deployment "$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE" \
|
|
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0)
|
|
if [ "${ready_replicas:-0}" = "${desired_replicas:-0}" ] && [ "${ready_replicas:-0}" -gt 0 ]; then
|
|
echo "HEALTHY: ${ready_replicas}/${desired_replicas} replicas ready"
|
|
else
|
|
echo "UNHEALTHY: ${ready_replicas:-0}/${desired_replicas:-0} replicas ready" >&2
|
|
_kubectl get pods --namespace "$FLEET_DAEMON_NAMESPACE" -l app.kubernetes.io/name="$FLEET_DAEMON_NAME" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
_do_restart() {
|
|
_kubectl rollout restart deployment/"$FLEET_DAEMON_NAME" --namespace "$FLEET_DAEMON_NAMESPACE"
|
|
_wait_for_deployment 300
|
|
echo "fleet_daemon restarted"
|
|
}
|
|
|
|
case "$CMD_TSK" in
|
|
install) _do_install ;;
|
|
update) _do_update ;;
|
|
delete) _do_delete ;;
|
|
purge) shift; _do_purge "${1:-}" ;;
|
|
health) _do_health ;;
|
|
restart) _do_restart ;;
|
|
*)
|
|
echo "ERROR: unknown CMD_TSK='$CMD_TSK'. Valid: install|update|delete|purge|health|restart" >&2
|
|
exit 1
|
|
;;
|
|
esac
|