323 lines
11 KiB
Bash
323 lines
11 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
[ -r "./env-cilium" ] && . ./env-cilium
|
||
|
|
|
||
|
|
# kubeconfig — env-cilium override → kubeadm → k0s → generate from k0s
|
||
|
|
K0S_DATA_DIR="${K0S_DATA_DIR:-/var/lib/k0s}"
|
||
|
|
if [ -z "${KUBECONFIG:-}" ] || [ ! -f "${KUBECONFIG}" ]; then
|
||
|
|
if [ -f "/etc/kubernetes/admin.conf" ]; then
|
||
|
|
KUBECONFIG="/etc/kubernetes/admin.conf"
|
||
|
|
elif [ -f "${K0S_DATA_DIR}/pki/admin.conf" ]; then
|
||
|
|
KUBECONFIG="${K0S_DATA_DIR}/pki/admin.conf"
|
||
|
|
else
|
||
|
|
KUBECONFIG="/tmp/k0s-admin.conf"
|
||
|
|
k0s kubeconfig admin > "${KUBECONFIG}"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
export KUBECONFIG
|
||
|
|
|
||
|
|
# Detect kubectl: native binary or k0s wrapper
|
||
|
|
if command -v kubectl &>/dev/null; then
|
||
|
|
KUBECTL="kubectl"
|
||
|
|
elif command -v k0s &>/dev/null; then
|
||
|
|
KUBECTL="k0s kubectl"
|
||
|
|
else
|
||
|
|
echo "ERROR: neither kubectl nor k0s found in PATH" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
OS="$(uname | tr '[:upper:]' '[:lower:]')"
|
||
|
|
ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/' -e 's/armv7l/arm/')"
|
||
|
|
|
||
|
|
CILIUM_VERSION="${CILIUM_VERSION:-1.19.3}"
|
||
|
|
CILIUM_CLUSTER="${CILIUM_CLUSTER_NAME:-}"
|
||
|
|
CILIUM_CLUSTER_ID="${CILIUM_CLUSTER_ID:-0}"
|
||
|
|
CILIUM_INGRESS="${CILIUM_ENABLE_INGRESS_CONTROLLER:-false}"
|
||
|
|
CILIUM_INGRESS_LB="${CILIUM_INGRESS_LB_MODE:-shared}"
|
||
|
|
CILIUM_GATEWAY_API="${CILIUM_ENABLE_GATEWAY_API:-false}"
|
||
|
|
CILIUM_GATEWAY_API_VERSION="${CILIUM_GATEWAY_API_VERSION:-v1.2.1}"
|
||
|
|
CILIUM_KPR="${CILIUM_KUBE_PROXY_REPLACEMENT:-false}"
|
||
|
|
CILIUM_SOCKET_LB="${CILIUM_SOCKET_LB:-true}"
|
||
|
|
CILIUM_ENVOY_ENABLED="${CILIUM_ENVOY_ENABLED:-true}"
|
||
|
|
CILIUM_L2_ANNOUNCEMENT="${CILIUM_ENABLE_L2_ANNOUNCEMENT:-false}"
|
||
|
|
CILIUM_L2_NEIGH="${CILIUM_ENABLE_L2_NEIGH_DISCOVERY:-false}"
|
||
|
|
|
||
|
|
# Install cilium CLI if missing or outdated
|
||
|
|
CILIUM_CLI_VERSION="${CILIUM_CLI_VERSION:-v0.19.0}"
|
||
|
|
if ! cilium version 2>/dev/null | grep -q "${CILIUM_CLI_VERSION}"; then
|
||
|
|
echo "=== cilium-cli: installing ${CILIUM_CLI_VERSION} ==="
|
||
|
|
curl -sL --remote-name-all \
|
||
|
|
"https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-${OS}-${ARCH}.tar.gz"{,.sha256sum}
|
||
|
|
sha256sum --check "cilium-${OS}-${ARCH}.tar.gz.sha256sum"
|
||
|
|
sudo tar xzfC "cilium-${OS}-${ARCH}.tar.gz" /usr/local/bin
|
||
|
|
rm "cilium-${OS}-${ARCH}.tar.gz" "cilium-${OS}-${ARCH}.tar.gz.sha256sum"
|
||
|
|
fi
|
||
|
|
|
||
|
|
CILIUM_ALREADY_INSTALLED=false
|
||
|
|
if cilium status --wait=false 2>/dev/null | grep -q "OK"; then
|
||
|
|
CILIUM_ALREADY_INSTALLED=true
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Gateway API CRDs — must exist before cilium install/upgrade
|
||
|
|
if [ "${CILIUM_GATEWAY_API}" = "true" ]; then
|
||
|
|
echo "=== cilium: installing Gateway API CRDs ${CILIUM_GATEWAY_API_VERSION} ==="
|
||
|
|
GW_STD="https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${CILIUM_GATEWAY_API_VERSION}/config/crd/standard"
|
||
|
|
for crd in \
|
||
|
|
gateway.networking.k8s.io_gatewayclasses.yaml \
|
||
|
|
gateway.networking.k8s.io_gateways.yaml \
|
||
|
|
gateway.networking.k8s.io_httproutes.yaml \
|
||
|
|
gateway.networking.k8s.io_referencegrants.yaml \
|
||
|
|
gateway.networking.k8s.io_grpcroutes.yaml; do
|
||
|
|
${KUBECTL} apply -f "${GW_STD}/${crd}"
|
||
|
|
done
|
||
|
|
# TLSRoute and TCPRoute are experimental-channel only; Cilium 1.19+ operator
|
||
|
|
# requires TLSRoute or the reconciler loop errors before creating the GatewayClass.
|
||
|
|
# TCPRoute is needed for raw-TCP Gateway listeners (e.g. Radicle P2P).
|
||
|
|
GW_EXP="https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${CILIUM_GATEWAY_API_VERSION}/config/crd/experimental"
|
||
|
|
${KUBECTL} apply -f "${GW_EXP}/gateway.networking.k8s.io_tlsroutes.yaml"
|
||
|
|
${KUBECTL} apply -f "${GW_EXP}/gateway.networking.k8s.io_tcproutes.yaml"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_ALREADY_INSTALLED}" = "true" ]; then
|
||
|
|
echo "=== cilium: already installed — upgrading to v${CILIUM_VERSION} ==="
|
||
|
|
else
|
||
|
|
echo "=== cilium: installing v${CILIUM_VERSION} ==="
|
||
|
|
fi
|
||
|
|
|
||
|
|
INSTALL_ARGS=(
|
||
|
|
"--version" "${CILIUM_VERSION}"
|
||
|
|
"--set" "ipam.mode=kubernetes"
|
||
|
|
)
|
||
|
|
|
||
|
|
if [ -n "${CILIUM_CLUSTER}" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "cluster.name=${CILIUM_CLUSTER}"
|
||
|
|
"--set" "cluster.id=${CILIUM_CLUSTER_ID}"
|
||
|
|
)
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_KPR}" = "true" ]; then
|
||
|
|
K8S_HOST="${CILIUM_K8S_SERVICE_HOST:-}"
|
||
|
|
K8S_PORT="${CILIUM_K8S_SERVICE_PORT:-6443}"
|
||
|
|
if [ -z "${K8S_HOST}" ]; then
|
||
|
|
_api_url="$(${KUBECTL} config view --raw -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true)"
|
||
|
|
if [ -z "${_api_url}" ]; then
|
||
|
|
echo "ERROR: kube-proxy-replacement=true requires CILIUM_K8S_SERVICE_HOST or a reachable KUBECONFIG" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
_stripped="${_api_url#https://}"
|
||
|
|
K8S_HOST="${_stripped%%:*}"
|
||
|
|
_maybe_port="${_stripped##*:}"
|
||
|
|
[ "${_maybe_port}" != "${K8S_HOST}" ] && K8S_PORT="${_maybe_port}"
|
||
|
|
fi
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "kubeProxyReplacement=true"
|
||
|
|
"--set" "k8sServiceHost=${K8S_HOST}"
|
||
|
|
"--set" "k8sServicePort=${K8S_PORT}"
|
||
|
|
)
|
||
|
|
echo " kube-proxy replacement: enabled (apiserver=${K8S_HOST}:${K8S_PORT})"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_SOCKET_LB}" = "false" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "socketLB.enabled=false"
|
||
|
|
)
|
||
|
|
echo " socketLB: disabled (prevents eBPF socket hooks on host network)"
|
||
|
|
else
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "socketLB.enabled=true"
|
||
|
|
)
|
||
|
|
echo " socketLB: enabled (required for source-IP preservation via FIPs)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_ENVOY_ENABLED}" = "false" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "envoy.enabled=false"
|
||
|
|
)
|
||
|
|
echo " envoy: disabled (L7 policies unavailable — saves ~200MB RAM on constrained nodes)"
|
||
|
|
else
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "envoy.enabled=true"
|
||
|
|
)
|
||
|
|
echo " envoy: enabled (required for Gateway API and L7 policies)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_INGRESS}" = "true" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "ingressController.enabled=true"
|
||
|
|
"--set" "ingressController.loadbalancerMode=${CILIUM_INGRESS_LB}"
|
||
|
|
"--set" "ingressController.default=true"
|
||
|
|
)
|
||
|
|
echo " ingress controller: enabled (lb-mode=${CILIUM_INGRESS_LB})"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_GATEWAY_API}" = "true" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "gatewayAPI.enabled=true"
|
||
|
|
)
|
||
|
|
echo " gateway API: enabled (CRDs ${CILIUM_GATEWAY_API_VERSION})"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_L2_NEIGH}" = "true" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "l2NeighDiscovery.enabled=true"
|
||
|
|
)
|
||
|
|
echo " L2 neighbor discovery: enabled (prerequisite for L2 announcements)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_L2_ANNOUNCEMENT}" = "true" ]; then
|
||
|
|
INSTALL_ARGS+=(
|
||
|
|
"--set" "l2announcements.enabled=true"
|
||
|
|
)
|
||
|
|
echo " L2 announcements: enabled (ARP-based VIP advertisement on L2 segments)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ "${CILIUM_ALREADY_INSTALLED}" = "true" ]; then
|
||
|
|
cilium upgrade "${INSTALL_ARGS[@]}"
|
||
|
|
else
|
||
|
|
cilium install "${INSTALL_ARGS[@]}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "=== cilium: waiting for status OK (timeout 5m) ==="
|
||
|
|
cilium status --wait --wait-duration 5m0s || cilium status
|
||
|
|
echo "=== cilium: ready ==="
|
||
|
|
|
||
|
|
# Provision cluster-level public Gateway when public_fip is declared in component settings.
|
||
|
|
# This is idempotent: kubectl apply is safe to re-run on upgrade.
|
||
|
|
# When gateway_sharing_key is set (HCCM clusters with shared LB IPs), the Gateway
|
||
|
|
# is annotated with lbipam.cilium.io/sharing-key so multiple Gateways can share the IP.
|
||
|
|
#
|
||
|
|
# The Gateway manifests below apply ONLY the base http:80 listener — but
|
||
|
|
# spec.listeners is x-kubernetes-list-type=map (merge key "name"), and every
|
||
|
|
# app component appends its own https listener afterward via a JSON-patch
|
||
|
|
# add (catalog/lib/gateway-tls.sh:_lib_gateway_patch_https). Plain `kubectl
|
||
|
|
# apply` (client-side, 3-way merge from this script's own last-applied-config)
|
||
|
|
# does not know about that OpenAPI merge-key semantics and replaces the whole
|
||
|
|
# array — wiping every app-added listener on the next cilium upgrade. Server-
|
||
|
|
# Side Apply with a stable field-manager respects the merge key and only ever
|
||
|
|
# asserts this script's own "http" entry, leaving other managers' entries
|
||
|
|
# alone. --force-conflicts takes ownership of fields previously set by plain
|
||
|
|
# client-side apply (the original bootstrap), which would otherwise conflict.
|
||
|
|
CILIUM_GATEWAY_PUBLIC_IP="${CILIUM_GATEWAY_PUBLIC_IP:-}"
|
||
|
|
CILIUM_GATEWAY_NAME="${CILIUM_GATEWAY_NAME:-${CILIUM_CLUSTER}}"
|
||
|
|
CILIUM_GATEWAY_SHARING_KEY="${CILIUM_GATEWAY_SHARING_KEY:-}"
|
||
|
|
|
||
|
|
if [ "${CILIUM_GATEWAY_API}" = "true" ] && [ -n "${CILIUM_GATEWAY_PUBLIC_IP}" ] && [ -n "${CILIUM_GATEWAY_NAME}" ]; then
|
||
|
|
echo "=== cilium: provisioning cluster Gateway '${CILIUM_GATEWAY_NAME}' @ ${CILIUM_GATEWAY_PUBLIC_IP} ==="
|
||
|
|
${KUBECTL} apply -f - <<CLUSTER_POOL
|
||
|
|
---
|
||
|
|
apiVersion: cilium.io/v2
|
||
|
|
kind: CiliumLoadBalancerIPPool
|
||
|
|
metadata:
|
||
|
|
name: ${CILIUM_GATEWAY_NAME}-public-pool
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: ${CILIUM_GATEWAY_NAME}
|
||
|
|
app.kubernetes.io/managed-by: provisioning
|
||
|
|
spec:
|
||
|
|
blocks:
|
||
|
|
- cidr: "${CILIUM_GATEWAY_PUBLIC_IP}/32"
|
||
|
|
serviceSelector:
|
||
|
|
matchLabels:
|
||
|
|
io.cilium.gateway/owning-gateway: ${CILIUM_GATEWAY_NAME}
|
||
|
|
CLUSTER_POOL
|
||
|
|
|
||
|
|
if [ -n "${CILIUM_GATEWAY_SHARING_KEY}" ]; then
|
||
|
|
${KUBECTL} apply --server-side --field-manager=cilium-taskserv --force-conflicts -f - <<CLUSTER_GW
|
||
|
|
---
|
||
|
|
apiVersion: gateway.networking.k8s.io/v1
|
||
|
|
kind: Gateway
|
||
|
|
metadata:
|
||
|
|
name: ${CILIUM_GATEWAY_NAME}
|
||
|
|
namespace: kube-system
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: ${CILIUM_GATEWAY_NAME}
|
||
|
|
app.kubernetes.io/managed-by: provisioning
|
||
|
|
annotations:
|
||
|
|
lbipam.cilium.io/ips: "${CILIUM_GATEWAY_PUBLIC_IP}"
|
||
|
|
lbipam.cilium.io/sharing-key: "${CILIUM_GATEWAY_SHARING_KEY}"
|
||
|
|
spec:
|
||
|
|
gatewayClassName: cilium
|
||
|
|
listeners:
|
||
|
|
- name: http
|
||
|
|
port: 80
|
||
|
|
protocol: HTTP
|
||
|
|
allowedRoutes:
|
||
|
|
namespaces:
|
||
|
|
from: All
|
||
|
|
CLUSTER_GW
|
||
|
|
else
|
||
|
|
${KUBECTL} apply --server-side --field-manager=cilium-taskserv --force-conflicts -f - <<CLUSTER_GW
|
||
|
|
---
|
||
|
|
apiVersion: gateway.networking.k8s.io/v1
|
||
|
|
kind: Gateway
|
||
|
|
metadata:
|
||
|
|
name: ${CILIUM_GATEWAY_NAME}
|
||
|
|
namespace: kube-system
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: ${CILIUM_GATEWAY_NAME}
|
||
|
|
app.kubernetes.io/managed-by: provisioning
|
||
|
|
annotations:
|
||
|
|
lbipam.cilium.io/ips: "${CILIUM_GATEWAY_PUBLIC_IP}"
|
||
|
|
spec:
|
||
|
|
gatewayClassName: cilium
|
||
|
|
listeners:
|
||
|
|
- name: http
|
||
|
|
port: 80
|
||
|
|
protocol: HTTP
|
||
|
|
allowedRoutes:
|
||
|
|
namespaces:
|
||
|
|
from: All
|
||
|
|
CLUSTER_GW
|
||
|
|
fi
|
||
|
|
echo " cluster Gateway '${CILIUM_GATEWAY_NAME}' → ${CILIUM_GATEWAY_PUBLIC_IP}:80${CILIUM_GATEWAY_SHARING_KEY:+ (sharing-key: ${CILIUM_GATEWAY_SHARING_KEY})}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Optional private/VPN Gateway for internal services sharing an internal LB IP.
|
||
|
|
CILIUM_GATEWAY_PRIVATE_IP="${CILIUM_GATEWAY_PRIVATE_IP:-}"
|
||
|
|
CILIUM_GATEWAY_PRIVATE_NAME="${CILIUM_GATEWAY_PRIVATE_NAME:-private-gateway}"
|
||
|
|
|
||
|
|
if [ "${CILIUM_GATEWAY_API}" = "true" ] && [ -n "${CILIUM_GATEWAY_PRIVATE_IP}" ]; then
|
||
|
|
echo "=== cilium: provisioning private Gateway '${CILIUM_GATEWAY_PRIVATE_NAME}' @ ${CILIUM_GATEWAY_PRIVATE_IP} ==="
|
||
|
|
${KUBECTL} apply -f - <<PRIVATE_POOL
|
||
|
|
---
|
||
|
|
apiVersion: cilium.io/v2
|
||
|
|
kind: CiliumLoadBalancerIPPool
|
||
|
|
metadata:
|
||
|
|
name: ${CILIUM_GATEWAY_PRIVATE_NAME}-pool
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: ${CILIUM_GATEWAY_PRIVATE_NAME}
|
||
|
|
app.kubernetes.io/managed-by: provisioning
|
||
|
|
spec:
|
||
|
|
blocks:
|
||
|
|
- cidr: "${CILIUM_GATEWAY_PRIVATE_IP}/32"
|
||
|
|
serviceSelector:
|
||
|
|
matchLabels:
|
||
|
|
io.cilium.gateway/owning-gateway: ${CILIUM_GATEWAY_PRIVATE_NAME}
|
||
|
|
PRIVATE_POOL
|
||
|
|
|
||
|
|
${KUBECTL} apply --server-side --field-manager=cilium-taskserv --force-conflicts -f - <<PRIVATE_GW
|
||
|
|
---
|
||
|
|
apiVersion: gateway.networking.k8s.io/v1
|
||
|
|
kind: Gateway
|
||
|
|
metadata:
|
||
|
|
name: ${CILIUM_GATEWAY_PRIVATE_NAME}
|
||
|
|
namespace: kube-system
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: ${CILIUM_GATEWAY_PRIVATE_NAME}
|
||
|
|
app.kubernetes.io/managed-by: provisioning
|
||
|
|
annotations:
|
||
|
|
lbipam.cilium.io/ips: "${CILIUM_GATEWAY_PRIVATE_IP}"
|
||
|
|
spec:
|
||
|
|
gatewayClassName: cilium
|
||
|
|
listeners:
|
||
|
|
- name: http
|
||
|
|
port: 80
|
||
|
|
protocol: HTTP
|
||
|
|
allowedRoutes:
|
||
|
|
namespaces:
|
||
|
|
from: All
|
||
|
|
PRIVATE_GW
|
||
|
|
echo " private Gateway '${CILIUM_GATEWAY_PRIVATE_NAME}' → ${CILIUM_GATEWAY_PRIVATE_IP}:80 (VPN/internal only)"
|
||
|
|
fi
|