268 lines
12 KiB
Bash
268 lines
12 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
[ -r "./env-longhorn" ] && . ./env-longhorn
|
||
|
|
|
||
|
|
LONGHORN_VERSION="${LONGHORN_VERSION:-1.11.1}"
|
||
|
|
LONGHORN_NAMESPACE="${LONGHORN_NAMESPACE:-longhorn-system}"
|
||
|
|
LONGHORN_STORAGE_CLASS="${LONGHORN_STORAGE_CLASS:-longhorn}"
|
||
|
|
LONGHORN_IS_DEFAULT="${LONGHORN_IS_DEFAULT:-true}"
|
||
|
|
LONGHORN_RECLAIM_POLICY="${LONGHORN_RECLAIM_POLICY:-Retain}"
|
||
|
|
LONGHORN_ALLOW_EXPANSION="${LONGHORN_ALLOW_EXPANSION:-true}"
|
||
|
|
LONGHORN_REPLICA_COUNT="${LONGHORN_REPLICA_COUNT:-2}"
|
||
|
|
LONGHORN_DATA_PATH="${LONGHORN_DATA_PATH:-/var/lib/longhorn}"
|
||
|
|
LONGHORN_DEMOTE_CLASS="${LONGHORN_DEMOTE_CLASS:-hcloud-volumes}"
|
||
|
|
LONGHORN_STORAGE_NET_IF="${LONGHORN_STORAGE_NET_IF:-}"
|
||
|
|
LONGHORN_TAINT_TOLERATION="${LONGHORN_TAINT_TOLERATION:-}"
|
||
|
|
LONGHORN_STORAGE_NODES="${LONGHORN_STORAGE_NODES:-}"
|
||
|
|
LONGHORN_UI_ENABLED="${LONGHORN_UI_ENABLED:-true}"
|
||
|
|
|
||
|
|
# When storage nodes are restricted via labels, tell Longhorn to only create
|
||
|
|
# default disks on those labeled nodes — prevents OS disk of cp/wrk nodes
|
||
|
|
# from being silently consumed as Longhorn storage (adr-020).
|
||
|
|
CREATE_DISK_LABELED_LINE=""
|
||
|
|
if [ -n "${LONGHORN_STORAGE_NODES}" ]; then
|
||
|
|
CREATE_DISK_LABELED_LINE=" create-default-disk-labeled-nodes: true"
|
||
|
|
fi
|
||
|
|
LONGHORN_UI_INGRESS_ENABLED="${LONGHORN_UI_INGRESS_ENABLED:-false}"
|
||
|
|
LONGHORN_UI_INGRESS_HOST="${LONGHORN_UI_INGRESS_HOST:-}"
|
||
|
|
LONGHORN_UI_INGRESS_CLASS="${LONGHORN_UI_INGRESS_CLASS:-cilium}"
|
||
|
|
LONGHORN_UI_INGRESS_TLS_SECRET="${LONGHORN_UI_INGRESS_TLS_SECRET:-}"
|
||
|
|
|
||
|
|
MANIFEST_URL="https://raw.githubusercontent.com/longhorn/longhorn/v${LONGHORN_VERSION}/deploy/longhorn.yaml"
|
||
|
|
|
||
|
|
# ── kubectl + kubeconfig resolution ──────────────────────────────────────────
|
||
|
|
# Mirror the pattern used by cilium, democratic_csi, hetzner_csi:
|
||
|
|
# k0s kubectl takes precedence; fall back to system kubectl.
|
||
|
|
# KUBECONFIG fallback chain: env override → kubeadm admin.conf → k0s pki → generate.
|
||
|
|
# The kubernetes component patches the server address to 127.0.0.1 in all
|
||
|
|
# kubeconfigs so local connectivity works regardless of Hetzner's networking.
|
||
|
|
if command -v k0s &>/dev/null && k0s kubectl version --client=true &>/dev/null; then
|
||
|
|
kubectl() { k0s kubectl "$@"; }
|
||
|
|
K0S_DATA_DIR="${K0S_DATA_DIR:-/var/lib/k0s}"
|
||
|
|
if [ -z "${KUBECONFIG:-}" ] || [ ! -f "${KUBECONFIG}" ]; then
|
||
|
|
if [ -f "${K0S_DATA_DIR}/pki/admin.conf" ]; then
|
||
|
|
export KUBECONFIG="${K0S_DATA_DIR}/pki/admin.conf"
|
||
|
|
else
|
||
|
|
export KUBECONFIG="/tmp/k0s-longhorn.conf"
|
||
|
|
k0s kubeconfig admin > "${KUBECONFIG}"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
elif command -v kubectl &>/dev/null; then
|
||
|
|
if [ -z "${KUBECONFIG:-}" ] || [ ! -f "${KUBECONFIG}" ]; then
|
||
|
|
if [ -f "/etc/kubernetes/admin.conf" ]; then
|
||
|
|
export KUBECONFIG="/etc/kubernetes/admin.conf"
|
||
|
|
elif [ -f "/root/.kube/config" ]; then
|
||
|
|
export KUBECONFIG="/root/.kube/config"
|
||
|
|
else
|
||
|
|
echo "ERROR: no kubeconfig found — run this taskserv after kubernetes(cp) completes" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo "ERROR: neither kubectl nor k0s found" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "=== longhorn: verifying cluster connectivity ==="
|
||
|
|
if ! kubectl cluster-info --request-timeout=10s &>/dev/null; then
|
||
|
|
echo "ERROR: kubectl cannot reach the cluster API (KUBECONFIG=${KUBECONFIG:-<default>})" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Auto-label storage nodes ─────────────────────────────────────────────────
|
||
|
|
# When LONGHORN_STORAGE_NODES is provided, apply the disk-creation label before
|
||
|
|
# the availability check. --overwrite is idempotent for re-runs.
|
||
|
|
if [ -n "${LONGHORN_STORAGE_NODES}" ]; then
|
||
|
|
echo "=== longhorn: labeling storage nodes ==="
|
||
|
|
for node in ${LONGHORN_STORAGE_NODES}; do
|
||
|
|
if kubectl get node "${node}" &>/dev/null; then
|
||
|
|
kubectl label node "${node}" \
|
||
|
|
node.longhorn.io/create-default-disk=true \
|
||
|
|
--overwrite
|
||
|
|
echo " labeled: ${node}"
|
||
|
|
else
|
||
|
|
echo "WARNING: node ${node} not found in cluster — skipping" >&2
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Storage node availability check ─────────────────────────────────────────
|
||
|
|
# Hard requirement: at least LONGHORN_REPLICA_COUNT nodes with the Longhorn
|
||
|
|
# label must be Ready. Installing with fewer nodes means every PVC create will
|
||
|
|
# fail immediately due to insufficient replicas — abort here instead.
|
||
|
|
echo "=== longhorn: verifying storage node availability (need >= ${LONGHORN_REPLICA_COUNT}) ==="
|
||
|
|
READY_STORAGE_NODES=$(kubectl get nodes \
|
||
|
|
-l "node.longhorn.io/create-default-disk=true" \
|
||
|
|
--no-headers 2>/dev/null \
|
||
|
|
| grep -v "NotReady\|SchedulingDisabled" \
|
||
|
|
| wc -l)
|
||
|
|
|
||
|
|
if [ "${READY_STORAGE_NODES}" -lt "${LONGHORN_REPLICA_COUNT}" ]; then
|
||
|
|
echo "ERROR: found ${READY_STORAGE_NODES} Ready storage node(s), need ${LONGHORN_REPLICA_COUNT}." >&2
|
||
|
|
echo " Provision and label all storage nodes before running the longhorn taskserv." >&2
|
||
|
|
echo " Label: kubectl label node <name> node.longhorn.io/create-default-disk=true" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo "=== longhorn: ${READY_STORAGE_NODES} Ready storage node(s) confirmed ==="
|
||
|
|
|
||
|
|
# Verify storage nodes are reachable via private IPs (10.0.8.x)
|
||
|
|
NON_PRIVATE=$(kubectl get nodes \
|
||
|
|
-l "node.longhorn.io/create-default-disk=true" \
|
||
|
|
-o jsonpath='{range .items[*]}{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}{end}' \
|
||
|
|
| grep -v "^10\." || true)
|
||
|
|
if [ -n "${NON_PRIVATE}" ]; then
|
||
|
|
echo "ERROR: storage node(s) have non-private internal IP(s): ${NON_PRIVATE}" >&2
|
||
|
|
echo " All storage nodes must use private network addresses (10.0.8.0/24)." >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Namespace + default settings ─────────────────────────────────────────────
|
||
|
|
echo "=== longhorn: creating namespace ${LONGHORN_NAMESPACE} ==="
|
||
|
|
kubectl create namespace "${LONGHORN_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
|
||
|
|
|
||
|
|
# Build the storage-network setting line — empty means Longhorn auto-selects using
|
||
|
|
# the K8s node internal IP (already private on Hetzner).
|
||
|
|
STORAGE_NET_LINE=""
|
||
|
|
if [ -n "${LONGHORN_STORAGE_NET_IF}" ]; then
|
||
|
|
STORAGE_NET_LINE=" storage-network-interface: ${LONGHORN_STORAGE_NET_IF}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "=== longhorn: applying default-setting ConfigMap ==="
|
||
|
|
kubectl apply -f - <<SETTINGS_EOF
|
||
|
|
apiVersion: v1
|
||
|
|
kind: ConfigMap
|
||
|
|
metadata:
|
||
|
|
name: longhorn-default-setting
|
||
|
|
namespace: ${LONGHORN_NAMESPACE}
|
||
|
|
data:
|
||
|
|
default-setting.yaml: |-
|
||
|
|
default-replica-count: ${LONGHORN_REPLICA_COUNT}
|
||
|
|
default-data-path: ${LONGHORN_DATA_PATH}
|
||
|
|
storage-over-provisioning-percentage: 200
|
||
|
|
storage-minimal-available-percentage: 15
|
||
|
|
node-down-pod-deletion-policy: delete-both-statefulset-and-deployment-pod
|
||
|
|
${STORAGE_NET_LINE}
|
||
|
|
${CREATE_DISK_LABELED_LINE}
|
||
|
|
SETTINGS_EOF
|
||
|
|
|
||
|
|
# ── Manifest apply ────────────────────────────────────────────────────────────
|
||
|
|
echo "=== longhorn: applying v${LONGHORN_VERSION} manifest (kubectl apply) ==="
|
||
|
|
# Strip the bundled StorageClass from the stream — we manage it explicitly below
|
||
|
|
# to control reclaimPolicy (immutable field) and is-default annotation.
|
||
|
|
curl -fsSL "${MANIFEST_URL}" \
|
||
|
|
| awk 'BEGIN{skip=0} /^kind: StorageClass/{skip=1} /^---/{if(skip){skip=0; next}} {if(!skip)print}' \
|
||
|
|
| kubectl apply -f -
|
||
|
|
|
||
|
|
echo "=== longhorn: waiting for manager DaemonSet rollout ==="
|
||
|
|
kubectl rollout status daemonset/longhorn-manager \
|
||
|
|
-n "${LONGHORN_NAMESPACE}" --timeout=300s
|
||
|
|
|
||
|
|
echo "=== longhorn: waiting for driver deployer ==="
|
||
|
|
kubectl rollout status deployment/longhorn-driver-deployer \
|
||
|
|
-n "${LONGHORN_NAMESPACE}" --timeout=180s
|
||
|
|
|
||
|
|
# ── StorageClass ──────────────────────────────────────────────────────────────
|
||
|
|
CURRENT_POLICY=$(kubectl get storageclass "${LONGHORN_STORAGE_CLASS}" \
|
||
|
|
-o jsonpath='{.reclaimPolicy}' 2>/dev/null || echo "")
|
||
|
|
|
||
|
|
# reclaimPolicy is immutable — recreate if it changed
|
||
|
|
if [ -n "${CURRENT_POLICY}" ] && [ "${CURRENT_POLICY}" != "${LONGHORN_RECLAIM_POLICY}" ]; then
|
||
|
|
echo "=== longhorn: reclaimPolicy changed — recreating StorageClass ${LONGHORN_STORAGE_CLASS} ==="
|
||
|
|
kubectl delete storageclass "${LONGHORN_STORAGE_CLASS}" 2>/dev/null || true
|
||
|
|
CURRENT_POLICY=""
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ -z "${CURRENT_POLICY}" ]; then
|
||
|
|
echo "=== longhorn: creating StorageClass ${LONGHORN_STORAGE_CLASS} (reclaim=${LONGHORN_RECLAIM_POLICY}, default=${LONGHORN_IS_DEFAULT}) ==="
|
||
|
|
kubectl apply -f - <<SC_EOF
|
||
|
|
apiVersion: storage.k8s.io/v1
|
||
|
|
kind: StorageClass
|
||
|
|
metadata:
|
||
|
|
name: ${LONGHORN_STORAGE_CLASS}
|
||
|
|
annotations:
|
||
|
|
storageclass.kubernetes.io/is-default-class: "${LONGHORN_IS_DEFAULT}"
|
||
|
|
provisioner: driver.longhorn.io
|
||
|
|
reclaimPolicy: ${LONGHORN_RECLAIM_POLICY}
|
||
|
|
allowVolumeExpansion: ${LONGHORN_ALLOW_EXPANSION}
|
||
|
|
volumeBindingMode: Immediate
|
||
|
|
parameters:
|
||
|
|
numberOfReplicas: "${LONGHORN_REPLICA_COUNT}"
|
||
|
|
dataLocality: "best-effort"
|
||
|
|
SC_EOF
|
||
|
|
else
|
||
|
|
kubectl annotate storageclass "${LONGHORN_STORAGE_CLASS}" \
|
||
|
|
"storageclass.kubernetes.io/is-default-class=${LONGHORN_IS_DEFAULT}" --overwrite
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Demote previous default StorageClass
|
||
|
|
if [ "${LONGHORN_IS_DEFAULT}" = "true" ] && [ -n "${LONGHORN_DEMOTE_CLASS}" ]; then
|
||
|
|
if kubectl get storageclass "${LONGHORN_DEMOTE_CLASS}" &>/dev/null; then
|
||
|
|
echo "=== longhorn: demoting ${LONGHORN_DEMOTE_CLASS} (removing is-default annotation) ==="
|
||
|
|
kubectl annotate storageclass "${LONGHORN_DEMOTE_CLASS}" \
|
||
|
|
"storageclass.kubernetes.io/is-default-class-" --overwrite 2>/dev/null || true
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Ingress for UI ────────────────────────────────────────────────────────────
|
||
|
|
if [ "${LONGHORN_UI_INGRESS_ENABLED}" = "true" ]; then
|
||
|
|
if [ -z "${LONGHORN_UI_INGRESS_HOST}" ]; then
|
||
|
|
echo "ERROR: ui_ingress_enabled=true but ui_ingress_host is not set" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "=== longhorn: creating Ingress for UI (class=${LONGHORN_UI_INGRESS_CLASS}, host=${LONGHORN_UI_INGRESS_HOST}) ==="
|
||
|
|
|
||
|
|
TLS_BLOCK=""
|
||
|
|
if [ -n "${LONGHORN_UI_INGRESS_TLS_SECRET}" ]; then
|
||
|
|
TLS_BLOCK=" tls:
|
||
|
|
- hosts:
|
||
|
|
- ${LONGHORN_UI_INGRESS_HOST}
|
||
|
|
secretName: ${LONGHORN_UI_INGRESS_TLS_SECRET}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
kubectl apply -f - <<INGRESS_EOF
|
||
|
|
apiVersion: networking.k8s.io/v1
|
||
|
|
kind: Ingress
|
||
|
|
metadata:
|
||
|
|
name: longhorn-ui
|
||
|
|
namespace: ${LONGHORN_NAMESPACE}
|
||
|
|
annotations:
|
||
|
|
kubernetes.io/ingress.class: "${LONGHORN_UI_INGRESS_CLASS}"
|
||
|
|
spec:
|
||
|
|
ingressClassName: ${LONGHORN_UI_INGRESS_CLASS}
|
||
|
|
${TLS_BLOCK}
|
||
|
|
rules:
|
||
|
|
- host: ${LONGHORN_UI_INGRESS_HOST}
|
||
|
|
http:
|
||
|
|
paths:
|
||
|
|
- path: /
|
||
|
|
pathType: Prefix
|
||
|
|
backend:
|
||
|
|
service:
|
||
|
|
name: longhorn-frontend
|
||
|
|
port:
|
||
|
|
number: 80
|
||
|
|
INGRESS_EOF
|
||
|
|
|
||
|
|
echo "=== longhorn: UI ingress ready — http${LONGHORN_UI_INGRESS_TLS_SECRET:+s}://${LONGHORN_UI_INGRESS_HOST}/ ==="
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Summary ───────────────────────────────────────────────────────────────────
|
||
|
|
echo ""
|
||
|
|
echo "=== longhorn: installation complete ==="
|
||
|
|
echo " version = ${LONGHORN_VERSION}"
|
||
|
|
echo " namespace = ${LONGHORN_NAMESPACE}"
|
||
|
|
echo " storage_class = ${LONGHORN_STORAGE_CLASS} (default=${LONGHORN_IS_DEFAULT})"
|
||
|
|
echo " replica_count = ${LONGHORN_REPLICA_COUNT}"
|
||
|
|
echo " data_path = ${LONGHORN_DATA_PATH}"
|
||
|
|
echo " storage_nodes = ${READY_STORAGE_NODES}"
|
||
|
|
if [ "${LONGHORN_UI_INGRESS_ENABLED}" = "true" ]; then
|
||
|
|
echo " ui = http${LONGHORN_UI_INGRESS_TLS_SECRET:+s}://${LONGHORN_UI_INGRESS_HOST}/"
|
||
|
|
else
|
||
|
|
echo " ui (port-fwd) = kubectl port-forward -n ${LONGHORN_NAMESPACE} svc/longhorn-frontend 8000:80"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
echo " nodes: kubectl get nodes.longhorn.io -n ${LONGHORN_NAMESPACE}"
|
||
|
|
echo " vols: kubectl get volumes.longhorn.io -n ${LONGHORN_NAMESPACE}"
|