617 lines
29 KiB
Bash
Executable file
617 lines
29 KiB
Bash
Executable file
#!/bin/bash
|
||
[ "$1" == "-h" ] && echo "install-kubernetes.sh install|update|reinstall|remove|fullremove|makejoin|restart|config" && exit 1
|
||
|
||
SERVICE_NAME="kubelet"
|
||
SCRIPTS_DEPLOY_PATH="/opt/kubernetes"
|
||
|
||
[ -r "env-kubernetes" ] && . ./env-kubernetes
|
||
[ -r "common.sh" ] && . ./common.sh
|
||
|
||
export LC_CTYPE=C.UTF-8 LANG=C.UTF-8
|
||
|
||
CMD_TSK=${CMD_TSK:-${1:-install}}
|
||
INSTALL_LOG=${INSTALL_LOG:-/tmp/k8s.log}
|
||
WORK_PATH=${WORK_PATH:-/tmp}
|
||
[ ! -d "$WORK_PATH" ] && sudo mkdir -p "$WORK_PATH"
|
||
[ ! -d "$(dirname "$INSTALL_LOG")" ] && mkdir -p "$(dirname "$INSTALL_LOG")"
|
||
|
||
[ -z "$CLUSTER_NAME" ] && echo "No CLUSTER_NAME value" && exit 1
|
||
[ -z "$VERSION" ] && echo "No VERSION value" && exit 1
|
||
|
||
if [ -z "$K8S_MODE" ]; then
|
||
if [[ "$HOSTNAME" == *-cp-* ]]; then
|
||
K8S_MODE="controlplane"
|
||
else
|
||
K8S_MODE="worker"
|
||
fi
|
||
fi
|
||
|
||
_check_resolution() {
|
||
local ip
|
||
[ "$K8S_MODE" == "controlplane" ] && local clustername="$CLUSTER_NAME"
|
||
# Do NOT remove the 127.0.1.1 entry — Debian default, keeps hostname self-routable
|
||
# (matches behaviour of manually-provisioned nodes like ko where any local address
|
||
# reaches the apiserver).
|
||
ip=$(grep "$MAIN_IP" /etc/hosts | grep -v "^#" | awk '{print $1}')
|
||
[ -z "$ip" ] && echo "$MAIN_IP $HOSTNAME ${clustername:-}" | sudo tee -a /etc/hosts >/dev/null
|
||
if [ "$HOSTNAME" != "$(cat /etc/hostname)" ]; then
|
||
echo "$HOSTNAME" | sudo tee /etc/hostname >/dev/null
|
||
sudo hostname "$HOSTNAME"
|
||
fi
|
||
}
|
||
|
||
_off_swap() {
|
||
local fs_swap
|
||
fs_swap=$(grep -v "^#" /etc/fstab | grep swap)
|
||
if [ -n "$fs_swap" ]; then
|
||
sudo sed -i "s;$fs_swap;#$fs_swap;g" /etc/fstab
|
||
fi
|
||
sudo swapoff -a
|
||
}
|
||
|
||
_update_binary() {
|
||
local curr_vers
|
||
curr_vers=$(kubectl version 2>/dev/null | grep Client | awk '{print $3}' | sed 's/^v//g')
|
||
[ "v$curr_vers" == "$K8S_VERSION" ] && return 0
|
||
chmod 1777 /tmp
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -q
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https gnupg2 curl
|
||
sudo rm -f /etc/apt/keyrings/kubernetes-apt-keyring.gpg
|
||
curl -fsSL "https://pkgs.k8s.io/core:/stable:/v${MAJOR_VERSION}/deb/Release.key" |
|
||
sudo gpg --batch --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
|
||
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${MAJOR_VERSION}/deb/ /" |
|
||
sudo tee /etc/apt/sources.list.d/kubernetes.list >/dev/null
|
||
sudo chmod 644 /etc/apt/sources.list.d/kubernetes.list
|
||
_off_swap
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -q
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-mark unhold kubelet kubectl kubeadm
|
||
local pkg_ver="${K8S_VERSION#v}-1.1"
|
||
if ! sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades \
|
||
"kubectl=${pkg_ver}" "kubelet=${pkg_ver}" "kubeadm=${pkg_ver}"; then
|
||
echo "error installing kubernetes packages version ${pkg_ver}"
|
||
return 1
|
||
fi
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-mark hold kubelet kubectl kubeadm
|
||
}
|
||
|
||
_kubernetes_kube() {
|
||
local user=${1:-root} home_user=${2:-/root}
|
||
local uid gid
|
||
uid=$(sudo id -u "$user" 2>/dev/null)
|
||
gid=$(sudo id -g "$user" 2>/dev/null)
|
||
[ ! -f "/etc/kubernetes/admin.conf" ] && return 0
|
||
sudo mkdir -p /root/.kube
|
||
sudo cp -f /etc/kubernetes/admin.conf /root/.kube/config
|
||
sudo chown root:root /root/.kube/config
|
||
# Hetzner: node cannot reach its own private IP — patch local kubeconfig to loopback.
|
||
# admin.conf keeps the real IP for external kubectl access.
|
||
[ -n "$MAIN_IP" ] && sudo sed -i \
|
||
"s|server: https://${MAIN_IP}:6443|server: https://127.0.0.1:6443|g" \
|
||
/root/.kube/config
|
||
if [ "$uid" != "0" ]; then
|
||
mkdir -p "$home_user/.kube"
|
||
sudo cp -f /etc/kubernetes/admin.conf "$home_user/.kube/config"
|
||
sudo chown -R "$uid:$gid" "$home_user/.kube"
|
||
[ -n "$MAIN_IP" ] && sudo sed -i \
|
||
"s|server: https://${MAIN_IP}:6443|server: https://127.0.0.1:6443|g" \
|
||
"$home_user/.kube/config"
|
||
fi
|
||
local has_aliases
|
||
has_aliases=$(grep bash_aliases "$HOME/.bashrc" 2>/dev/null)
|
||
[ -z "$has_aliases" ] && echo "[ -f ~/.bash_aliases ] && . ~/.bash_aliases" | sudo tee -a "$HOME/.bashrc" >/dev/null
|
||
}
|
||
|
||
_kubectl_apply() {
|
||
export KUBECONFIG=/etc/kubernetes/admin.conf
|
||
[ ! -r "$KUBECONFIG" ] && echo "$KUBECONFIG not found" && return 1
|
||
[ ! -r "$1" ] && echo "File $1 not found" && return 1
|
||
kubectl apply -f "$1" || echo "Error kubectl apply $1"
|
||
}
|
||
|
||
_kubernetes_taint() {
|
||
# K8s 1.24+ uses control-plane taint; master is deprecated. Use --timeout to avoid
|
||
# hanging if apiserver is briefly unresponsive after init.
|
||
case "$TAINT_NODE" in
|
||
no_schedule)
|
||
kubectl taint nodes "$HOSTNAME" node-role.kubernetes.io/control-plane:NoSchedule \
|
||
--timeout=30s 2>/dev/null || true
|
||
;;
|
||
schedule)
|
||
# Remove any lingering taint so the control-plane can schedule workloads.
|
||
kubectl taint nodes "$HOSTNAME" node-role.kubernetes.io/control-plane:NoSchedule- \
|
||
--timeout=30s 2>/dev/null || true
|
||
;;
|
||
esac
|
||
return 0
|
||
}
|
||
|
||
_kubernetes_label_roles() {
|
||
[ -z "${NODE_ROLES:-}" ] && return 0
|
||
local kubeconfig="/etc/kubernetes/admin.conf"
|
||
for role in ${NODE_ROLES//,/ }; do
|
||
if [ -f "$kubeconfig" ]; then
|
||
kubectl --kubeconfig="$kubeconfig" label node "$HOSTNAME" \
|
||
"node-role.kubernetes.io/${role}=" --overwrite 2>/dev/null || true
|
||
else
|
||
# Worker nodes lack admin.conf — delegate to the control-plane via SSH.
|
||
ssh -o StrictHostKeyChecking=no -o BatchMode=yes "$K8S_MASTER" \
|
||
kubectl label node "$HOSTNAME" \
|
||
"node-role.kubernetes.io/${role}=" --overwrite 2>/dev/null || true
|
||
fi
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | node role set: node-role.kubernetes.io/${role}" | sudo tee -a "$INSTALL_LOG"
|
||
done
|
||
}
|
||
|
||
# CNI is now installed as a separate taskserv (cilium) in the formula DAG.
|
||
# kubernetes → cilium ordering is enforced by the formula node dependencies.
|
||
# Keeping the function as no-op so existing call sites don't break.
|
||
_kubernetes_cni() { return 0; }
|
||
|
||
_kubernetes_addons() {
|
||
local yaml_file
|
||
for item in ${K8S_ADDONS//,/ }; do
|
||
if [ -r "addons/$item/install.sh" ]; then
|
||
echo "Install addon $item" | sudo tee -a "$INSTALL_LOG"
|
||
# shellcheck disable=SC1090
|
||
. "addons/$item/install.sh"
|
||
if [ "$item" == "istio" ] && [ -n "$K8S_EXTERNAL_IPS" ]; then
|
||
yaml_file=/tmp/externalIPs.yaml
|
||
{
|
||
echo "spec:"
|
||
echo " externalIPs:"
|
||
for ip in ${K8S_EXTERNAL_IPS//,/ }; do echo " - $ip"; done
|
||
} >"$yaml_file"
|
||
kubectl patch service -n istio-system istio-ingressgateway --type merge --patch-file "$yaml_file"
|
||
fi
|
||
fi
|
||
done
|
||
}
|
||
|
||
_reset_kubernetes() {
|
||
sudo rm -f "$INSTALL_LOG"
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Kubernetes RESET on $HOSTNAME ..." | sudo tee "$INSTALL_LOG"
|
||
|
||
sudo systemctl stop kubelet 2>/dev/null || true
|
||
|
||
for _cid in $(crictl ps -q 2>/dev/null); do
|
||
crictl stop "$_cid" 2>/dev/null || true
|
||
crictl rm -f "$_cid" 2>/dev/null || true
|
||
done
|
||
for _pod in $(crictl pods -q 2>/dev/null); do
|
||
crictl stopp "$_pod" 2>/dev/null || true
|
||
crictl rmp "$_pod" 2>/dev/null || true
|
||
done
|
||
|
||
# Full kubelet state wipe: config.yaml and kubeadm-flags.env are written by
|
||
# kubeadm init and carry the old CA reference — keeping them causes the new
|
||
# kubelet to present stale certs before bootstrap-kubelet.conf can take over.
|
||
sudo rm -rf /var/lib/kubelet/pki /var/lib/kubelet/config.yaml \
|
||
/var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/cpu_manager_state \
|
||
/var/lib/kubelet/memory_manager_state /var/lib/kubelet/device-plugins
|
||
|
||
# Clean stale CNI state and iptables rules left by previous installs.
|
||
sudo rm -rf /etc/cni/net.d /run/flannel /run/cilium
|
||
for _dev in cilium_vxlan cilium_host cilium_net lxc_health kube-ipvs0 flannel.1 cni0; do
|
||
ip link delete "$_dev" 2>/dev/null || true
|
||
done
|
||
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X 2>/dev/null || true
|
||
ip6tables -F && ip6tables -t nat -F && ip6tables -t mangle -F && ip6tables -X 2>/dev/null || true
|
||
|
||
if [ "$ETCD_MODE" == "external" ]; then
|
||
sudo rm -rf /etc/kubernetes
|
||
|
||
local etcd_cfg=/etc/etcd/config.yaml
|
||
local etcd_ca etcd_cert etcd_key
|
||
etcd_ca=$(sudo awk '/trusted-ca-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
etcd_cert=$(sudo awk '/cert-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
etcd_key=$(sudo awk '/key-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
if [ -n "$etcd_ca" ] && [ -n "$etcd_cert" ] && [ -n "$etcd_key" ]; then
|
||
# Wipe current cluster prefix only. CLUSTER_NAME is set from the taskserv
|
||
# config (e.g. "sgoyol"), which matches the etcd-prefix used in kubeadm init.
|
||
local _all_prefixes="/${CLUSTER_NAME}/"
|
||
for _pfx in $_all_prefixes; do
|
||
local _count
|
||
# --keys-only and --count-only are mutually exclusive in etcd 3.6+.
|
||
# --count-only requires --write-out=fields to print to stdout.
|
||
_count=$(ETCDCTL_API=3 sudo etcdctl \
|
||
--endpoints="https://${MAIN_IP}:2379" \
|
||
--cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
|
||
get "$_pfx" --prefix --count-only --write-out=fields 2>/dev/null |
|
||
awk -F'\t' '/Count/{print $2}' || echo 0)
|
||
if [ "${_count:-0}" -gt 0 ] 2>/dev/null; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Wiping etcd prefix $_pfx (${_count} keys) ..." | sudo tee -a "$INSTALL_LOG"
|
||
ETCDCTL_API=3 sudo etcdctl \
|
||
--endpoints="https://${MAIN_IP}:2379" \
|
||
--cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
|
||
del "$_pfx" --prefix 2>&1 | sudo tee -a "$INSTALL_LOG" ||
|
||
echo "Warning: etcd wipe failed for $_pfx" | sudo tee -a "$INSTALL_LOG"
|
||
fi
|
||
done
|
||
else
|
||
echo "Warning: cannot parse etcd TLS config from $etcd_cfg, skipping prefix wipe" | sudo tee -a "$INSTALL_LOG"
|
||
fi
|
||
else
|
||
sudo kubeadm reset -f 2>/dev/null || true
|
||
sudo rm -rf /etc/kubernetes
|
||
fi
|
||
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Kubernetes ready for reinstall on $HOSTNAME" | sudo tee -a "$INSTALL_LOG"
|
||
}
|
||
|
||
_etcd_wipe_prefix() {
|
||
# Wipe the cluster's etcd key-space before kubeadm init so stale data from
|
||
# previous installs never interferes. Runs on every install, not just reinstall.
|
||
[ "$ETCD_MODE" != "external" ] && return 0
|
||
local etcd_cfg=/etc/etcd/config.yaml
|
||
local etcd_ca etcd_cert etcd_key
|
||
etcd_ca=$(sudo awk '/trusted-ca-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
etcd_cert=$(sudo awk '/cert-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
etcd_key=$(sudo awk '/key-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
|
||
[ -z "$etcd_ca" ] || [ -z "$etcd_cert" ] || [ -z "$etcd_key" ] &&
|
||
echo "Warning: cannot parse etcd TLS config — skipping prefix wipe" && return 0
|
||
local _pfx="/${CLUSTER_NAME}/"
|
||
local _count
|
||
_count=$(ETCDCTL_API=3 sudo etcdctl \
|
||
--endpoints="https://${MAIN_IP}:2379" \
|
||
--cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
|
||
get "$_pfx" --prefix --count-only --write-out=fields 2>/dev/null |
|
||
awk -F'\t' '/Count/{print $2}' || echo 0)
|
||
if [ "${_count:-0}" -gt 0 ] 2>/dev/null; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Wiping etcd prefix $_pfx (${_count} keys) ..." | sudo tee -a "$INSTALL_LOG"
|
||
ETCDCTL_API=3 sudo etcdctl \
|
||
--endpoints="https://${MAIN_IP}:2379" \
|
||
--cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
|
||
del "$_pfx" --prefix 2>&1 | sudo tee -a "$INSTALL_LOG" ||
|
||
echo "Warning: etcd wipe failed for $_pfx" | sudo tee -a "$INSTALL_LOG"
|
||
else
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | etcd prefix $_pfx is empty — nothing to wipe"
|
||
fi
|
||
}
|
||
|
||
_install_master_0() {
|
||
_check_resolution
|
||
[ ! -r "resources/$K8S_CONFIG" ] && echo "resources/$K8S_CONFIG not found" && exit 1
|
||
# Always wipe the etcd prefix before kubeadm init — idempotent, safe on empty prefix.
|
||
_etcd_wipe_prefix
|
||
if [ "$ETCD_MODE" == "external" ] && [ -d "etcd_certs" ]; then
|
||
[ ! -d "/etc/kubernetes/pki/etcd" ] && sudo mkdir -p /etc/kubernetes/pki/etcd
|
||
sudo cp -pr etcd_certs/* /etc/kubernetes/pki/etcd
|
||
# kubeadm never generates apiserver-etcd-client in external etcd mode.
|
||
# Pre-provision it using the healthcheck-client cert (clientAuth, signed by etcd CA).
|
||
if [ -f "/etc/kubernetes/pki/etcd/healthcheck-client.crt" ]; then
|
||
sudo cp /etc/kubernetes/pki/etcd/healthcheck-client.crt /etc/kubernetes/pki/apiserver-etcd-client.crt
|
||
sudo cp /etc/kubernetes/pki/etcd/healthcheck-client.key /etc/kubernetes/pki/apiserver-etcd-client.key
|
||
fi
|
||
if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" != "$INSTALL_MASTER" ] && [ -d "pki" ]; then
|
||
sudo cp -pr pki/* /etc/kubernetes/pki
|
||
fi
|
||
fi
|
||
[ "resources/$K8S_CONFIG" != "$WORK_PATH/kubeadm-config.yaml" ] && sudo cp "resources/$K8S_CONFIG" "$WORK_PATH/kubeadm-config.yaml"
|
||
local has_apiserver
|
||
has_apiserver=$(sudo ps -aux | awk '{print $11}' | grep "kube-apiserver")
|
||
if [ -z "$has_apiserver" ]; then
|
||
sudo rm -f "$INSTALL_LOG"
|
||
sudo systemctl start kubelet 2>/dev/null
|
||
sudo kubeadm init --config "$WORK_PATH/kubeadm-config.yaml" --ignore-preflight-errors=all | sudo tee "$INSTALL_LOG"
|
||
fi
|
||
# Fallback: kubeadm init output can be truncated when piped through tee
|
||
# (e.g. the [addons] phase runs async and its output may not reach the log).
|
||
# Trust admin.conf existence + reachable livez as the real success signal.
|
||
local init_ok=false
|
||
if sudo grep -q "initialized successfully" "$INSTALL_LOG" 2>/dev/null; then
|
||
init_ok=true
|
||
elif [ -f "/etc/kubernetes/admin.conf" ]; then
|
||
local livez_resp
|
||
# Use 127.0.0.1: on Hetzner the node cannot reach its own private IP locally.
|
||
livez_resp=$(curl -sk --max-time 10 "https://127.0.0.1:6443/livez" 2>/dev/null)
|
||
[ "$livez_resp" == "ok" ] && init_ok=true
|
||
fi
|
||
if $init_ok; then
|
||
# Hetzner private network: the node cannot connect to its own private IP (/32).
|
||
# kubeadm sets probe host to the advertise address (private IP), making every
|
||
# liveness/startup probe return "connection refused" from the kubelet itself.
|
||
# Patch all probe hosts to 127.0.0.1 (apiserver binds to 0.0.0.0).
|
||
sudo sed -i 's/host: '"${MAIN_IP}"'/host: 127.0.0.1/g' \
|
||
/etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null || true
|
||
|
||
# Increase liveness probe tolerance for collocated etcd setups (e.g. CAX11).
|
||
# Default failureThreshold:8 × period:10s = 80s is too tight when etcd and
|
||
# apiserver share vCPUs and gRPC watches cause transient connection storms.
|
||
# failureThreshold:20 gives 200s — enough for etcd backpressure to resolve.
|
||
# initialDelaySeconds:30 prevents false kills during cold-start gRPC setup.
|
||
# Tune apiserver probes for collocated etcd nodes (shared vCPU contention):
|
||
# - startupProbe: 300s window (30×10s) — covers cold-start gRPC watch burst
|
||
# - livenessProbe: 120s window (12×10s) after startup — normal running tolerance
|
||
# - readinessProbe: fast (3×5s) — remove from LB quickly if unhealthy
|
||
# - bind-address=0.0.0.0: force IPv4 loopback client (prevents [::1] 403 on bootstrap)
|
||
# Edit the static manifest kubeadm just generated — pure awk/sed, no extra
|
||
# tooling on the node (mirrors the --etcd-servers / kubeconfig sed edits
|
||
# below). kubeadm emits deterministic 2-space YAML: probe keys at 4-space
|
||
# indent, their scalar fields at 6-space; awk tracks the current probe block
|
||
# and rewrites only its own fields, leaving httpGet/timeouts untouched.
|
||
APISERVER_MANIFEST=/etc/kubernetes/manifests/kube-apiserver.yaml
|
||
# bind-address=0.0.0.0 → IPv4 loopback client (prevents [::1] 403 on bootstrap)
|
||
if ! grep -q -- '--bind-address' "$APISERVER_MANIFEST"; then
|
||
sudo awk '{print} /^ - kube-apiserver$/{print " - --bind-address=0.0.0.0"}' \
|
||
"$APISERVER_MANIFEST" | sudo tee "$APISERVER_MANIFEST.new" >/dev/null \
|
||
&& sudo mv "$APISERVER_MANIFEST.new" "$APISERVER_MANIFEST"
|
||
fi
|
||
# Probe tuning for etcd-collocated CP nodes (shared vCPU contention):
|
||
# startup 300s window (30×10s, drop initialDelay), liveness 120s (12×10s,
|
||
# initialDelay 15s), readiness fast fail (3×5s).
|
||
if sudo awk '
|
||
/^ (startupProbe|livenessProbe|readinessProbe):$/ {
|
||
if ($0 ~ /startupProbe/) probe="startup"
|
||
else if ($0 ~ /livenessProbe/) probe="liveness"
|
||
else probe="readiness"
|
||
print; next
|
||
}
|
||
/^ [^ ]/ { probe="" }
|
||
probe!="" && /^ failureThreshold:/ {
|
||
if (probe=="startup") { print " failureThreshold: 30"; next }
|
||
if (probe=="liveness") { print " failureThreshold: 12"; next }
|
||
if (probe=="readiness") { print " failureThreshold: 3"; next }
|
||
}
|
||
probe!="" && /^ periodSeconds:/ {
|
||
if (probe=="startup") { print " periodSeconds: 10"; next }
|
||
if (probe=="liveness") { print " periodSeconds: 10"; next }
|
||
if (probe=="readiness") { print " periodSeconds: 5"; next }
|
||
}
|
||
probe!="" && /^ initialDelaySeconds:/ {
|
||
if (probe=="startup") { next }
|
||
if (probe=="liveness") { print " initialDelaySeconds: 15"; next }
|
||
}
|
||
{ print }
|
||
' "$APISERVER_MANIFEST" | sudo tee "$APISERVER_MANIFEST.new" >/dev/null; then
|
||
sudo mv "$APISERVER_MANIFEST.new" "$APISERVER_MANIFEST"
|
||
echo "probes patched"
|
||
else
|
||
sudo rm -f "$APISERVER_MANIFEST.new"
|
||
echo "Warning: probe patch failed"
|
||
fi
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | apiserver probes patched: startup=300s liveness=120s readiness=15s" | sudo tee -a "$INSTALL_LOG"
|
||
|
||
# Hetzner private network: kubeadm writes the advertiseAddress (private IP)
|
||
# as the apiserver endpoint in ALL kubeconfigs. On Hetzner the node cannot
|
||
# reach its own private IP via TCP — patch everything to 127.0.0.1.
|
||
sudo sed -i \
|
||
's|server: https://'"${MAIN_IP}"':6443|server: https://127.0.0.1:6443|g' \
|
||
/etc/kubernetes/kubelet.conf \
|
||
/etc/kubernetes/scheduler.conf \
|
||
/etc/kubernetes/controller-manager.conf 2>/dev/null || true
|
||
|
||
# Reorder etcd endpoints: move the first endpoint (typically the leader/seed)
|
||
# to the end. gRPC pickfirst balancer hammers the first endpoint with all
|
||
# new channels during startup. Distributing avoids thundering herd on one node.
|
||
# Read current endpoints from the manifest that kubeadm just generated.
|
||
local _etcd_line _etcd_eps
|
||
_etcd_line=$(grep -- '--etcd-servers=' /etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null | sed 's/.*--etcd-servers=//')
|
||
if [ -n "$_etcd_line" ]; then
|
||
local _first_ep _rest_ep
|
||
_first_ep=$(echo "$_etcd_line" | cut -d',' -f1)
|
||
_rest_ep=$(echo "$_etcd_line" | cut -d',' -f2-)
|
||
if [ -n "$_rest_ep" ]; then
|
||
local _reordered="${_rest_ep},${_first_ep}"
|
||
sudo sed -i "s|--etcd-servers=.*|--etcd-servers=${_reordered}|" \
|
||
/etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null || true
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | etcd endpoints reordered: ${_reordered}" | sudo tee -a "$INSTALL_LOG"
|
||
fi
|
||
fi
|
||
|
||
# Add the cluster CA to the system trust store so kubelet's TLS probe
|
||
# (K8s 1.35+ verifies server cert) can validate the apiserver certificate.
|
||
if [ -f "/etc/kubernetes/pki/ca.crt" ]; then
|
||
sudo cp /etc/kubernetes/pki/ca.crt /usr/local/share/ca-certificates/kubernetes-ca.crt
|
||
sudo update-ca-certificates 2>/dev/null || true
|
||
sudo systemctl restart kubelet 2>/dev/null || true
|
||
fi
|
||
|
||
if echo "${K8S_SKIP_PHASES:-}" | grep -q "kube-proxy"; then
|
||
sudo kubeadm init phase addon coredns --config "$WORK_PATH/kubeadm-config.yaml" 2>/dev/null || true
|
||
else
|
||
sudo kubeadm init phase addon all --config "$WORK_PATH/kubeadm-config.yaml" 2>/dev/null || true
|
||
fi
|
||
sudo kubeadm token create --print-join-command 2>/dev/null | sudo tee "$WORK_PATH/k8s_join.sh" >/dev/null ||
|
||
sudo grep -A1 "^kubeadm join" "$INSTALL_LOG" | sudo tee "$WORK_PATH/k8s_join.sh"
|
||
sudo chmod +x "$WORK_PATH/k8s_join.sh"
|
||
[ "$WORK_PATH" != "/tmp" ] && cp "$WORK_PATH/k8s_join.sh" /tmp
|
||
_kubernetes_cni
|
||
_kubernetes_addons
|
||
# Copy log to WORK_PATH for collection; keep /tmp/k8s.log in place for debugging.
|
||
[ "$WORK_PATH" != "/tmp" ] && sudo cp "$INSTALL_LOG" "$WORK_PATH/" 2>/dev/null || true
|
||
[ -r "runtimes.yaml" ] && _kubectl_apply runtimes.yaml
|
||
fi
|
||
# Always overwrite kubeconfig — runs unconditionally so reinstalls and
|
||
# partial inits always leave a valid ~/.kube/config.
|
||
_kubernetes_kube "$(whoami)"
|
||
|
||
# Wait for RBAC bootstrap to be fully committed to etcd.
|
||
# The clusterrole-aggregation-controller (controller-manager) must run and
|
||
# aggregate roles before scheduler/worker joins can succeed without "forbidden".
|
||
# Checking auth can-i for system:kube-scheduler proves bootstrap is complete.
|
||
if [ "$ETCD_MODE" == "external" ] || [ -f "/etc/kubernetes/admin.conf" ]; then
|
||
export KUBECONFIG=/etc/kubernetes/admin.conf
|
||
local _rbac_ready=false
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Waiting for RBAC bootstrap to complete..." | sudo tee -a "$INSTALL_LOG"
|
||
for _i in $(seq 1 30); do
|
||
if kubectl auth can-i list pods --as=system:kube-scheduler 2>/dev/null | grep -q yes; then
|
||
_rbac_ready=true
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | RBAC bootstrap complete (attempt $_i)" | sudo tee -a "$INSTALL_LOG"
|
||
break
|
||
fi
|
||
sleep 5
|
||
done
|
||
if ! $_rbac_ready; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Warning: RBAC bootstrap not confirmed after 150s — scheduler may need extra time on first start" | sudo tee -a "$INSTALL_LOG"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
_join_kubernetes() {
|
||
local join_path cmd_join
|
||
[ -r "k8s_join.sh" ] && join_path="k8s_join.sh"
|
||
[ -r "/tmp/k8s_join.sh" ] && join_path="/tmp/k8s_join.sh"
|
||
[ -z "$join_path" ] && echo "No k8s_join.sh found" && return 0
|
||
if [ "$1" == "controlplane" ]; then
|
||
cmd_join=$(sed 's/join /join --control-plane /g' <"$join_path")
|
||
else
|
||
cmd_join=$(sed 's/\\//g' <"$join_path")
|
||
fi
|
||
[ -z "$cmd_join" ] && echo "Error: empty cmd_join" && exit 1
|
||
# shellcheck disable=SC2086
|
||
sudo $cmd_join --ignore-preflight-errors=all | sudo tee "$INSTALL_LOG" >/dev/null ||
|
||
{
|
||
echo "Error: $HOSTNAME join failed"
|
||
exit 1
|
||
}
|
||
}
|
||
|
||
_install_controlplane() {
|
||
if [ "$ETCD_MODE" == "external" ] && [ -d "etcd_certs" ]; then
|
||
[ ! -d "/etc/kubernetes/pki/etcd" ] && sudo mkdir -p /etc/kubernetes/pki/etcd
|
||
sudo cp -pr etcd_certs/* /etc/kubernetes/pki/etcd
|
||
if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" != "$INSTALL_MASTER" ] && [ -d "pki" ]; then
|
||
sudo cp -pr pki/* /etc/kubernetes/pki
|
||
fi
|
||
fi
|
||
_join_kubernetes controlplane
|
||
_kubernetes_kube "$USER" "$USER_HOME"
|
||
_kubernetes_cni
|
||
_kubernetes_addons
|
||
}
|
||
|
||
_install_kubernetes() {
|
||
[ ! -d "/etc/${K8S_CRI}" ] && echo "No /etc/${K8S_CRI} path found" && exit 1
|
||
sudo systemctl start "${K8S_CRI}"
|
||
_check_resolution
|
||
|
||
if [ -f "/etc/kubernetes/admin.conf" ] || [ -f "/etc/kubernetes/kubelet.conf" ]; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | ❌ /etc/kubernetes already exists on $HOSTNAME — run reset first" | sudo tee -a "$INSTALL_LOG"
|
||
exit 1
|
||
fi
|
||
|
||
local has_kubelet
|
||
has_kubelet=$(sudo ps -aux | awk '{print $11}' | grep "kubelet")
|
||
if [ -n "$has_kubelet" ]; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | ❌ kubelet already running on $HOSTNAME — run reset first" | sudo tee -a "$INSTALL_LOG"
|
||
exit 1
|
||
fi
|
||
|
||
if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" == "$K8S_MASTER" ]; then
|
||
_install_master_0
|
||
_kubernetes_taint
|
||
_kubernetes_label_roles
|
||
else
|
||
case "$K8S_MODE" in
|
||
controlplane)
|
||
_install_controlplane
|
||
_kubernetes_taint
|
||
_kubernetes_label_roles
|
||
;;
|
||
worker)
|
||
_join_kubernetes worker
|
||
_kubernetes_label_roles
|
||
;;
|
||
*) echo "mode $K8S_MODE not defined" && exit 1 ;;
|
||
esac
|
||
fi
|
||
}
|
||
|
||
_config_kubernetes() {
|
||
[ ! -d "/etc/${K8S_CRI}" ] && echo "No /etc/${K8S_CRI} path found" && exit 1
|
||
# Ensure CRI is enabled and running — survives reboots (e.g. after server upgrade)
|
||
sudo systemctl enable "${K8S_CRI}" 2>/dev/null || true
|
||
sudo systemctl start "${K8S_CRI}"
|
||
|
||
# Persist br_netfilter so /proc/sys/net/bridge/ exists after reboot.
|
||
if ! grep -q "br_netfilter" /etc/modules-load.d/kubernetes.conf 2>/dev/null; then
|
||
printf 'br_netfilter\noverlay\n' | sudo tee /etc/modules-load.d/kubernetes.conf >/dev/null
|
||
fi
|
||
sudo modprobe br_netfilter
|
||
sudo modprobe overlay
|
||
|
||
# Write all required sysctls to a persistent drop-in; sysctl.conf is not
|
||
# guaranteed to contain the commented-out key on all distro variants.
|
||
sudo tee /etc/sysctl.d/99-kubernetes.conf >/dev/null <<'EOF'
|
||
net.bridge.bridge-nf-call-iptables = 1
|
||
net.bridge.bridge-nf-call-ip6tables = 1
|
||
net.ipv4.ip_forward = 1
|
||
net.ipv4.ip_nonlocal_bind = 1
|
||
EOF
|
||
sudo sysctl --system >/dev/null
|
||
}
|
||
|
||
_make_join() {
|
||
kubeadm token create --print-join-command >"$WORK_PATH/k8s_join.sh" ||
|
||
{
|
||
echo "Error generating join token"
|
||
exit 1
|
||
}
|
||
}
|
||
|
||
case "$CMD_TSK" in
|
||
reinstall)
|
||
_common_service_stop
|
||
_reset_kubernetes
|
||
;;
|
||
install)
|
||
_config_kubernetes
|
||
if [ -f "/etc/kubernetes/admin.conf" ] || [ -f "/etc/kubernetes/kubelet.conf" ]; then
|
||
if [ -f "/etc/kubernetes/manifests/kube-apiserver.yaml" ]; then
|
||
# Control-plane node: check whether the cluster is actually live.
|
||
# A live cluster (apiserver responding) must never be wiped by a plain install.
|
||
if sudo kubectl --kubeconfig /etc/kubernetes/admin.conf cluster-info 2>/dev/null | grep -q "running\|Kubernetes control plane"; then
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | ✅ Cluster already running on $HOSTNAME — skipping install (use reinstall to force)" | sudo tee -a "$INSTALL_LOG"
|
||
exit 0
|
||
fi
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Stale CP /etc/kubernetes detected — cleaning before install" | sudo tee -a "$INSTALL_LOG"
|
||
sudo systemctl stop kubelet 2>/dev/null || true
|
||
sudo kubeadm reset -f 2>/dev/null || true
|
||
sudo rm -rf /etc/kubernetes /etc/cni/net.d
|
||
sudo rm -f /var/lib/kubelet/pki/kubelet-client-*.pem
|
||
else
|
||
# Worker node: no apiserver manifest, safe to reset and re-join.
|
||
echo "$(date +%Y_%m_%d_%H%M%S) | Worker /etc/kubernetes found — resetting before re-join" | sudo tee -a "$INSTALL_LOG"
|
||
sudo systemctl stop kubelet 2>/dev/null || true
|
||
sudo kubeadm reset -f 2>/dev/null || true
|
||
sudo rm -rf /etc/kubernetes /etc/cni/net.d
|
||
sudo rm -f /var/lib/kubelet/pki/kubelet-client-*.pem
|
||
fi
|
||
fi
|
||
_update_binary || {
|
||
echo "error kubernetes install"
|
||
exit 1
|
||
}
|
||
_install_kubernetes
|
||
sudo systemctl enable kubelet 2>/dev/null
|
||
sudo systemctl start kubelet 2>/dev/null || true
|
||
;;
|
||
update) _common_update ;;
|
||
config) _config_kubernetes ;;
|
||
restart) _common_service_restart ;;
|
||
makejoin) _make_join ;;
|
||
remove)
|
||
sudo systemctl stop kubelet
|
||
sudo systemctl disable kubelet
|
||
;;
|
||
fullremove)
|
||
sudo systemctl stop kubelet
|
||
sudo systemctl disable kubelet
|
||
sudo kubeadm reset -y
|
||
sudo rm -rf /etc/kubernetes /etc/cni
|
||
;;
|
||
delete)
|
||
sudo systemctl stop kubelet >/dev/null 2>&1 || true
|
||
sudo systemctl disable kubelet >/dev/null 2>&1 || true
|
||
_reset_kubernetes
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-mark unhold kubelet kubectl kubeadm 2>/dev/null
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-get remove -y kubelet kubectl kubeadm 2>/dev/null
|
||
sudo rm -rf /etc/kubernetes /etc/cni /var/lib/kubelet /root/.kube
|
||
sudo rm -f /etc/modules-load.d/kubernetes.conf /etc/sysctl.d/99-kubernetes.conf
|
||
sudo rm -f /usr/local/share/ca-certificates/kubernetes-ca.crt
|
||
sudo update-ca-certificates 2>/dev/null || true
|
||
sudo rm -f /lib/systemd/system/kubelet.service
|
||
sudo systemctl daemon-reload >/dev/null 2>&1
|
||
sudo rm -rf /opt/kubernetes
|
||
;;
|
||
esac
|