#!/bin/bash
[ "$1" == "-h" ] && echo "install-kubernetes.sh install|update|reinstall|remove|fullremove|makejoin|restart|config" && exit 1

SERVICE_NAME="kubelet"
SCRIPTS_DEPLOY_PATH="/opt/kubernetes"

[ -r "env-kubernetes" ] && . ./env-kubernetes
[ -r "common.sh" ] && . ./common.sh

export LC_CTYPE=C.UTF-8 LANG=C.UTF-8

CMD_TSK=${CMD_TSK:-${1:-install}}
INSTALL_LOG=${INSTALL_LOG:-/tmp/k8s.log}
WORK_PATH=${WORK_PATH:-/tmp}
[ ! -d "$WORK_PATH" ] && sudo mkdir -p "$WORK_PATH"
[ ! -d "$(dirname "$INSTALL_LOG")" ] && mkdir -p "$(dirname "$INSTALL_LOG")"

[ -z "$CLUSTER_NAME" ] && echo "No CLUSTER_NAME value" && exit 1
[ -z "$VERSION" ] && echo "No VERSION value" && exit 1

if [ -z "$K8S_MODE" ]; then
    if [[ "$HOSTNAME" == *-cp-* ]]; then
        K8S_MODE="controlplane"
    else
        K8S_MODE="worker"
    fi
fi

_check_resolution() {
    local ip
    [ "$K8S_MODE" == "controlplane" ] && local clustername="$CLUSTER_NAME"
    # Do NOT remove the 127.0.1.1 entry — Debian default, keeps hostname self-routable
    # (matches behaviour of manually-provisioned nodes like ko where any local address
    #  reaches the apiserver).
    ip=$(grep "$MAIN_IP" /etc/hosts | grep -v "^#" | awk '{print $1}')
    [ -z "$ip" ] && echo "$MAIN_IP $HOSTNAME ${clustername:-}" | sudo tee -a /etc/hosts >/dev/null
    if [ "$HOSTNAME" != "$(cat /etc/hostname)" ]; then
        echo "$HOSTNAME" | sudo tee /etc/hostname >/dev/null
        sudo hostname "$HOSTNAME"
    fi
}

_off_swap() {
    local fs_swap
    fs_swap=$(grep -v "^#" /etc/fstab | grep swap)
    if [ -n "$fs_swap" ]; then
        sudo sed -i "s;$fs_swap;#$fs_swap;g" /etc/fstab
    fi
    sudo swapoff -a
}

_update_binary() {
    local curr_vers
    curr_vers=$(kubectl version 2>/dev/null | grep Client | awk '{print $3}' | sed 's/^v//g')
    [ "v$curr_vers" == "$K8S_VERSION" ] && return 0
    chmod 1777 /tmp
    sudo DEBIAN_FRONTEND=noninteractive apt-get update -q
    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https gnupg2 curl
    sudo rm -f /etc/apt/keyrings/kubernetes-apt-keyring.gpg
    curl -fsSL "https://pkgs.k8s.io/core:/stable:/v${MAJOR_VERSION}/deb/Release.key" |
        sudo gpg --batch --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
    echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${MAJOR_VERSION}/deb/ /" |
        sudo tee /etc/apt/sources.list.d/kubernetes.list >/dev/null
    sudo chmod 644 /etc/apt/sources.list.d/kubernetes.list
    _off_swap
    sudo DEBIAN_FRONTEND=noninteractive apt-get update -q
    sudo DEBIAN_FRONTEND=noninteractive apt-mark unhold kubelet kubectl kubeadm
    local pkg_ver="${K8S_VERSION#v}-1.1"
    if ! sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades \
        "kubectl=${pkg_ver}" "kubelet=${pkg_ver}" "kubeadm=${pkg_ver}"; then
        echo "error installing kubernetes packages version ${pkg_ver}"
        return 1
    fi
    sudo DEBIAN_FRONTEND=noninteractive apt-mark hold kubelet kubectl kubeadm
}

_kubernetes_kube() {
    local user=${1:-root} home_user=${2:-/root}
    local uid gid
    uid=$(sudo id -u "$user" 2>/dev/null)
    gid=$(sudo id -g "$user" 2>/dev/null)
    [ ! -f "/etc/kubernetes/admin.conf" ] && return 0
    sudo mkdir -p /root/.kube
    sudo cp -f /etc/kubernetes/admin.conf /root/.kube/config
    sudo chown root:root /root/.kube/config
    # Hetzner: node cannot reach its own private IP — patch local kubeconfig to loopback.
    # admin.conf keeps the real IP for external kubectl access.
    [ -n "$MAIN_IP" ] && sudo sed -i \
        "s|server: https://${MAIN_IP}:6443|server: https://127.0.0.1:6443|g" \
        /root/.kube/config
    if [ "$uid" != "0" ]; then
        mkdir -p "$home_user/.kube"
        sudo cp -f /etc/kubernetes/admin.conf "$home_user/.kube/config"
        sudo chown -R "$uid:$gid" "$home_user/.kube"
        [ -n "$MAIN_IP" ] && sudo sed -i \
            "s|server: https://${MAIN_IP}:6443|server: https://127.0.0.1:6443|g" \
            "$home_user/.kube/config"
    fi
    local has_aliases
    has_aliases=$(grep bash_aliases "$HOME/.bashrc" 2>/dev/null)
    [ -z "$has_aliases" ] && echo "[ -f ~/.bash_aliases ] && . ~/.bash_aliases" | sudo tee -a "$HOME/.bashrc" >/dev/null
}

_kubectl_apply() {
    export KUBECONFIG=/etc/kubernetes/admin.conf
    [ ! -r "$KUBECONFIG" ] && echo "$KUBECONFIG not found" && return 1
    [ ! -r "$1" ] && echo "File $1 not found" && return 1
    kubectl apply -f "$1" || echo "Error kubectl apply $1"
}

_kubernetes_taint() {
    # K8s 1.24+ uses control-plane taint; master is deprecated. Use --timeout to avoid
    # hanging if apiserver is briefly unresponsive after init.
    case "$TAINT_NODE" in
    no_schedule)
        kubectl taint nodes "$HOSTNAME" node-role.kubernetes.io/control-plane:NoSchedule \
            --timeout=30s 2>/dev/null || true
        ;;
    schedule)
        # Remove any lingering taint so the control-plane can schedule workloads.
        kubectl taint nodes "$HOSTNAME" node-role.kubernetes.io/control-plane:NoSchedule- \
            --timeout=30s 2>/dev/null || true
        ;;
    esac
    return 0
}

_kubernetes_label_roles() {
    [ -z "${NODE_ROLES:-}" ] && return 0
    local kubeconfig="/etc/kubernetes/admin.conf"
    for role in ${NODE_ROLES//,/ }; do
        if [ -f "$kubeconfig" ]; then
            kubectl --kubeconfig="$kubeconfig" label node "$HOSTNAME" \
                "node-role.kubernetes.io/${role}=" --overwrite 2>/dev/null || true
        else
            # Worker nodes lack admin.conf — delegate to the control-plane via SSH.
            ssh -o StrictHostKeyChecking=no -o BatchMode=yes "$K8S_MASTER" \
                kubectl label node "$HOSTNAME" \
                "node-role.kubernetes.io/${role}=" --overwrite 2>/dev/null || true
        fi
        echo "$(date +%Y_%m_%d_%H%M%S) | node role set: node-role.kubernetes.io/${role}" | sudo tee -a "$INSTALL_LOG"
    done
}

# CNI is now installed as a separate taskserv (cilium) in the formula DAG.
# kubernetes → cilium ordering is enforced by the formula node dependencies.
# Keeping the function as no-op so existing call sites don't break.
_kubernetes_cni() { return 0; }

_kubernetes_addons() {
    local yaml_file
    for item in ${K8S_ADDONS//,/ }; do
        if [ -r "addons/$item/install.sh" ]; then
            echo "Install addon $item" | sudo tee -a "$INSTALL_LOG"
            # shellcheck disable=SC1090
            . "addons/$item/install.sh"
            if [ "$item" == "istio" ] && [ -n "$K8S_EXTERNAL_IPS" ]; then
                yaml_file=/tmp/externalIPs.yaml
                {
                    echo "spec:"
                    echo "   externalIPs:"
                    for ip in ${K8S_EXTERNAL_IPS//,/ }; do echo "   - $ip"; done
                } >"$yaml_file"
                kubectl patch service -n istio-system istio-ingressgateway --type merge --patch-file "$yaml_file"
            fi
        fi
    done
}

_reset_kubernetes() {
    sudo rm -f "$INSTALL_LOG"
    echo "$(date +%Y_%m_%d_%H%M%S) | Kubernetes RESET on $HOSTNAME ..." | sudo tee "$INSTALL_LOG"

    sudo systemctl stop kubelet 2>/dev/null || true

    for _cid in $(crictl ps -q 2>/dev/null); do
        crictl stop "$_cid" 2>/dev/null || true
        crictl rm -f "$_cid" 2>/dev/null || true
    done
    for _pod in $(crictl pods -q 2>/dev/null); do
        crictl stopp "$_pod" 2>/dev/null || true
        crictl rmp "$_pod" 2>/dev/null || true
    done

    # Full kubelet state wipe: config.yaml and kubeadm-flags.env are written by
    # kubeadm init and carry the old CA reference — keeping them causes the new
    # kubelet to present stale certs before bootstrap-kubelet.conf can take over.
    sudo rm -rf /var/lib/kubelet/pki /var/lib/kubelet/config.yaml \
        /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/cpu_manager_state \
        /var/lib/kubelet/memory_manager_state /var/lib/kubelet/device-plugins

    # Clean stale CNI state and iptables rules left by previous installs.
    sudo rm -rf /etc/cni/net.d /run/flannel /run/cilium
    for _dev in cilium_vxlan cilium_host cilium_net lxc_health kube-ipvs0 flannel.1 cni0; do
        ip link delete "$_dev" 2>/dev/null || true
    done
    iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X 2>/dev/null || true
    ip6tables -F && ip6tables -t nat -F && ip6tables -t mangle -F && ip6tables -X 2>/dev/null || true

    if [ "$ETCD_MODE" == "external" ]; then
        sudo rm -rf /etc/kubernetes

        local etcd_cfg=/etc/etcd/config.yaml
        local etcd_ca etcd_cert etcd_key
        etcd_ca=$(sudo awk '/trusted-ca-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
        etcd_cert=$(sudo awk '/cert-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
        etcd_key=$(sudo awk '/key-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
        if [ -n "$etcd_ca" ] && [ -n "$etcd_cert" ] && [ -n "$etcd_key" ]; then
            # Wipe current cluster prefix only. CLUSTER_NAME is set from the taskserv
            # config (e.g. "sgoyol"), which matches the etcd-prefix used in kubeadm init.
            local _all_prefixes="/${CLUSTER_NAME}/"
            for _pfx in $_all_prefixes; do
                local _count
                # --keys-only and --count-only are mutually exclusive in etcd 3.6+.
                # --count-only requires --write-out=fields to print to stdout.
                _count=$(ETCDCTL_API=3 sudo etcdctl \
                    --endpoints="https://${MAIN_IP}:2379" \
                    --cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
                    get "$_pfx" --prefix --count-only --write-out=fields 2>/dev/null |
                    awk -F'\t' '/Count/{print $2}' || echo 0)
                if [ "${_count:-0}" -gt 0 ] 2>/dev/null; then
                    echo "$(date +%Y_%m_%d_%H%M%S) | Wiping etcd prefix $_pfx (${_count} keys) ..." | sudo tee -a "$INSTALL_LOG"
                    ETCDCTL_API=3 sudo etcdctl \
                        --endpoints="https://${MAIN_IP}:2379" \
                        --cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
                        del "$_pfx" --prefix 2>&1 | sudo tee -a "$INSTALL_LOG" ||
                        echo "Warning: etcd wipe failed for $_pfx" | sudo tee -a "$INSTALL_LOG"
                fi
            done
        else
            echo "Warning: cannot parse etcd TLS config from $etcd_cfg, skipping prefix wipe" | sudo tee -a "$INSTALL_LOG"
        fi
    else
        sudo kubeadm reset -f 2>/dev/null || true
        sudo rm -rf /etc/kubernetes
    fi

    echo "$(date +%Y_%m_%d_%H%M%S) | Kubernetes ready for reinstall on $HOSTNAME" | sudo tee -a "$INSTALL_LOG"
}

_etcd_wipe_prefix() {
    # Wipe the cluster's etcd key-space before kubeadm init so stale data from
    # previous installs never interferes. Runs on every install, not just reinstall.
    [ "$ETCD_MODE" != "external" ] && return 0
    local etcd_cfg=/etc/etcd/config.yaml
    local etcd_ca etcd_cert etcd_key
    etcd_ca=$(sudo awk '/trusted-ca-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
    etcd_cert=$(sudo awk '/cert-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
    etcd_key=$(sudo awk '/key-file:/{print $2; exit}' "$etcd_cfg" 2>/dev/null | tr -d '"')
    [ -z "$etcd_ca" ] || [ -z "$etcd_cert" ] || [ -z "$etcd_key" ] &&
        echo "Warning: cannot parse etcd TLS config — skipping prefix wipe" && return 0
    local _pfx="/${CLUSTER_NAME}/"
    local _count
    _count=$(ETCDCTL_API=3 sudo etcdctl \
        --endpoints="https://${MAIN_IP}:2379" \
        --cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
        get "$_pfx" --prefix --count-only --write-out=fields 2>/dev/null |
        awk -F'\t' '/Count/{print $2}' || echo 0)
    if [ "${_count:-0}" -gt 0 ] 2>/dev/null; then
        echo "$(date +%Y_%m_%d_%H%M%S) | Wiping etcd prefix $_pfx (${_count} keys) ..." | sudo tee -a "$INSTALL_LOG"
        ETCDCTL_API=3 sudo etcdctl \
            --endpoints="https://${MAIN_IP}:2379" \
            --cacert="$etcd_ca" --cert="$etcd_cert" --key="$etcd_key" \
            del "$_pfx" --prefix 2>&1 | sudo tee -a "$INSTALL_LOG" ||
            echo "Warning: etcd wipe failed for $_pfx" | sudo tee -a "$INSTALL_LOG"
    else
        echo "$(date +%Y_%m_%d_%H%M%S) | etcd prefix $_pfx is empty — nothing to wipe"
    fi
}

_install_master_0() {
    _check_resolution
    [ ! -r "resources/$K8S_CONFIG" ] && echo "resources/$K8S_CONFIG not found" && exit 1
    # Always wipe the etcd prefix before kubeadm init — idempotent, safe on empty prefix.
    _etcd_wipe_prefix
    if [ "$ETCD_MODE" == "external" ] && [ -d "etcd_certs" ]; then
        [ ! -d "/etc/kubernetes/pki/etcd" ] && sudo mkdir -p /etc/kubernetes/pki/etcd
        sudo cp -pr etcd_certs/* /etc/kubernetes/pki/etcd
        # kubeadm never generates apiserver-etcd-client in external etcd mode.
        # Pre-provision it using the healthcheck-client cert (clientAuth, signed by etcd CA).
        if [ -f "/etc/kubernetes/pki/etcd/healthcheck-client.crt" ]; then
            sudo cp /etc/kubernetes/pki/etcd/healthcheck-client.crt /etc/kubernetes/pki/apiserver-etcd-client.crt
            sudo cp /etc/kubernetes/pki/etcd/healthcheck-client.key /etc/kubernetes/pki/apiserver-etcd-client.key
        fi
        if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" != "$INSTALL_MASTER" ] && [ -d "pki" ]; then
            sudo cp -pr pki/* /etc/kubernetes/pki
        fi
    fi
    [ "resources/$K8S_CONFIG" != "$WORK_PATH/kubeadm-config.yaml" ] && sudo cp "resources/$K8S_CONFIG" "$WORK_PATH/kubeadm-config.yaml"
    local has_apiserver
    has_apiserver=$(sudo ps -aux | awk '{print $11}' | grep "kube-apiserver")
    if [ -z "$has_apiserver" ]; then
        sudo rm -f "$INSTALL_LOG"
        sudo systemctl start kubelet 2>/dev/null
        sudo kubeadm init --config "$WORK_PATH/kubeadm-config.yaml" --ignore-preflight-errors=all | sudo tee "$INSTALL_LOG"
    fi
    # Fallback: kubeadm init output can be truncated when piped through tee
    # (e.g. the [addons] phase runs async and its output may not reach the log).
    # Trust admin.conf existence + reachable livez as the real success signal.
    local init_ok=false
    if sudo grep -q "initialized successfully" "$INSTALL_LOG" 2>/dev/null; then
        init_ok=true
    elif [ -f "/etc/kubernetes/admin.conf" ]; then
        local livez_resp
        # Use 127.0.0.1: on Hetzner the node cannot reach its own private IP locally.
        livez_resp=$(curl -sk --max-time 10 "https://127.0.0.1:6443/livez" 2>/dev/null)
        [ "$livez_resp" == "ok" ] && init_ok=true
    fi
    if $init_ok; then
        # Hetzner private network: the node cannot connect to its own private IP (/32).
        # kubeadm sets probe host to the advertise address (private IP), making every
        # liveness/startup probe return "connection refused" from the kubelet itself.
        # Patch all probe hosts to 127.0.0.1 (apiserver binds to 0.0.0.0).
        sudo sed -i 's/host: '"${MAIN_IP}"'/host: 127.0.0.1/g' \
            /etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null || true

        # Increase liveness probe tolerance for collocated etcd setups (e.g. CAX11).
        # Default failureThreshold:8 × period:10s = 80s is too tight when etcd and
        # apiserver share vCPUs and gRPC watches cause transient connection storms.
        # failureThreshold:20 gives 200s — enough for etcd backpressure to resolve.
        # initialDelaySeconds:30 prevents false kills during cold-start gRPC setup.
        # Tune apiserver probes for collocated etcd nodes (shared vCPU contention):
        # - startupProbe: 300s window (30×10s) — covers cold-start gRPC watch burst
        # - livenessProbe: 120s window (12×10s) after startup — normal running tolerance
        # - readinessProbe: fast (3×5s) — remove from LB quickly if unhealthy
        # - bind-address=0.0.0.0: force IPv4 loopback client (prevents [::1] 403 on bootstrap)
        # Edit the static manifest kubeadm just generated — pure awk/sed, no extra
        # tooling on the node (mirrors the --etcd-servers / kubeconfig sed edits
        # below). kubeadm emits deterministic 2-space YAML: probe keys at 4-space
        # indent, their scalar fields at 6-space; awk tracks the current probe block
        # and rewrites only its own fields, leaving httpGet/timeouts untouched.
        APISERVER_MANIFEST=/etc/kubernetes/manifests/kube-apiserver.yaml
        # bind-address=0.0.0.0 → IPv4 loopback client (prevents [::1] 403 on bootstrap)
        if ! grep -q -- '--bind-address' "$APISERVER_MANIFEST"; then
            sudo awk '{print} /^    - kube-apiserver$/{print "    - --bind-address=0.0.0.0"}' \
                "$APISERVER_MANIFEST" | sudo tee "$APISERVER_MANIFEST.new" >/dev/null \
                && sudo mv "$APISERVER_MANIFEST.new" "$APISERVER_MANIFEST"
        fi
        # Probe tuning for etcd-collocated CP nodes (shared vCPU contention):
        #   startup 300s window (30×10s, drop initialDelay), liveness 120s (12×10s,
        #   initialDelay 15s), readiness fast fail (3×5s).
        if sudo awk '
            /^    (startupProbe|livenessProbe|readinessProbe):$/ {
                if ($0 ~ /startupProbe/) probe="startup"
                else if ($0 ~ /livenessProbe/) probe="liveness"
                else probe="readiness"
                print; next
            }
            /^    [^ ]/ { probe="" }
            probe!="" && /^      failureThreshold:/ {
                if (probe=="startup")   { print "      failureThreshold: 30"; next }
                if (probe=="liveness")  { print "      failureThreshold: 12"; next }
                if (probe=="readiness") { print "      failureThreshold: 3";  next }
            }
            probe!="" && /^      periodSeconds:/ {
                if (probe=="startup")   { print "      periodSeconds: 10"; next }
                if (probe=="liveness")  { print "      periodSeconds: 10"; next }
                if (probe=="readiness") { print "      periodSeconds: 5";  next }
            }
            probe!="" && /^      initialDelaySeconds:/ {
                if (probe=="startup")  { next }
                if (probe=="liveness") { print "      initialDelaySeconds: 15"; next }
            }
            { print }
        ' "$APISERVER_MANIFEST" | sudo tee "$APISERVER_MANIFEST.new" >/dev/null; then
            sudo mv "$APISERVER_MANIFEST.new" "$APISERVER_MANIFEST"
            echo "probes patched"
        else
            sudo rm -f "$APISERVER_MANIFEST.new"
            echo "Warning: probe patch failed"
        fi
        echo "$(date +%Y_%m_%d_%H%M%S) | apiserver probes patched: startup=300s liveness=120s readiness=15s" | sudo tee -a "$INSTALL_LOG"

        # Hetzner private network: kubeadm writes the advertiseAddress (private IP)
        # as the apiserver endpoint in ALL kubeconfigs. On Hetzner the node cannot
        # reach its own private IP via TCP — patch everything to 127.0.0.1.
        sudo sed -i \
            's|server: https://'"${MAIN_IP}"':6443|server: https://127.0.0.1:6443|g' \
            /etc/kubernetes/kubelet.conf \
            /etc/kubernetes/scheduler.conf \
            /etc/kubernetes/controller-manager.conf 2>/dev/null || true

        # Reorder etcd endpoints: move the first endpoint (typically the leader/seed)
        # to the end. gRPC pickfirst balancer hammers the first endpoint with all
        # new channels during startup. Distributing avoids thundering herd on one node.
        # Read current endpoints from the manifest that kubeadm just generated.
        local _etcd_line _etcd_eps
        _etcd_line=$(grep -- '--etcd-servers=' /etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null | sed 's/.*--etcd-servers=//')
        if [ -n "$_etcd_line" ]; then
            local _first_ep _rest_ep
            _first_ep=$(echo "$_etcd_line" | cut -d',' -f1)
            _rest_ep=$(echo "$_etcd_line" | cut -d',' -f2-)
            if [ -n "$_rest_ep" ]; then
                local _reordered="${_rest_ep},${_first_ep}"
                sudo sed -i "s|--etcd-servers=.*|--etcd-servers=${_reordered}|" \
                    /etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null || true
                echo "$(date +%Y_%m_%d_%H%M%S) | etcd endpoints reordered: ${_reordered}" | sudo tee -a "$INSTALL_LOG"
            fi
        fi

        # Add the cluster CA to the system trust store so kubelet's TLS probe
        # (K8s 1.35+ verifies server cert) can validate the apiserver certificate.
        if [ -f "/etc/kubernetes/pki/ca.crt" ]; then
            sudo cp /etc/kubernetes/pki/ca.crt /usr/local/share/ca-certificates/kubernetes-ca.crt
            sudo update-ca-certificates 2>/dev/null || true
            sudo systemctl restart kubelet 2>/dev/null || true
        fi

        if echo "${K8S_SKIP_PHASES:-}" | grep -q "kube-proxy"; then
            sudo kubeadm init phase addon coredns --config "$WORK_PATH/kubeadm-config.yaml" 2>/dev/null || true
        else
            sudo kubeadm init phase addon all --config "$WORK_PATH/kubeadm-config.yaml" 2>/dev/null || true
        fi
        sudo kubeadm token create --print-join-command 2>/dev/null | sudo tee "$WORK_PATH/k8s_join.sh" >/dev/null ||
            sudo grep -A1 "^kubeadm join" "$INSTALL_LOG" | sudo tee "$WORK_PATH/k8s_join.sh"
        sudo chmod +x "$WORK_PATH/k8s_join.sh"
        [ "$WORK_PATH" != "/tmp" ] && cp "$WORK_PATH/k8s_join.sh" /tmp
        _kubernetes_cni
        _kubernetes_addons
        # Copy log to WORK_PATH for collection; keep /tmp/k8s.log in place for debugging.
        [ "$WORK_PATH" != "/tmp" ] && sudo cp "$INSTALL_LOG" "$WORK_PATH/" 2>/dev/null || true
        [ -r "runtimes.yaml" ] && _kubectl_apply runtimes.yaml
    fi
    # Always overwrite kubeconfig — runs unconditionally so reinstalls and
    # partial inits always leave a valid ~/.kube/config.
    _kubernetes_kube "$(whoami)"

    # Wait for RBAC bootstrap to be fully committed to etcd.
    # The clusterrole-aggregation-controller (controller-manager) must run and
    # aggregate roles before scheduler/worker joins can succeed without "forbidden".
    # Checking auth can-i for system:kube-scheduler proves bootstrap is complete.
    if [ "$ETCD_MODE" == "external" ] || [ -f "/etc/kubernetes/admin.conf" ]; then
        export KUBECONFIG=/etc/kubernetes/admin.conf
        local _rbac_ready=false
        echo "$(date +%Y_%m_%d_%H%M%S) | Waiting for RBAC bootstrap to complete..." | sudo tee -a "$INSTALL_LOG"
        for _i in $(seq 1 30); do
            if kubectl auth can-i list pods --as=system:kube-scheduler 2>/dev/null | grep -q yes; then
                _rbac_ready=true
                echo "$(date +%Y_%m_%d_%H%M%S) | RBAC bootstrap complete (attempt $_i)" | sudo tee -a "$INSTALL_LOG"
                break
            fi
            sleep 5
        done
        if ! $_rbac_ready; then
            echo "$(date +%Y_%m_%d_%H%M%S) | Warning: RBAC bootstrap not confirmed after 150s — scheduler may need extra time on first start" | sudo tee -a "$INSTALL_LOG"
        fi
    fi
}

_join_kubernetes() {
    local join_path cmd_join
    [ -r "k8s_join.sh" ] && join_path="k8s_join.sh"
    [ -r "/tmp/k8s_join.sh" ] && join_path="/tmp/k8s_join.sh"
    [ -z "$join_path" ] && echo "No k8s_join.sh found" && return 0
    if [ "$1" == "controlplane" ]; then
        cmd_join=$(sed 's/join /join --control-plane /g' <"$join_path")
    else
        cmd_join=$(sed 's/\\//g' <"$join_path")
    fi
    [ -z "$cmd_join" ] && echo "Error: empty cmd_join" && exit 1
    # shellcheck disable=SC2086
    sudo $cmd_join --ignore-preflight-errors=all | sudo tee "$INSTALL_LOG" >/dev/null ||
        {
            echo "Error: $HOSTNAME join failed"
            exit 1
        }
}

_install_controlplane() {
    if [ "$ETCD_MODE" == "external" ] && [ -d "etcd_certs" ]; then
        [ ! -d "/etc/kubernetes/pki/etcd" ] && sudo mkdir -p /etc/kubernetes/pki/etcd
        sudo cp -pr etcd_certs/* /etc/kubernetes/pki/etcd
        if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" != "$INSTALL_MASTER" ] && [ -d "pki" ]; then
            sudo cp -pr pki/* /etc/kubernetes/pki
        fi
    fi
    _join_kubernetes controlplane
    _kubernetes_kube "$USER" "$USER_HOME"
    _kubernetes_cni
    _kubernetes_addons
}

_install_kubernetes() {
    [ ! -d "/etc/${K8S_CRI}" ] && echo "No /etc/${K8S_CRI} path found" && exit 1
    sudo systemctl start "${K8S_CRI}"
    _check_resolution

    if [ -f "/etc/kubernetes/admin.conf" ] || [ -f "/etc/kubernetes/kubelet.conf" ]; then
        echo "$(date +%Y_%m_%d_%H%M%S) | ❌ /etc/kubernetes already exists on $HOSTNAME — run reset first" | sudo tee -a "$INSTALL_LOG"
        exit 1
    fi

    local has_kubelet
    has_kubelet=$(sudo ps -aux | awk '{print $11}' | grep "kubelet")
    if [ -n "$has_kubelet" ]; then
        echo "$(date +%Y_%m_%d_%H%M%S) | ❌ kubelet already running on $HOSTNAME — run reset first" | sudo tee -a "$INSTALL_LOG"
        exit 1
    fi

    if [ -n "$HOSTNAME" ] && [ "$HOSTNAME" == "$K8S_MASTER" ]; then
        _install_master_0
        _kubernetes_taint
        _kubernetes_label_roles
    else
        case "$K8S_MODE" in
        controlplane)
            _install_controlplane
            _kubernetes_taint
            _kubernetes_label_roles
            ;;
        worker)
            _join_kubernetes worker
            _kubernetes_label_roles
            ;;
        *) echo "mode $K8S_MODE not defined" && exit 1 ;;
        esac
    fi
}

_config_kubernetes() {
    [ ! -d "/etc/${K8S_CRI}" ] && echo "No /etc/${K8S_CRI} path found" && exit 1
    # Ensure CRI is enabled and running — survives reboots (e.g. after server upgrade)
    sudo systemctl enable "${K8S_CRI}" 2>/dev/null || true
    sudo systemctl start "${K8S_CRI}"

    # Persist br_netfilter so /proc/sys/net/bridge/ exists after reboot.
    if ! grep -q "br_netfilter" /etc/modules-load.d/kubernetes.conf 2>/dev/null; then
        printf 'br_netfilter\noverlay\n' | sudo tee /etc/modules-load.d/kubernetes.conf >/dev/null
    fi
    sudo modprobe br_netfilter
    sudo modprobe overlay

    # Write all required sysctls to a persistent drop-in; sysctl.conf is not
    # guaranteed to contain the commented-out key on all distro variants.
    sudo tee /etc/sysctl.d/99-kubernetes.conf >/dev/null <<'EOF'
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
net.ipv4.ip_nonlocal_bind           = 1
EOF
    sudo sysctl --system >/dev/null
}

_make_join() {
    kubeadm token create --print-join-command >"$WORK_PATH/k8s_join.sh" ||
        {
            echo "Error generating join token"
            exit 1
        }
}

case "$CMD_TSK" in
reinstall)
    _common_service_stop
    _reset_kubernetes
    ;;
install)
    _config_kubernetes
    if [ -f "/etc/kubernetes/admin.conf" ] || [ -f "/etc/kubernetes/kubelet.conf" ]; then
        if [ -f "/etc/kubernetes/manifests/kube-apiserver.yaml" ]; then
            # Control-plane node: check whether the cluster is actually live.
            # A live cluster (apiserver responding) must never be wiped by a plain install.
            if sudo kubectl --kubeconfig /etc/kubernetes/admin.conf cluster-info 2>/dev/null | grep -q "running\|Kubernetes control plane"; then
                echo "$(date +%Y_%m_%d_%H%M%S) | ✅ Cluster already running on $HOSTNAME — skipping install (use reinstall to force)" | sudo tee -a "$INSTALL_LOG"
                exit 0
            fi
            echo "$(date +%Y_%m_%d_%H%M%S) | Stale CP /etc/kubernetes detected — cleaning before install" | sudo tee -a "$INSTALL_LOG"
            sudo systemctl stop kubelet 2>/dev/null || true
            sudo kubeadm reset -f 2>/dev/null || true
            sudo rm -rf /etc/kubernetes /etc/cni/net.d
            sudo rm -f /var/lib/kubelet/pki/kubelet-client-*.pem
        else
            # Worker node: no apiserver manifest, safe to reset and re-join.
            echo "$(date +%Y_%m_%d_%H%M%S) | Worker /etc/kubernetes found — resetting before re-join" | sudo tee -a "$INSTALL_LOG"
            sudo systemctl stop kubelet 2>/dev/null || true
            sudo kubeadm reset -f 2>/dev/null || true
            sudo rm -rf /etc/kubernetes /etc/cni/net.d
            sudo rm -f /var/lib/kubelet/pki/kubelet-client-*.pem
        fi
    fi
    _update_binary || {
        echo "error kubernetes install"
        exit 1
    }
    _install_kubernetes
    sudo systemctl enable kubelet 2>/dev/null
    sudo systemctl start kubelet 2>/dev/null || true
    ;;
update) _common_update ;;
config) _config_kubernetes ;;
restart) _common_service_restart ;;
makejoin) _make_join ;;
remove)
    sudo systemctl stop kubelet
    sudo systemctl disable kubelet
    ;;
fullremove)
    sudo systemctl stop kubelet
    sudo systemctl disable kubelet
    sudo kubeadm reset -y
    sudo rm -rf /etc/kubernetes /etc/cni
    ;;
delete)
    sudo systemctl stop kubelet >/dev/null 2>&1 || true
    sudo systemctl disable kubelet >/dev/null 2>&1 || true
    _reset_kubernetes
    sudo DEBIAN_FRONTEND=noninteractive apt-mark unhold kubelet kubectl kubeadm 2>/dev/null
    sudo DEBIAN_FRONTEND=noninteractive apt-get remove -y kubelet kubectl kubeadm 2>/dev/null
    sudo rm -rf /etc/kubernetes /etc/cni /var/lib/kubelet /root/.kube
    sudo rm -f /etc/modules-load.d/kubernetes.conf /etc/sysctl.d/99-kubernetes.conf
    sudo rm -f /usr/local/share/ca-certificates/kubernetes-ca.crt
    sudo update-ca-certificates 2>/dev/null || true
    sudo rm -f /lib/systemd/system/kubelet.service
    sudo systemctl daemon-reload >/dev/null 2>&1
    sudo rm -rf /opt/kubernetes
    ;;
esac