257 lines
10 KiB
Bash
257 lines
10 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
# Source the rendered env file from the bundle — the runner invokes this under
|
||
|
|
# `sudo bash` which strips the GARAGE_* environment, so without this every var
|
||
|
|
# below falls back to its default (empty node_selector, empty buckets → no keys).
|
||
|
|
# Mirrors install-loki.sh.
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
[ -f "${SCRIPT_DIR}/env-garage" ] && source "${SCRIPT_DIR}/env-garage"
|
||
|
|
|
||
|
|
GARAGE_VERSION="${GARAGE_VERSION:-v2.3.0}"
|
||
|
|
GARAGE_NAMESPACE="${GARAGE_NAMESPACE:-garage}"
|
||
|
|
GARAGE_REPLICAS="${GARAGE_REPLICAS:-1}"
|
||
|
|
GARAGE_REPLICATION_FACTOR="${GARAGE_REPLICATION_FACTOR:-1}"
|
||
|
|
GARAGE_STORAGE_CLASS="${GARAGE_STORAGE_CLASS:-hcloud-volumes}"
|
||
|
|
GARAGE_DATA_VOLUME_SIZE="${GARAGE_DATA_VOLUME_SIZE:-10Gi}"
|
||
|
|
GARAGE_S3_API_PORT="${GARAGE_S3_API_PORT:-3900}"
|
||
|
|
GARAGE_ADMIN_API_PORT="${GARAGE_ADMIN_API_PORT:-3903}"
|
||
|
|
GARAGE_RPC_PORT="${GARAGE_RPC_PORT:-3901}"
|
||
|
|
GARAGE_S3_REGION="${GARAGE_S3_REGION:-garage}"
|
||
|
|
GARAGE_SECRET_NAME="${GARAGE_SECRET_NAME:-garage-rpc-admin}"
|
||
|
|
# NOTE: do NOT write ${VAR:-{}} — bash parses the first '}' as the end of the
|
||
|
|
# expansion, appending a stray '}' to a non-empty value ("{...}}", invalid YAML).
|
||
|
|
GARAGE_NODE_SELECTOR_JSON="${GARAGE_NODE_SELECTOR_JSON:-}"
|
||
|
|
[ -n "${GARAGE_NODE_SELECTOR_JSON}" ] || GARAGE_NODE_SELECTOR_JSON="{}"
|
||
|
|
GARAGE_BUCKETS="${GARAGE_BUCKETS:-}"
|
||
|
|
|
||
|
|
IMAGE="dxflrs/garage:${GARAGE_VERSION}"
|
||
|
|
|
||
|
|
echo "=== garage: checking prerequisites ==="
|
||
|
|
|
||
|
|
if command -v k0s &>/dev/null && k0s kubectl version --client=true &>/dev/null; then
|
||
|
|
kubectl() { k0s kubectl "$@"; }
|
||
|
|
elif ! command -v kubectl &>/dev/null; then
|
||
|
|
echo "ERROR: neither kubectl nor k0s found" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
kubectl create namespace "${GARAGE_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
|
||
|
|
|
||
|
|
if ! kubectl -n "${GARAGE_NAMESPACE}" get secret "${GARAGE_SECRET_NAME}" &>/dev/null; then
|
||
|
|
echo "ERROR: Secret '${GARAGE_SECRET_NAME}' not found in ${GARAGE_NAMESPACE}." >&2
|
||
|
|
echo " Provisioned out-of-band by the 'prepare' step from SOPS — rerun prepare or check the .sops.yaml." >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
RPC_SECRET="$(kubectl -n "${GARAGE_NAMESPACE}" get secret "${GARAGE_SECRET_NAME}" -o jsonpath='{.data.rpc_secret}' | base64 -d)"
|
||
|
|
ADMIN_TOKEN="$(kubectl -n "${GARAGE_NAMESPACE}" get secret "${GARAGE_SECRET_NAME}" -o jsonpath='{.data.admin_token}' | base64 -d)"
|
||
|
|
|
||
|
|
echo "=== garage: rendering config secret ==="
|
||
|
|
|
||
|
|
GARAGE_TOML="$(cat <<TOMLEOF
|
||
|
|
metadata_dir = "/var/lib/garage/meta"
|
||
|
|
data_dir = "/var/lib/garage/data"
|
||
|
|
db_engine = "lmdb"
|
||
|
|
|
||
|
|
replication_factor = ${GARAGE_REPLICATION_FACTOR}
|
||
|
|
|
||
|
|
rpc_bind_addr = "[::]:${GARAGE_RPC_PORT}"
|
||
|
|
rpc_public_addr = "127.0.0.1:${GARAGE_RPC_PORT}"
|
||
|
|
rpc_secret = "${RPC_SECRET}"
|
||
|
|
|
||
|
|
[s3_api]
|
||
|
|
s3_region = "${GARAGE_S3_REGION}"
|
||
|
|
api_bind_addr = "[::]:${GARAGE_S3_API_PORT}"
|
||
|
|
root_domain = ".s3.garage.local"
|
||
|
|
|
||
|
|
[admin]
|
||
|
|
api_bind_addr = "[::]:${GARAGE_ADMIN_API_PORT}"
|
||
|
|
admin_token = "${ADMIN_TOKEN}"
|
||
|
|
metrics_token = "${ADMIN_TOKEN}"
|
||
|
|
TOMLEOF
|
||
|
|
)"
|
||
|
|
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" create secret generic garage-config \
|
||
|
|
--from-literal="garage.toml=${GARAGE_TOML}" \
|
||
|
|
--dry-run=client -o yaml | kubectl apply -f -
|
||
|
|
|
||
|
|
echo "=== garage: deploying StatefulSet (${GARAGE_VERSION}, ${GARAGE_REPLICAS} replica(s)) ==="
|
||
|
|
|
||
|
|
# node_selector arrives as compact JSON ({"k":"v"}); go-yaml (kubectl) rejects a
|
||
|
|
# flow map with no space after the colon ("did not find expected key"). Insert it.
|
||
|
|
# The empty case "{}" has no colon and passes through unchanged.
|
||
|
|
GARAGE_NODE_SELECTOR_YAML="$(printf '%s' "${GARAGE_NODE_SELECTOR_JSON}" | sed 's/":"/": "/g')"
|
||
|
|
|
||
|
|
# Render to a file and apply -f file, NOT -f - via heredoc: feeding the heredoc
|
||
|
|
# straight to `kubectl apply -f -` was rejected ("yaml: line 16 did not find
|
||
|
|
# expected key") even though the identical bytes apply cleanly from a file.
|
||
|
|
GARAGE_MANIFEST="$(mktemp /tmp/garage-manifest.XXXXXX.yaml)"
|
||
|
|
cat > "${GARAGE_MANIFEST}" <<EOF
|
||
|
|
apiVersion: v1
|
||
|
|
kind: Service
|
||
|
|
metadata:
|
||
|
|
name: garage
|
||
|
|
namespace: ${GARAGE_NAMESPACE}
|
||
|
|
spec:
|
||
|
|
clusterIP: None
|
||
|
|
selector:
|
||
|
|
app: garage
|
||
|
|
ports:
|
||
|
|
- name: s3-api
|
||
|
|
port: ${GARAGE_S3_API_PORT}
|
||
|
|
- name: admin-api
|
||
|
|
port: ${GARAGE_ADMIN_API_PORT}
|
||
|
|
- name: rpc
|
||
|
|
port: ${GARAGE_RPC_PORT}
|
||
|
|
---
|
||
|
|
# Stable ClusterIP alongside the headless Service above. The headless one exists
|
||
|
|
# for the StatefulSet's per-pod DNS identity (required); it has no virtual IP, so
|
||
|
|
# out-of-cluster clients (e.g. WireGuard fleet peers routed to 10.96.0.0/12) have
|
||
|
|
# nothing stable to target — only the pod IP, which changes on reschedule. This
|
||
|
|
# Service gives them a fixed address for the S3 API without touching the
|
||
|
|
# StatefulSet or its DNS.
|
||
|
|
apiVersion: v1
|
||
|
|
kind: Service
|
||
|
|
metadata:
|
||
|
|
name: garage-clusterip
|
||
|
|
namespace: ${GARAGE_NAMESPACE}
|
||
|
|
spec:
|
||
|
|
selector:
|
||
|
|
app: garage
|
||
|
|
ports:
|
||
|
|
- name: s3-api
|
||
|
|
port: ${GARAGE_S3_API_PORT}
|
||
|
|
---
|
||
|
|
apiVersion: apps/v1
|
||
|
|
kind: StatefulSet
|
||
|
|
metadata:
|
||
|
|
name: garage
|
||
|
|
namespace: ${GARAGE_NAMESPACE}
|
||
|
|
spec:
|
||
|
|
serviceName: garage
|
||
|
|
replicas: ${GARAGE_REPLICAS}
|
||
|
|
selector:
|
||
|
|
matchLabels:
|
||
|
|
app: garage
|
||
|
|
template:
|
||
|
|
metadata:
|
||
|
|
labels:
|
||
|
|
app: garage
|
||
|
|
spec:
|
||
|
|
nodeSelector: ${GARAGE_NODE_SELECTOR_YAML}
|
||
|
|
containers:
|
||
|
|
- name: garage
|
||
|
|
image: ${IMAGE}
|
||
|
|
# dxflrs/garage has no ENTRYPOINT (FROM scratch); binary is at /garage.
|
||
|
|
command: ["/garage"]
|
||
|
|
args: ["server"]
|
||
|
|
ports:
|
||
|
|
- containerPort: ${GARAGE_S3_API_PORT}
|
||
|
|
name: s3-api
|
||
|
|
- containerPort: ${GARAGE_ADMIN_API_PORT}
|
||
|
|
name: admin-api
|
||
|
|
- containerPort: ${GARAGE_RPC_PORT}
|
||
|
|
name: rpc
|
||
|
|
volumeMounts:
|
||
|
|
- name: data
|
||
|
|
mountPath: /var/lib/garage
|
||
|
|
- name: config
|
||
|
|
mountPath: /etc/garage.toml
|
||
|
|
subPath: garage.toml
|
||
|
|
readinessProbe:
|
||
|
|
httpGet: { path: /health, port: ${GARAGE_ADMIN_API_PORT} }
|
||
|
|
initialDelaySeconds: 5
|
||
|
|
periodSeconds: 10
|
||
|
|
livenessProbe:
|
||
|
|
# TCP liveness (process alive), NOT /health: garage's admin /health is
|
||
|
|
# readiness semantics (returns 503 until a layout is assigned), so using
|
||
|
|
# it for liveness kills the pod during the very layout bootstrap below.
|
||
|
|
tcpSocket: { port: ${GARAGE_S3_API_PORT} }
|
||
|
|
initialDelaySeconds: 15
|
||
|
|
periodSeconds: 20
|
||
|
|
failureThreshold: 6
|
||
|
|
volumes:
|
||
|
|
- name: config
|
||
|
|
secret:
|
||
|
|
secretName: garage-config
|
||
|
|
volumeClaimTemplates:
|
||
|
|
- metadata:
|
||
|
|
name: data
|
||
|
|
spec:
|
||
|
|
accessModes: ["ReadWriteOnce"]
|
||
|
|
storageClassName: ${GARAGE_STORAGE_CLASS}
|
||
|
|
resources:
|
||
|
|
requests:
|
||
|
|
storage: ${GARAGE_DATA_VOLUME_SIZE}
|
||
|
|
EOF
|
||
|
|
|
||
|
|
kubectl apply -f "${GARAGE_MANIFEST}"
|
||
|
|
rm -f "${GARAGE_MANIFEST}"
|
||
|
|
|
||
|
|
# Wait for the pod to be Running (container started) — NOT Ready. The readiness
|
||
|
|
# probe (/health) stays 503 until a layout is assigned, so waiting for Ready here
|
||
|
|
# would deadlock: the layout is assigned in the step right below.
|
||
|
|
echo "=== garage: waiting for pod to start ==="
|
||
|
|
for _ in $(seq 1 80); do
|
||
|
|
phase="$(kubectl -n "${GARAGE_NAMESPACE}" get pod garage-0 -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||
|
|
[ "${phase}" = "Running" ] && break
|
||
|
|
sleep 3
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "=== garage: bootstrapping cluster layout (idempotent) ==="
|
||
|
|
|
||
|
|
# `garage node id -q` prints "<hex-id>@<rpc-addr>"; `layout assign` wants the bare
|
||
|
|
# hex id (the @addr suffix makes it reject the node). Strip it.
|
||
|
|
NODE_ID="$(kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage node id -q 2>/dev/null | tr -d '\r' | cut -d@ -f1)"
|
||
|
|
|
||
|
|
ALREADY_APPLIED="$(kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage layout show 2>/dev/null | grep -c "^Current cluster layout version: [1-9]" || true)"
|
||
|
|
|
||
|
|
if [ "${ALREADY_APPLIED}" = "0" ]; then
|
||
|
|
# `layout assign -c` parses capacity with bytesize (garage >=0.9): "10G" is valid,
|
||
|
|
# the K8s-style "10Gi" is not. Strip the trailing 'i' from the PVC-format size.
|
||
|
|
GARAGE_LAYOUT_CAPACITY="${GARAGE_DATA_VOLUME_SIZE%i}"
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage layout assign -z fsn1 -c "${GARAGE_LAYOUT_CAPACITY}" "${NODE_ID}"
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage layout apply --version 1
|
||
|
|
echo "=== garage: layout applied ==="
|
||
|
|
else
|
||
|
|
echo "=== garage: layout already applied — skipping ==="
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Layout assigned → node has a role → /health flips to 200 and the pod goes Ready.
|
||
|
|
# Confirm before bucket ops (garage returns "Layout not ready" until it propagates).
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" rollout status statefulset/garage --timeout=120s
|
||
|
|
|
||
|
|
echo "=== garage: reconciling buckets (idempotent) ==="
|
||
|
|
|
||
|
|
# GARAGE_BUCKETS is space-separated "name:key_name" pairs (rendered by
|
||
|
|
# env-garage.j2). Pure bash parsing — runs on cluster nodes with no extra tooling.
|
||
|
|
for pair in ${GARAGE_BUCKETS}; do
|
||
|
|
bucket_name="${pair%%:*}"
|
||
|
|
key_name="${pair##*:}"
|
||
|
|
[ -z "${bucket_name}" ] && continue
|
||
|
|
|
||
|
|
# `garage bucket list` prints a table ("ID date <alias>"), so the bucket name
|
||
|
|
# is a word in a column — match with -w, NOT -x (whole-line, never matches →
|
||
|
|
# CreateBucket 409 BucketAlreadyExists on every reconcile).
|
||
|
|
if ! kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage bucket list 2>/dev/null | grep -qw "${bucket_name}"; then
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage bucket create "${bucket_name}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if ! kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage key list 2>/dev/null | grep -qw "${key_name}"; then
|
||
|
|
echo "=== garage: creating key '${key_name}' for bucket '${bucket_name}' — CAPTURE THESE CREDENTIALS NOW, store in SOPS ==="
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage key create "${key_name}"
|
||
|
|
kubectl -n "${GARAGE_NAMESPACE}" exec garage-0 -- /garage bucket allow --read --write --key "${key_name}" "${bucket_name}"
|
||
|
|
else
|
||
|
|
echo "=== garage: key '${key_name}' already exists — not reprinting credentials ==="
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# Report the real S3 endpoint consumers must use. The cluster DNS domain is NOT
|
||
|
|
# always "cluster.local" — kubeadm can set --cluster-domain (libre-wuji uses
|
||
|
|
# "libre-wuji.local"). Detect it from the coredns Corefile so the printed
|
||
|
|
# endpoint is correct per cluster; fall back to cluster.local if undetectable.
|
||
|
|
CLUSTER_DOMAIN="$(kubectl -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' 2>/dev/null | grep -oE 'kubernetes[[:space:]]+[^[:space:]]+' | awk '{print $2}' | head -1)"
|
||
|
|
CLUSTER_DOMAIN="${CLUSTER_DOMAIN:-cluster.local}"
|
||
|
|
echo "=== garage: ready (namespace=${GARAGE_NAMESPACE}, replicas=${GARAGE_REPLICAS}, s3_region=${GARAGE_S3_REGION}, s3_endpoint=http://garage.${GARAGE_NAMESPACE}.svc.${CLUSTER_DOMAIN}:${GARAGE_S3_API_PORT}) ==="
|