ontoref/justfiles/_secrets_lib.sh

246 lines
12 KiB
Bash
Raw Normal View History

feat: #[onto_mcp_tool] catalog, OCI credential vault layer, validate ADR-018 mode hierarchy ontoref-derive: #[onto_mcp_tool] attribute macro registers MCP tool unit-structs in the catalog at link time via inventory::submit!; annotated item is emitted unchanged, ToolBase/AsyncTool impls stay on the struct. All 34 tools migrated from manual wiring (net +5: ontoref_list_projects, ontoref_search, ontoref_describe, ontoref_list_ontology_extensions, ontoref_get_ontology_extension). validate modes (ADR-018): reads level_hierarchy from workflow.ncl and checks every .ncl mode for level declared, strategy declared, delegate chain coherent, compose extends valid. mode resolve <id> shows which hierarchy level handles a mode and why. --self-test generates synthetic fixtures in a temp dir for CI smoke-testing. validate run-cargo: two-step Cargo.toml resolution — workspace layout first (crates/<check.crate>/Cargo.toml), single-crate fallback by package name or repo basename. Lets the same ADR constraint shape apply to workspace and single-crate repos. ontology/schemas/manifest.ncl: registry_topology_type contract — multi-registry coordination, push targets, participant scopes, per-namespace capability. reflection/requirements/base.ncl: oras ≥1.2.0, cosign ≥2.0.0, sops ≥3.9.0, age ≥1.1.0, restic declared as Hard/Soft requirements with version_min, check_cmd, and install_hint (ADR-017 toolchain surface). ADR-019: per-file recipient routing for tenant isolation without multi-vault. Schema additions: sops.recipient_groups + sops.recipient_rules in ontoref-project.ncl. secrets-bootstrap generates .sops.yaml from project.ncl in declarative mode. Three new secrets-audit checks: recipient-routing-coherent, recipient-routing-coverage, no-multi-vault. Adoption templates: single-team/, multi-tenant/, agent-first/. Integration templates: domain-producer/, mode-producer/, mode-consumer/. UI: project_picker surfaces registry badge (⟳ participant) and vault badge (⛁ vault_id · N, green=declarative / amber=legacy) per project card. Expanded panel adds collapsible Registry section with namespace, endpoint, and push/pull capability. manage.html gains Runtime Services card — MCP and GraphQL toggleable without restart via HTMX POST /ui/manage/services/{service}/toggle. describe.nu: capabilities JSON includes registry_topology and vault_state per project. sync.nu: drift check extended to detect //! absence on newly registered crates. qa.ncl: six entries — credential-vault-best-practice (layered data-flow diagram), credential-vault-templates (paths A/B/C), credential-vault-troubleshooting (15 named errors), integration-what-and-why (ADR-042 OCI federation), integration-how-to-implement, integration-troubleshooting. on+re: core.ncl + manifest.ncl updated to reflect OCI, MCP, and mode-hierarchy nodes. Deleted stale presentation assets (2026-02 slides + voice notes).
2026-05-12 04:46:15 +01:00
#!/usr/bin/env bash
# justfiles/_secrets_lib.sh — shared loader for secrets-* recipes (ADR-017).
# Sourced from each recipe via: source "{{justfile_directory()}}/justfiles/_secrets_lib.sh"
#
# After sourcing, the following variables are set and validated:
# PROJECT_ROOT, PROJECT_NCL, IMPORT_PATH
# PROJECT_JSON — exported project.ncl as JSON (jq-able)
# GLOBAL_JSON — exported ~/.config/ontoref/config.ncl as JSON
# SLUG — project.ncl::slug
# VAULT_ID — project.ncl::sops.vault_id, fallback to SLUG
# VAULT_TOOL — project.ncl::sops.vault_backend (restic|kopia)
# REGISTRY — project.ncl::sops.registry_endpoint
# MASTER_KEY — project override → ~/.config/ontoref/config.ncl::vault.master_key_path
# COSIGN_KEY_PATH — global vault.cosign.key_path (signing)
# COSIGN_PUB_PATH — global vault.cosign.pub_path (verification)
# VAULT_DIR — ~/.config/ontoref/vaults/<vault_id>
# ACCESS_SOPS — $VAULT_DIR/access.sops.yaml
set -euo pipefail
PROJECT_ROOT="${ONTOREF_PROJECT_ROOT:-$(pwd)}"
PROJECT_NCL="${PROJECT_ROOT}/.ontoref/project.ncl"
[[ -f "$PROJECT_NCL" ]] || { echo "No .ontoref/project.ncl found at $PROJECT_ROOT"; exit 1; }
IMPORT_PATH="${NICKEL_IMPORT_PATH:-${PROVISIONING:-}}"
PROJECT_JSON=$(nickel export --import-path "$IMPORT_PATH" "$PROJECT_NCL")
GLOBAL_JSON=$(nickel export --import-path "$IMPORT_PATH" "$HOME/.config/ontoref/config.ncl" 2>/dev/null || echo '{}')
SLUG=$(echo "$PROJECT_JSON" | jq -r '.slug // empty')
VAULT_ID=$(echo "$PROJECT_JSON" | jq -r '.sops.vault_id // empty')
VAULT_ID="${VAULT_ID:-$SLUG}"
VAULT_TOOL=$(echo "$PROJECT_JSON" | jq -r '.sops.vault_backend // "restic"')
REGISTRY=$(echo "$PROJECT_JSON" | jq -r '.sops.registry_endpoint // empty')
MASTER_KEY=$(echo "$PROJECT_JSON" | jq -r '.sops.master_key_path // empty')
if [[ -z "$MASTER_KEY" ]]; then
MASTER_KEY=$(echo "$GLOBAL_JSON" | jq -r '.vault.master_key_path // empty')
fi
COSIGN_KEY_PATH=$(echo "$GLOBAL_JSON" | jq -r '.vault.cosign.key_path // empty')
COSIGN_PUB_PATH=$(echo "$GLOBAL_JSON" | jq -r '.vault.cosign.pub_path // empty')
COSIGN_TLOG=$(echo "$GLOBAL_JSON" | jq -r '.vault.cosign.tlog // false')
COSIGN_SIGNING_CONFIG=$(echo "$GLOBAL_JSON" | jq -r '.vault.cosign.signing_config_path // empty')
# Sign/verify flags derived from tlog policy. When tlog=false we sign without
# Rekor (cosign 2+ requires --signing-config that excludes rekorTlogUrls/Config)
# and verify with --insecure-ignore-tlog so missing entries do not fail.
# Symmetric — sign and verify must agree.
if [[ "$COSIGN_TLOG" == "true" ]]; then
COSIGN_SIGN_FLAGS=""
COSIGN_VERIFY_FLAGS=""
else
if [[ -z "$COSIGN_SIGNING_CONFIG" ]]; then
echo "vault.cosign.tlog=false but signing_config_path not set — generate one and declare it" >&2
exit 1
fi
[[ -f "$COSIGN_SIGNING_CONFIG" ]] || {
echo "Cosign signing-config not found at $COSIGN_SIGNING_CONFIG" >&2
exit 1
}
COSIGN_SIGN_FLAGS="--signing-config $COSIGN_SIGNING_CONFIG"
COSIGN_VERIFY_FLAGS="--insecure-ignore-tlog"
fi
VAULT_DIR="$HOME/.config/ontoref/vaults/$VAULT_ID"
ACCESS_SOPS="$VAULT_DIR/access.sops.yaml"
[[ -n "$VAULT_ID" ]] || { echo "Cannot determine vault_id (slug empty and sops.vault_id absent)"; exit 1; }
[[ -n "$MASTER_KEY" ]] || { echo "master_key_path not declared (project.ncl::sops or ~/.config/ontoref/config.ncl::vault)"; exit 1; }
[[ -f "$MASTER_KEY" ]] || { echo "Master key (.kage) not found at $MASTER_KEY"; exit 1; }
# Auto-rotate audit log on every recipe invocation (call at end of this file
# after vault_rotate_audit_log is defined — bash does not hoist functions).
# Vault lock TTL (minutes). Past this, locks are considered abandoned and
# admin can force-close. Per ADR-017 G1 — cooperative lock atop ZOT ACL.
VAULT_LOCK_TTL_MIN=60
VAULT_LOCK_TAG="lock" # OCI ref: src-vault/<id>:lock
# Audit log rotation thresholds. access.jsonl is rotated when EITHER applies:
# size: current file > VAULT_AUDIT_ROTATE_BYTES (default 10MB)
# age: current file mtime older than VAULT_AUDIT_ROTATE_DAYS (default 90)
# Rotated to access.jsonl.<UTC-ISO>.gz inside the same logs/ dir.
VAULT_AUDIT_ROTATE_BYTES=${VAULT_AUDIT_ROTATE_BYTES:-10485760}
VAULT_AUDIT_ROTATE_DAYS=${VAULT_AUDIT_ROTATE_DAYS:-90}
# Rotate $VAULT_DIR/logs/access.jsonl when size or age exceeds thresholds.
# Idempotent: noop when file absent or under both thresholds. Safe to call at
# the start of every secrets-* operation.
vault_rotate_audit_log() {
local log="$VAULT_DIR/logs/access.jsonl"
[[ -f "$log" ]] || return 0
local size_bytes age_days now_epoch mtime_epoch should_rotate=0
size_bytes=$(stat -f %z "$log" 2>/dev/null || stat -c %s "$log" 2>/dev/null || echo 0)
[[ "$size_bytes" -gt "$VAULT_AUDIT_ROTATE_BYTES" ]] && should_rotate=1
if [[ "$should_rotate" -eq 0 ]]; then
now_epoch=$(date -u +%s)
mtime_epoch=$(stat -f %m "$log" 2>/dev/null || stat -c %Y "$log" 2>/dev/null || echo "$now_epoch")
age_days=$(( (now_epoch - mtime_epoch) / 86400 ))
[[ "$age_days" -gt "$VAULT_AUDIT_ROTATE_DAYS" ]] && should_rotate=1
fi
if [[ "$should_rotate" -eq 1 ]]; then
local stamp rotated
stamp=$(date -u +%Y-%m-%dT%H-%M-%SZ)
rotated="$VAULT_DIR/logs/access.jsonl.${stamp}.gz"
gzip -c "$log" > "$rotated" && : > "$log"
# Append a marker entry recording the rotation so audit chain stays continuous.
printf '{"ts":"%s","actor":"%s","op":"audit-log-rotated","detail":"previous=%s size=%s"}\n' \
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${ONTOREF_ACTOR:-system}" \
"access.jsonl.${stamp}.gz" "$size_bytes" >> "$log"
fi
}
# Fetch current lock state from ZOT. Returns JSON {locked_by,actor,since,expires}
# or empty string if no lock or fetch fails. ZOT credentials must be in $TMPDIR_CFG/config.json.
# Caller provides $TMPDIR_CFG before invoking.
vault_lock_fetch() {
local out
out=$(DOCKER_CONFIG="$TMPDIR_CFG" oras manifest fetch \
"${REGISTRY}/src-vault/${VAULT_ID}:${VAULT_LOCK_TAG}" 2>/dev/null) || return 0
# Lock metadata is in the manifest annotations — extract our keys.
echo "$out" | jq -c '{
locked_by: (.annotations."ontoref.vault.locked_by" // ""),
actor: (.annotations."ontoref.vault.actor" // ""),
since: (.annotations."ontoref.vault.since" // ""),
expires: (.annotations."ontoref.vault.expires" // ""),
force_closed_by: (.annotations."ontoref.vault.force_closed_by" // "")
}'
}
# Push a lock artifact with the given metadata. Requires $TMPDIR_CFG with ZOT creds.
# Args: locked_by actor since expires
vault_lock_push() {
local locked_by="$1" actor="$2" since="$3" expires="$4"
# oras rejects absolute paths — work in a tmp dir with relative filename.
local lock_dir lock_file rc
lock_dir=$(mktemp -d)
lock_file="lock.json"
# Lock metadata is in OCI annotations; the blob just needs ≥ 1 byte —
# ZOT rejects empty blobs (returns 400 on the PUT). Use a minimal JSON
# placeholder so the file is self-describing if pulled.
printf '{"kind":"vault-lock"}\n' > "$lock_dir/$lock_file"
set +e
( cd "$lock_dir" && DOCKER_CONFIG="$TMPDIR_CFG" oras push \
"${REGISTRY}/src-vault/${VAULT_ID}:${VAULT_LOCK_TAG}" \
--artifact-type "application/vnd.ontoref.vault.lock.v1" \
--annotation "ontoref.vault.locked_by=${locked_by}" \
--annotation "ontoref.vault.actor=${actor}" \
--annotation "ontoref.vault.since=${since}" \
--annotation "ontoref.vault.expires=${expires}" \
"${lock_file}:application/vnd.ontoref.vault.lock.metadata.v1" ) >/dev/null 2>&1
rc=$?
set -e
rm -rf "$lock_dir"
return $rc
}
# Delete the lock artifact from ZOT. Requires $TMPDIR_CFG.
vault_lock_release() {
DOCKER_CONFIG="$TMPDIR_CFG" oras manifest delete -f \
"${REGISTRY}/src-vault/${VAULT_ID}:${VAULT_LOCK_TAG}" >/dev/null 2>&1 || true
}
# Build a $TMPDIR_CFG from the decrypted access.sops.yaml. Sets the global
# TMPDIR_CFG variable; caller is responsible for `rm -rf "$TMPDIR_CFG"` later.
vault_zot_config_open() {
[[ -f "$ACCESS_SOPS" ]] || { echo "[lock] access.sops.yaml absent — bootstrap or sync first"; return 1; }
local creds zu zp
creds=$(SOPS_AGE_KEY_FILE="$MASTER_KEY" sops --decrypt "$ACCESS_SOPS")
zu=$(echo "$creds" | grep '^zot_username:' | awk '{print $2}')
zp=$(echo "$creds" | grep '^zot_password:' | awk '{print $2}')
TMPDIR_CFG=$(mktemp -d)
local b64
b64=$(printf '%s:%s' "$zu" "$zp" | base64)
printf '{"auths":{"%s":{"auth":"%s"}}}' "$REGISTRY" "$b64" > "$TMPDIR_CFG/config.json"
unset creds zu zp b64
}
# ── ADR-017 G2 — Impact analysis ───────────────────────────────────────────────
# vault_impact_report: list relative paths of *.sops.yaml files in the vault
# that have changed since the last restic snapshot. Empty list = no credential
# changes. Output one path per line, relative to $VAULT_DIR.
vault_impact_changed_creds() {
local repo="$VAULT_DIR/repo"
[[ -d "$repo" ]] || return 0
local vk
vk=$(SOPS_AGE_KEY_FILE="$MASTER_KEY" sops --decrypt --extract '["vault_key"]' "$ACCESS_SOPS" 2>/dev/null)
[[ -n "$vk" ]] || return 0
local snaps
snaps=$(RESTIC_PASSWORD="$vk" restic -r "$repo" snapshots --json 2>/dev/null) || return 0
local last_id
last_id=$(echo "$snaps" | jq -r 'last | .id // empty' 2>/dev/null)
[[ -n "$last_id" ]] || return 0
# Take an in-memory snapshot of current files (no restic backup — too invasive
# for a dry diff). Use a hash comparison instead: hash each *.sops.yaml in the
# last snapshot via 'restic dump' and compare to current hash.
local cur_dir snap_dir tmp_diff
cur_dir="$VAULT_DIR"
tmp_diff=$(mktemp)
while IFS= read -r rel; do
[[ -z "$rel" ]] && continue
local cur_hash="" snap_hash=""
if [[ -f "$cur_dir/$rel" ]]; then
cur_hash=$(shasum -a 256 "$cur_dir/$rel" | awk '{print $1}')
fi
snap_hash=$(RESTIC_PASSWORD="$vk" restic -r "$repo" dump "$last_id" "$rel" 2>/dev/null \
| shasum -a 256 | awk '{print $1}')
if [[ "$cur_hash" != "$snap_hash" ]]; then
echo "$rel" >> "$tmp_diff"
fi
done < <(find "$VAULT_DIR" -name "*.sops.yaml" -type f 2>/dev/null \
| while read -r f; do echo "${f#${VAULT_DIR}/}"; done)
cat "$tmp_diff"
rm -f "$tmp_diff"
}
# vault_impact_registries: given a list of changed sops paths (relative), find
# which RegistryEntry IDs in the project manifest reference them via
# credential_sops or credential_sops_rw. Output: <entry_id>\t<sops_path>\t<field>
vault_impact_registries() {
local manifest_ncl="$PROJECT_ROOT/.ontology/manifest.ncl"
[[ -f "$manifest_ncl" ]] || return 0
local manifest_json
manifest_json=$(nickel export --import-path "$IMPORT_PATH" "$manifest_ncl" 2>/dev/null) || return 0
local entries
entries=$(echo "$manifest_json" | jq -c '.registry_provides.registries.registries // []' 2>/dev/null)
[[ "$entries" != "null" && -n "$entries" ]] || return 0
while IFS= read -r changed_full; do
# Strip leading 'src-vault/' since credential_sops paths are relative to src-vault/.
local rel="${changed_full#src-vault/}"
echo "$entries" | jq -r --arg rel "$rel" '
.[] | select(.credential_sops == $rel or .credential_sops_rw == $rel)
| "\(.id)\t\($rel)\t\(if .credential_sops == $rel then "credential_sops" else "credential_sops_rw" end)"
'
done
}
# Final action of this lib: rotate audit log if needed. Idempotent.
vault_rotate_audit_log