434 lines
22 KiB
Text
434 lines
22 KiB
Text
|
use utils.nu *
|
||
|
|
# REMOVED: use lib_provisioning * - causes circular import
|
||
|
|
use ../../../../catalog/providers/prov_lib/middleware.nu [mw_get_ip]
|
||
|
|
use run.nu *
|
||
|
|
use check_mode.nu *
|
||
|
|
use platform/config/accessor.nu *
|
||
|
|
use primitives/io/logging.nu [is-debug-enabled, is-debug-check-enabled]
|
||
|
|
use orchestration/servers/utils.nu [servers_selector, wait_for_server]
|
||
|
|
use primitives/io/hints.nu *
|
||
|
|
use domain/workspace/state.nu *
|
||
|
|
use tools/nickel/process.nu [ncl-eval]
|
||
|
|
|
||
|
|
# Resolve taskserv directory: checks direct (flat) then category subdirectories (hierarchical).
|
||
|
|
# Also tries underscore variant of hyphenated names (vol-prepare → vol_prepare).
|
||
|
|
def find-taskserv-path [taskservs_path: string, name: string]: nothing -> string {
|
||
|
|
let alt = ($name | str replace --all "-" "_")
|
||
|
|
let names = if $alt != $name { [$name, $alt] } else { [$name] }
|
||
|
|
for n in $names {
|
||
|
|
let direct = ($taskservs_path | path join $n)
|
||
|
|
if ($direct | path exists) { return $direct }
|
||
|
|
}
|
||
|
|
if not ($taskservs_path | path exists) { return "" }
|
||
|
|
for n in $names {
|
||
|
|
let found = (do -i { ls $taskservs_path } | where type == "dir" | each {|cat|
|
||
|
|
let candidate = ($cat.name | path join $n)
|
||
|
|
if ($candidate | path exists) { $candidate } else { null }
|
||
|
|
} | compact)
|
||
|
|
if ($found | is-not-empty) { return ($found | first) }
|
||
|
|
}
|
||
|
|
""
|
||
|
|
}
|
||
|
|
|
||
|
|
#use ../catalog/taskservs/run.nu run_taskserv
|
||
|
|
def install_from_server [
|
||
|
|
defs: record
|
||
|
|
server_taskserv_path: string
|
||
|
|
wk_server: string
|
||
|
|
]: nothing -> bool {
|
||
|
|
_print (
|
||
|
|
$"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " +
|
||
|
|
$"($defs.server.hostname) (_ansi default_dimmed)install(_ansi reset) " +
|
||
|
|
$"(_ansi purple_bold)from ($defs.taskserv_install_mode)(_ansi reset)"
|
||
|
|
)
|
||
|
|
let run_taskservs_path = (get-run-taskservs-path)
|
||
|
|
(run_taskserv $defs
|
||
|
|
($run_taskservs_path | path join $defs.taskserv.name | path join $server_taskserv_path)
|
||
|
|
($wk_server | path join $defs.taskserv.name))
|
||
|
|
}
|
||
|
|
def install_from_library [
|
||
|
|
defs: record
|
||
|
|
server_taskserv_path: string
|
||
|
|
wk_server: string
|
||
|
|
]: nothing -> bool {
|
||
|
|
_print (
|
||
|
|
$"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " +
|
||
|
|
$"($defs.server.hostname) (_ansi default_dimmed)install(_ansi reset) " +
|
||
|
|
$"(_ansi purple_bold)from library(_ansi reset)"
|
||
|
|
)
|
||
|
|
let taskservs_path = (get-taskservs-path)
|
||
|
|
let taskserv_dir = (find-taskserv-path $taskservs_path $defs.taskserv.name)
|
||
|
|
let profile_path = ($taskserv_dir | path join $defs.taskserv_profile)
|
||
|
|
# Components use a flat taskserv/ dir instead of per-profile subdirs (wrks/ format).
|
||
|
|
# Fall back to taskserv/ when the profile subdir doesn't exist.
|
||
|
|
# cluster/ is the canonical directory for manifest-plan cluster components.
|
||
|
|
let resolved_path = if ($profile_path | path exists) {
|
||
|
|
$profile_path
|
||
|
|
} else if ($taskserv_dir | path join "taskserv" | path exists) {
|
||
|
|
($taskserv_dir | path join "taskserv")
|
||
|
|
} else if ($taskserv_dir | path join "cluster" | path exists) {
|
||
|
|
($taskserv_dir | path join "cluster")
|
||
|
|
} else {
|
||
|
|
$profile_path
|
||
|
|
}
|
||
|
|
(run_taskserv $defs $resolved_path ($wk_server | path join $defs.taskserv.name))
|
||
|
|
}
|
||
|
|
|
||
|
|
# Build a map of taskserv_name → [depends_on taskserv_names] from a formula DAG.
|
||
|
|
# Reads the formula whose id matches "<hostname>-formula".
|
||
|
|
# Returns {} if the formula is not found or the DAG file does not exist.
|
||
|
|
def load-dag-deps [settings: record, hostname: string]: nothing -> record {
|
||
|
|
let dag_path = ($settings.infra_path | path join "dag.ncl")
|
||
|
|
if not ($dag_path | path exists) { return {} }
|
||
|
|
let prov_path = ($env.PROVISIONING? | default "/usr/local/provisioning")
|
||
|
|
let dag = (try {
|
||
|
|
ncl-eval $dag_path [$prov_path]
|
||
|
|
} catch {
|
||
|
|
return {}
|
||
|
|
})
|
||
|
|
let formula_id = $"($hostname)-formula"
|
||
|
|
let formula = ($dag.composition?.formulas? | default []
|
||
|
|
| where {|f| $f.formula_id? == $formula_id} | get 0?)
|
||
|
|
if ($formula | is-empty) { return {} }
|
||
|
|
|
||
|
|
# Build map: taskserv_name → [dep_taskserv_names]
|
||
|
|
# Formula nodes have: { id, taskserv: {name}, depends_on: [{node_id}] }
|
||
|
|
# We need to resolve node_id → taskserv.name via the nodes list.
|
||
|
|
let nodes = ($formula.nodes? | default [])
|
||
|
|
let id_to_name = ($nodes | each {|n|
|
||
|
|
{ id: $n.id, name: ($n.taskserv?.name? | default $n.id) }
|
||
|
|
})
|
||
|
|
$nodes | each {|n|
|
||
|
|
let ts_name = ($n.taskserv?.name? | default $n.id)
|
||
|
|
let dep_names = ($n.depends_on? | default [] | each {|d|
|
||
|
|
let resolved = ($id_to_name | where id == $d.node_id | first?)
|
||
|
|
if ($resolved | is-not-empty) { $resolved.name } else { $d.node_id }
|
||
|
|
})
|
||
|
|
{ $ts_name: $dep_names }
|
||
|
|
} | reduce -f {} {|it, acc| $acc | merge $it }
|
||
|
|
}
|
||
|
|
|
||
|
|
# Synthesize taskserv records from DAG formulas targeting a specific server.
|
||
|
|
# Searches all formula blocks (not just the hostname-formula) for nodes whose
|
||
|
|
# taskserv.name matches — covering L3 services that live in separate formulas.
|
||
|
|
def dag-synth-taskservs [
|
||
|
|
settings: record
|
||
|
|
hostname: string
|
||
|
|
match_taskserv: string
|
||
|
|
]: nothing -> list<record> {
|
||
|
|
let dag_path = ($settings.infra_path | path join "dag.ncl")
|
||
|
|
if not ($dag_path | path exists) { return [] }
|
||
|
|
let prov_path = ($env.PROVISIONING? | default "/usr/local/provisioning")
|
||
|
|
let dag = (try { ncl-eval $dag_path [$prov_path] } catch { return [] })
|
||
|
|
($dag.formulas? | default [])
|
||
|
|
| where {|f| ($f.server? | default "") == $hostname }
|
||
|
|
| each {|f|
|
||
|
|
($f.nodes? | default [])
|
||
|
|
| where {|n| ($n.taskserv?.name? | default "") == $match_taskserv }
|
||
|
|
| each {|n| {
|
||
|
|
name: $n.taskserv.name,
|
||
|
|
profile: ($n.taskserv?.profile? | default $n.id),
|
||
|
|
install_mode: "library",
|
||
|
|
depends_on: [],
|
||
|
|
on_error: ($n.on_error? | default "Continue" | into string),
|
||
|
|
max_retries: 0,
|
||
|
|
params: {},
|
||
|
|
target_save_path: "",
|
||
|
|
}}
|
||
|
|
}
|
||
|
|
| flatten
|
||
|
|
}
|
||
|
|
|
||
|
|
export def on_taskservs [
|
||
|
|
settings: record
|
||
|
|
match_taskserv: string
|
||
|
|
match_taskserv_profile: string
|
||
|
|
match_server: string
|
||
|
|
iptype: string
|
||
|
|
check: bool
|
||
|
|
upload: bool = false
|
||
|
|
reset: bool = false
|
||
|
|
cmd: string = ""
|
||
|
|
force_delete: bool = false
|
||
|
|
] {
|
||
|
|
_print $"Running (_ansi yellow_bold)taskservs(_ansi reset) ..."
|
||
|
|
let provisioning_sops = ($env.PROVISIONING_SOPS? | default "")
|
||
|
|
if $provisioning_sops == "" {
|
||
|
|
# A SOPS load env
|
||
|
|
$env.CURRENT_INFRA_PATH = $settings.infra_path
|
||
|
|
use platform/sops/lib.nu [on_sops]
|
||
|
|
}
|
||
|
|
let ip_type = if $iptype == "" { "public" } else { $iptype }
|
||
|
|
let str_created_taskservs_dirpath = ( $settings.data | get -o created_taskservs_dirpath | default (["/tmp"] | path join) |
|
||
|
|
str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME | str replace "NOW" $env.NOW
|
||
|
|
)
|
||
|
|
let created_taskservs_dirpath = if ($str_created_taskservs_dirpath | str starts-with "/" ) { $str_created_taskservs_dirpath } else { $settings.src_path | path join $str_created_taskservs_dirpath }
|
||
|
|
let root_wk_server = ($created_taskservs_dirpath | path join "on-server")
|
||
|
|
if not ($root_wk_server | path exists ) { ^mkdir "-p" $root_wk_server }
|
||
|
|
let dflt_clean_created_taskservs = ($settings.data.clean_created_taskservs? | default $created_taskservs_dirpath |
|
||
|
|
str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME
|
||
|
|
)
|
||
|
|
let run_ops = if (is-debug-enabled) { "bash -x" } else { "" }
|
||
|
|
let server_failed_flags = $settings.data.servers
|
||
|
|
| enumerate
|
||
|
|
| where {|it|
|
||
|
|
$match_server == "" or $it.item.hostname == $match_server
|
||
|
|
}
|
||
|
|
| each {|it|
|
||
|
|
let server_pos = $it.index
|
||
|
|
let srvr = $it.item
|
||
|
|
_print $"on (_ansi green_bold)($srvr.hostname)(_ansi reset) pos ($server_pos) ..."
|
||
|
|
let clean_created_taskservs = ($settings.data.servers | get $server_pos | get -o clean_created_taskservs | default $dflt_clean_created_taskservs)
|
||
|
|
|
||
|
|
# Determine IP address — resolve real IP when upload inspection is requested
|
||
|
|
let ip = if (is-debug-check-enabled) or ($check and not $upload) {
|
||
|
|
"127.0.0.1"
|
||
|
|
} else {
|
||
|
|
let curr_ip = (mw_get_ip $settings $srvr $ip_type false | default "")
|
||
|
|
if $curr_ip == "" {
|
||
|
|
_print $"🛑 No IP ($ip_type) found for (_ansi green_bold)($srvr.hostname)(_ansi reset) ($server_pos) "
|
||
|
|
null
|
||
|
|
} else {
|
||
|
|
# Check if server is in running state
|
||
|
|
if not (wait_for_server $server_pos $srvr $settings $curr_ip) {
|
||
|
|
_print $"🛑 server ($srvr.hostname) ($curr_ip) (_ansi red_bold)not in running state(_ansi reset)"
|
||
|
|
null
|
||
|
|
} else {
|
||
|
|
$curr_ip
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# Process server only if we have valid IP
|
||
|
|
if ($ip != null) {
|
||
|
|
let server = ($srvr | merge { ip_addresses: { pub: $ip, priv: ($srvr | get -o network_private_ip | default ($srvr | get -o networking.private_ip | default "")) }})
|
||
|
|
let wk_server = ($root_wk_server | path join $server.hostname)
|
||
|
|
let workspace_path = if ($settings.infra_path? | is-not-empty) {
|
||
|
|
$settings.infra_path | path dirname | path dirname
|
||
|
|
} else {
|
||
|
|
$settings.src_path? | default $env.PWD
|
||
|
|
}
|
||
|
|
let dag_deps = (load-dag-deps $settings $server.hostname)
|
||
|
|
if ($wk_server | path exists ) { rm -rf $wk_server }
|
||
|
|
^mkdir "-p" $wk_server
|
||
|
|
let taskserv_list = if $force_delete and $cmd == "delete" {
|
||
|
|
# --force: build taskserv list from state file, servers.ncl, or explicit name.
|
||
|
|
# Covers: removed from servers.ncl, never tracked in state, or both.
|
||
|
|
let st_taskservs = (state-read $workspace_path
|
||
|
|
| get -o servers | default {}
|
||
|
|
| get -o $server.hostname | default {}
|
||
|
|
| get -o taskservs | default {})
|
||
|
|
let from_state = ($st_taskservs | transpose name state_data | where {|it|
|
||
|
|
let matches_taskserv = ($match_taskserv == "" or $match_taskserv == $it.name)
|
||
|
|
let matches_profile = ($match_taskserv_profile == "" or $match_taskserv_profile == ($it.state_data.profile? | default "default"))
|
||
|
|
$matches_taskserv and $matches_profile
|
||
|
|
} | each {|it| $it.name })
|
||
|
|
|
||
|
|
# If explicit taskserv requested but not found in state, force-create a synthetic entry
|
||
|
|
let names = if ($match_taskserv | is-not-empty) and $match_taskserv not-in $from_state {
|
||
|
|
$from_state | append $match_taskserv
|
||
|
|
} else {
|
||
|
|
$from_state
|
||
|
|
}
|
||
|
|
|
||
|
|
$names | enumerate | each {|it| {
|
||
|
|
index: $it.index,
|
||
|
|
item: {
|
||
|
|
name: $it.item,
|
||
|
|
install_mode: "library",
|
||
|
|
profile: ($st_taskservs | get -o $it.item | default {} | get -o profile | default "default"),
|
||
|
|
target_save_path: "",
|
||
|
|
depends_on: [],
|
||
|
|
on_error: "Continue",
|
||
|
|
max_retries: 0,
|
||
|
|
params: {},
|
||
|
|
},
|
||
|
|
}}
|
||
|
|
} else {
|
||
|
|
let from_server = ($server.taskservs | enumerate | where {|it|
|
||
|
|
let taskserv = $it.item
|
||
|
|
let matches_taskserv = ($match_taskserv == "" or $match_taskserv == $taskserv.name)
|
||
|
|
let matches_profile = ($match_taskserv_profile == "" or $match_taskserv_profile == $taskserv.profile)
|
||
|
|
$matches_taskserv and $matches_profile
|
||
|
|
})
|
||
|
|
# L3 services (e.g. nats) live in separate DAG formula blocks, not in server.taskservs.
|
||
|
|
# Fall back to the DAG when the server record doesn't cover the requested component.
|
||
|
|
if ($match_taskserv | is-not-empty) and ($from_server | is-empty) {
|
||
|
|
(dag-synth-taskservs $settings $server.hostname $match_taskserv) | enumerate
|
||
|
|
} else {
|
||
|
|
$from_server
|
||
|
|
}
|
||
|
|
}
|
||
|
|
# Loud fail when caller asked for a specific taskserv but the resolved
|
||
|
|
# list is empty — this used to silently print "Tasks completed" and
|
||
|
|
# exit 0 without doing any work (caused operator confusion: bundle
|
||
|
|
# never shipped, install script never ran, but exit code = success).
|
||
|
|
# Now: name the gap so the operator knows what to declare, return
|
||
|
|
# `true` (failed flag) so the outer aggregation reports the failure,
|
||
|
|
# then `error make` so the bash wrapper sees a non-zero exit code.
|
||
|
|
if ($match_taskserv | is-not-empty) and ($taskserv_list | is-empty) {
|
||
|
|
_print $"🛑 (_ansi red_bold)taskserv `($match_taskserv)` not found on ($server.hostname)(_ansi reset)"
|
||
|
|
_print $" server.taskservs in servers.ncl does not list it, and"
|
||
|
|
_print $" no DAG formula synthesises it for this host."
|
||
|
|
_print $" → fix: add it to `taskservs = [...]` for ($server.hostname) in your workspace's servers.ncl"
|
||
|
|
error make {
|
||
|
|
msg: $"taskserv `($match_taskserv)` not declared on ($server.hostname)"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
mut stop_on_error = false
|
||
|
|
mut any_failed = false
|
||
|
|
for it in $taskserv_list {
|
||
|
|
if $stop_on_error { break }
|
||
|
|
let taskserv = $it.item
|
||
|
|
let taskserv_pos = $it.index
|
||
|
|
let taskservs_path = (get-taskservs-path)
|
||
|
|
let taskserv_dir = (find-taskserv-path $taskservs_path $taskserv.name)
|
||
|
|
|
||
|
|
# Check if taskserv path exists - skip if not found
|
||
|
|
if ($taskserv_dir | is-empty) {
|
||
|
|
_print $"taskserv path: ($taskservs_path)/($taskserv.name) (_ansi red_bold)not found(_ansi reset)"
|
||
|
|
} else {
|
||
|
|
# ── Resolve effective taskserv (cmd_task override) ────────────
|
||
|
|
let effective_taskserv = if ($cmd | is-not-empty) {
|
||
|
|
$taskserv | merge { cmd_task: $cmd }
|
||
|
|
} else if $reset {
|
||
|
|
$taskserv | merge { cmd_task: "reinstall" }
|
||
|
|
} else {
|
||
|
|
$taskserv
|
||
|
|
}
|
||
|
|
# Derive operation label and whether this is a deploy (install/reinstall)
|
||
|
|
# vs a maintenance op (update, scripts, restart, config, remove).
|
||
|
|
let effective_cmd = ($effective_taskserv.cmd_task? | default "install")
|
||
|
|
let effective_operation = match $effective_cmd {
|
||
|
|
"install" | "reinstall" => "create",
|
||
|
|
$op => $op,
|
||
|
|
}
|
||
|
|
# Only fresh installs go through the state-gate.
|
||
|
|
# reinstall/reset always runs regardless of current state.
|
||
|
|
let is_deploy = $effective_cmd == "install"
|
||
|
|
|
||
|
|
# ── State gate (fresh install only) ──────────────────────────
|
||
|
|
# reinstall, update, scripts, restart, config bypass the gate.
|
||
|
|
if not $check {
|
||
|
|
if $is_deploy {
|
||
|
|
let depends_on = ($dag_deps | get -o $taskserv.name | default [])
|
||
|
|
let decision = (state-node-decision-with-deps $workspace_path $server.hostname $taskserv.name $depends_on)
|
||
|
|
match $decision {
|
||
|
|
"skip" => {
|
||
|
|
let node = (state-node-get $workspace_path $server.hostname $taskserv.name)
|
||
|
|
_print $"⊘ ($taskserv.name) on ($server.hostname) — state=completed \(ended ($node.ended_at? | default '?')). Run reset first."
|
||
|
|
continue
|
||
|
|
},
|
||
|
|
$d if ($d | str starts-with "blocked:") => {
|
||
|
|
let blocker = ($d | str replace "blocked:" "")
|
||
|
|
_print $"⛔ ($taskserv.name) on ($server.hostname) — blocked by ($blocker) \(not completed)"
|
||
|
|
continue
|
||
|
|
},
|
||
|
|
"rerun" => {
|
||
|
|
_print $"↻ ($taskserv.name) on ($server.hostname) — failed, re-running"
|
||
|
|
},
|
||
|
|
_ => {},
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
_print $"↺ ($taskserv.name) on ($server.hostname) — ($effective_cmd)"
|
||
|
|
}
|
||
|
|
let actor = ($env.USER? | default "system")
|
||
|
|
let profile = ($taskserv.profile? | default "")
|
||
|
|
state-node-start $workspace_path $server.hostname $taskserv.name --actor $actor --source "orchestrator" --operation $effective_operation --profile $profile
|
||
|
|
}
|
||
|
|
# ─────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
# Taskserv path exists, proceed with processing
|
||
|
|
if not ($wk_server | path join $taskserv.name| path exists) { ^mkdir "-p" ($wk_server | path join $taskserv.name) }
|
||
|
|
let $taskserv_profile = if $taskserv.profile == "" { "default" } else { $taskserv.profile }
|
||
|
|
let $taskserv_install_mode = if $taskserv.install_mode == "" { "library" } else { $taskserv.install_mode }
|
||
|
|
let server_taskserv_path = ($server.hostname | path join $taskserv_profile)
|
||
|
|
let defs = {
|
||
|
|
settings: $settings, server: $server, taskserv: $effective_taskserv,
|
||
|
|
taskserv_install_mode: $taskserv_install_mode, taskserv_profile: $taskserv_profile,
|
||
|
|
taskserv_dir: $taskserv_dir,
|
||
|
|
pos: { server: $"($server_pos)", taskserv: $taskserv_pos}, ip: $ip, check: $check }
|
||
|
|
|
||
|
|
# Enhanced check mode
|
||
|
|
if $check {
|
||
|
|
let check_result = (run-check-mode $taskserv.name $taskserv_profile $settings $server --verbose=(is-debug-enabled))
|
||
|
|
if $check_result.overall_valid {
|
||
|
|
# Check passed, proceed (no action needed, validation was successful)
|
||
|
|
} else {
|
||
|
|
_print $"(_ansi red)⊘ Skipping deployment due to validation errors(_ansi reset)"
|
||
|
|
}
|
||
|
|
if $upload {
|
||
|
|
run-upload-inspection $defs --verbose=(is-debug-enabled)
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
# Normal installation mode — functions return bool; false = failure
|
||
|
|
let install_ok = match $taskserv.install_mode {
|
||
|
|
"server" | "getfile" => {
|
||
|
|
(install_from_server $defs $server_taskserv_path $wk_server)
|
||
|
|
},
|
||
|
|
"library-server" => {
|
||
|
|
let a = (install_from_library $defs $server_taskserv_path $wk_server)
|
||
|
|
let b = (install_from_server $defs $server_taskserv_path $wk_server)
|
||
|
|
$a and $b
|
||
|
|
},
|
||
|
|
"server-library" => {
|
||
|
|
let a = (install_from_server $defs $server_taskserv_path $wk_server)
|
||
|
|
let b = (install_from_library $defs $server_taskserv_path $wk_server)
|
||
|
|
$a and $b
|
||
|
|
},
|
||
|
|
"library" => {
|
||
|
|
(install_from_library $defs $server_taskserv_path $wk_server)
|
||
|
|
},
|
||
|
|
"local" => {
|
||
|
|
# Runs install script on the provisioning machine (not via SSH).
|
||
|
|
# Used for tools like k0sctl that manage their own remote connections.
|
||
|
|
(install_from_library $defs $server_taskserv_path $wk_server)
|
||
|
|
},
|
||
|
|
}
|
||
|
|
if not $install_ok {
|
||
|
|
_print $"🛑 ($taskserv.name) on ($server.hostname) failed"
|
||
|
|
$any_failed = true
|
||
|
|
state-node-finish $workspace_path $server.hostname $taskserv.name --source "orchestrator"
|
||
|
|
if ($taskserv.on_error? | default "Continue") == "Stop" {
|
||
|
|
$stop_on_error = true
|
||
|
|
}
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
}
|
||
|
|
# Write completed state after successful execution.
|
||
|
|
# reinstall = reset-only: transition back to pending so the next
|
||
|
|
# install create goes through the gate normally.
|
||
|
|
if not $check {
|
||
|
|
if $effective_cmd == "delete" {
|
||
|
|
state-node-delete $workspace_path $server.hostname $taskserv.name
|
||
|
|
} else if $effective_cmd == "reinstall" {
|
||
|
|
state-node-reset $workspace_path $server.hostname $taskserv.name --source "orchestrator" --actor ($env.USER? | default "system")
|
||
|
|
} else {
|
||
|
|
state-node-finish $workspace_path $server.hostname $taskserv.name --success --source "orchestrator"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if $clean_created_taskservs == "yes" { rm -rf ($wk_server | path join $taskserv.name) }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if $clean_created_taskservs == "yes" { rm -rf $wk_server }
|
||
|
|
_print $"Tasks completed on ($server.hostname)"
|
||
|
|
$any_failed
|
||
|
|
} else {
|
||
|
|
false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let any_failed = ($server_failed_flags | any {|it| $it })
|
||
|
|
if ("/tmp/k8s_join.sh" | path exists) { cp "/tmp/k8s_join.sh" $root_wk_server ; rm -r /tmp/k8s_join.sh }
|
||
|
|
if $dflt_clean_created_taskservs == "yes" { rm -rf $root_wk_server }
|
||
|
|
if $any_failed {
|
||
|
|
_print $"❌ Tasks (_ansi red_bold)completed with failures(_ansi reset) ($match_server) ($match_taskserv) ($match_taskserv_profile)"
|
||
|
|
} else {
|
||
|
|
_print $"✅ Tasks (_ansi green_bold)completed(_ansi reset) ($match_server) ($match_taskserv) ($match_taskserv_profile) ....."
|
||
|
|
if not $check and ($match_taskserv | is-not-empty) {
|
||
|
|
show-next-step "taskserv_create" {name: $match_taskserv}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
true
|
||
|
|
}
|