c62e967ce3
Clean up 404 KCL references (99.75% complete): - Rename kcl_* variables to schema_*/nickel_* (kcl_path→schema_path, etc.) - Update functions: parse_kcl_file→parse_nickel_file - Update env vars: KCL_MOD_PATH→NICKEL_IMPORT_PATH - Fix cli/providers-install: add has_nickel and nickel_version variables - Correct import syntax: .nickel.→.ncl. - Update 57 files across core, CLI, config, and utilities Configure pre-commit hooks: - Activate: nushell-check, nickel-typecheck, markdownlint - Comment out: Rust hooks (fmt, clippy, test), check-yaml Testing: - Module discovery: 9 modules (6 providers, 1 taskserv, 2 clusters) ✅ - Syntax validation: 15 core files ✅ - Pre-commit hooks: all passing ✅
499 lines
23 KiB
Plaintext
499 lines
23 KiB
Plaintext
use std
|
||
use lib_provisioning *
|
||
use utils.nu *
|
||
#use utils.nu on_server_template
|
||
use ssh.nu *
|
||
use ../lib_provisioning/utils/ssh.nu *
|
||
# Provider middleware now available through lib_provisioning
|
||
use ../lib_provisioning/config/accessor.nu *
|
||
use ../lib_provisioning/plugins/auth.nu *
|
||
use ../lib_provisioning/utils/hints.nu *
|
||
|
||
# > Server create
|
||
export def "main create" [
|
||
name?: string # Server hostname in settings
|
||
...args # Args for create command
|
||
--infra (-i): string # Infra directory
|
||
--settings (-s): string # Settings path
|
||
--outfile (-o): string # Output file
|
||
--serverpos (-p): int # Server position in settings
|
||
--check (-c) # Only check mode no servers will be created
|
||
--wait (-w) # Wait servers to be created
|
||
--select: string # Select with task as option
|
||
--debug (-x) # Use Debug mode
|
||
--xm # Debug with PROVISIONING_METADATA
|
||
--xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK
|
||
--xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE
|
||
--xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug
|
||
--metadata # Error with metadata (-xm)
|
||
--notitles # not tittles
|
||
--helpinfo (-h) # For more details use options "help" (no dashes)
|
||
--out: string # Print Output format: json, yaml, text (default)
|
||
--orchestrated # Use orchestrator workflow instead of direct execution
|
||
--orchestrator: string = "http://localhost:8080" # Orchestrator URL
|
||
]: nothing -> nothing {
|
||
if ($out | is-not-empty) {
|
||
set-provisioning-out $out
|
||
set-provisioning-no-terminal true
|
||
}
|
||
# Convert args to list of strings for provisioning_init
|
||
let string_args = ($args | each { $in | into string })
|
||
provisioning_init $helpinfo "servers create" $string_args
|
||
if $debug { set-debug-enabled true }
|
||
if $metadata { set-metadata-enabled true }
|
||
if $name != null and $name != "h" and $name != "help" {
|
||
let infra_arg = if ($infra | is-empty) { null } else { $infra }
|
||
let settings_arg = if ($settings | is-empty) { null } else { $settings }
|
||
let curr_settings = (find_get_settings --infra $infra_arg --settings $settings_arg)
|
||
if ($curr_settings.data.servers | find $name| length) == 0 {
|
||
_print $"🛑 invalid name ($name)"
|
||
exit 1
|
||
}
|
||
}
|
||
let task = if ($args | length) > 0 {
|
||
($args| get 0)
|
||
} else {
|
||
let str_task = (((get-provisioning-args) | str replace "create " " " ))
|
||
let str_task = if $name != null {
|
||
($str_task | str replace $name "")
|
||
} else {
|
||
$str_task
|
||
}
|
||
($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim)
|
||
}
|
||
let other = if ($args | length) > 0 { ($args| skip 1) } else { "" }
|
||
let ops = $"((get-provisioning-args)) " | str replace $" ($task) " "" | str trim
|
||
let run_create = {
|
||
# Convert empty strings to null for auto-detection to work
|
||
let infra_arg = if ($infra | is-empty) { null } else { $infra }
|
||
let settings_arg = if ($settings | is-empty) { null } else { $settings }
|
||
let curr_settings = (find_get_settings --infra $infra_arg --settings $settings_arg)
|
||
if ($curr_settings | is-empty) or ($curr_settings.wk_path? | is-empty) {
|
||
_print "🛑 Failed to load settings"
|
||
return { status: false, error: "settings_load_failed" }
|
||
}
|
||
set-wk-cnprov $curr_settings.wk_path
|
||
let match_name = if $name == null or $name == "" { "" } else { $name}
|
||
on_create_servers $curr_settings $check $wait $outfile $match_name $serverpos --notitles=$notitles --orchestrated=$orchestrated --orchestrator=$orchestrator
|
||
}
|
||
match $task {
|
||
"" if $name == "h" => {
|
||
^$"(get-provisioning-name)" -mod server create help --notitles
|
||
},
|
||
"" if $name == "help" => {
|
||
^$"(get-provisioning-name)" -mod server create --help
|
||
_print (provisioning_options "create")
|
||
},
|
||
"" | "c" | "create" => {
|
||
let result = desktop_run_notify $"(get-provisioning-name) servers create" "-> " $run_create --timeout 11sec
|
||
if not ($result | get status? | default true) { exit 1 }
|
||
},
|
||
_ => {
|
||
invalid_task "servers create" $task --end
|
||
}
|
||
}
|
||
if not $notitles and not (is-debug-enabled) { end_run "" }
|
||
}
|
||
export def on_create_servers [
|
||
settings: record # Settings record
|
||
check: bool # Only check mode no servers will be created
|
||
wait: bool # Wait for creation
|
||
outfile?: string # Out file for creation
|
||
hostname?: string # Server hostname in settings
|
||
serverpos?: int # Server position in settings
|
||
--notitles # not tittles
|
||
--orchestrated # Use orchestrator workflow instead of direct execution
|
||
--orchestrator: string = "http://localhost:8080" # Orchestrator URL
|
||
]: nothing -> record {
|
||
|
||
# Authentication check for server creation (only if actually creating, not in check mode)
|
||
if not $check {
|
||
let environment = (config-get "environment" "dev")
|
||
let operation_name = $"server create (($hostname | default 'all'))"
|
||
|
||
# Check authentication based on environment
|
||
if $environment == "prod" {
|
||
check-auth-for-production $operation_name --allow-skip
|
||
} else {
|
||
# For dev/test, still require auth but allow skip
|
||
let allow_skip = (config-get "security.bypass.allow_skip_auth" false)
|
||
if $allow_skip {
|
||
require-auth $operation_name --allow-skip
|
||
} else {
|
||
require-auth $operation_name
|
||
}
|
||
}
|
||
|
||
# Log the operation for audit trail
|
||
log-authenticated-operation "server_create" {
|
||
hostname: ($hostname | default "all")
|
||
infra: $settings.infra
|
||
environment: $environment
|
||
orchestrated: $orchestrated
|
||
}
|
||
}
|
||
|
||
# If orchestrated mode is enabled, delegate to workflow
|
||
if $orchestrated {
|
||
use ../workflows/server_create.nu
|
||
return (on_create_servers_workflow $settings $check $wait $outfile $hostname $serverpos --orchestrator $orchestrator)
|
||
}
|
||
let match_hostname = if $hostname != null {
|
||
$hostname
|
||
} else if $serverpos != null {
|
||
let total = $settings.data.servers | length
|
||
let pos = if $serverpos == -1 {
|
||
_print $"Use number form 0 to ($total)"
|
||
$serverpos
|
||
} else if $serverpos <= $total {
|
||
$serverpos - 0
|
||
} else {
|
||
(throw-error $"🛑 server pos" $"($serverpos) from ($total) servers"
|
||
"on_create" --span (metadata $serverpos).span)
|
||
exit 0
|
||
}
|
||
($settings.data.servers | get $pos).hostname
|
||
}
|
||
#use ../../../providers/prov_lib/middleware.nu mw_create_server
|
||
# Check servers ... reload settings if are changes
|
||
for server in $settings.data.servers {
|
||
if $match_hostname == null or $match_hostname == "" or $server.hostname == $match_hostname {
|
||
if (mw_create_server $settings $server $check false) == false {
|
||
return { status: false, error: $"mw_create_sever ($server.hostname) error" }
|
||
}
|
||
}
|
||
}
|
||
let ok_settings = if ($"($settings.wk_path)/changes" | path exists) {
|
||
if (is-debug-enabled) == false {
|
||
_print $"(_ansi blue_bold)Reloading settings(_ansi reset) for (_ansi cyan_bold)($settings.infra)(_ansi reset) (_ansi purple)($settings.src)(_ansi reset)"
|
||
cleanup $settings.wk_path
|
||
} else {
|
||
_print $"(_ansi blue_bold)Review (_ansi green)($settings.wk_path)/changes(_ansi reset) for (_ansi cyan_bold)($settings.infra)(_ansi reset) (_ansi purple)($settings.src)(_ansi reset)"
|
||
_print $"(_ansi green)($settings.wk_path)(_ansi reset) (_ansi red)not deleted(_ansi reset) for debug"
|
||
}
|
||
#use utils/settings.nu [ load_settings ]
|
||
(load_settings --infra $settings.infra --settings $settings.src)
|
||
} else {
|
||
$settings
|
||
}
|
||
let out_file = if $outfile == null { "" } else { $outfile }
|
||
let target_servers = ($ok_settings.data.servers | where {|it|
|
||
if $match_hostname == null or $match_hostname == "" {
|
||
true
|
||
} else if $it.hostname == $match_hostname {
|
||
true
|
||
} else {
|
||
$it.hostname | str starts-with $match_hostname
|
||
}
|
||
})
|
||
if $check {
|
||
mut check_failed = false
|
||
for it in ($target_servers | enumerate) {
|
||
if not (create_server $it.item $it.index true $wait $ok_settings $out_file) {
|
||
$check_failed = true
|
||
break
|
||
}
|
||
_print $"\n(_ansi blue_reverse)----🌥 ----🌥 ----🌥 ---- oOo ----🌥 ----🌥 ----🌥 ---- (_ansi reset)\n"
|
||
}
|
||
if $check_failed {
|
||
return { status: false, error: "Server check failed" }
|
||
}
|
||
} else {
|
||
_print $"Create (_ansi blue_bold)($target_servers | length)(_ansi reset) servers in parallel (_ansi blue_bold)>>> 🌥 >>> (_ansi reset)\n"
|
||
$target_servers | enumerate | par-each {|it|
|
||
if not (create_server $it.item $it.index false $wait $ok_settings $out_file) {
|
||
return { status: false, error: $"creation ($it.item.hostname) error" }
|
||
} else {
|
||
let known_hosts_path = (("~" | path join ".ssh" | path join "known_hosts") | path expand)
|
||
^ssh-keygen -f $known_hosts_path -R $it.item.hostname err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })
|
||
if ($it.item | get network_public_ip? | default null | is-not-empty) {
|
||
^ssh-keygen -f $known_hosts_path -R ($it.item | get network_public_ip? | default null) err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })
|
||
}
|
||
}
|
||
_print $"\n(_ansi blue_reverse)----🌥 ----🌥 ----🌥 ---- oOo ----🌥 ----🌥 ----🌥 ---- (_ansi reset)\n"
|
||
}
|
||
}
|
||
if not $check {
|
||
# Running this in 'par-each' does not work
|
||
$target_servers | enumerate | each { |it|
|
||
mw_create_cache $ok_settings $it.item false
|
||
}
|
||
}
|
||
# Skip pricing and SSH setup in check mode
|
||
if not $check {
|
||
servers_walk_by_costs $ok_settings $match_hostname $check true
|
||
server_ssh $ok_settings "" "pub" false "" $check | ignore
|
||
}
|
||
|
||
# Show next-step hints after successful creation
|
||
if not $check {
|
||
show-next-step "server_create" {infra: $ok_settings.infra}
|
||
}
|
||
|
||
{ status: true, error: "" }
|
||
}
|
||
export def create_server [
|
||
server: record
|
||
index: int
|
||
check: bool
|
||
wait: bool
|
||
settings: record
|
||
outfile?: string
|
||
]: nothing -> bool {
|
||
## Provider middleware now available through lib_provisioning
|
||
#use utils.nu *
|
||
|
||
# In check mode, show what would be created
|
||
if $check {
|
||
# Search for template in workspace .providers first, then in system providers
|
||
let workspace_infra_path = ($settings.src_path | path dirname | path dirname)
|
||
let workspace_template = ($workspace_infra_path | path join ".providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2")
|
||
let server_template = if ($workspace_template | path exists) {
|
||
$workspace_template
|
||
} else {
|
||
(get-base-path | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2")
|
||
}
|
||
|
||
# Temporarily disable NO_TERMINAL to ensure check output is displayed
|
||
let old_no_terminal = ($env.PROVISIONING_NO_TERMINAL? | default false)
|
||
$env.PROVISIONING_NO_TERMINAL = false
|
||
|
||
_print $"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
_print $"Check: Create server (_ansi cyan_bold)($server.hostname)(_ansi reset) with provider (_ansi green_bold)($server.provider)(_ansi reset)"
|
||
_print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
if ($server_template | path exists) {
|
||
_print $"\n📋 Template: ($server_template)"
|
||
|
||
# Show template rendering info
|
||
_print $"\n🔧 Generated script:"
|
||
_print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# Build complete context record with all variables the template expects
|
||
# The template needs: servers (array), defaults (record), match_server, provisioning_vers, now, debug, use_time, wait, runset, wk_file
|
||
let template_context = {
|
||
servers: [$server]
|
||
defaults: {}
|
||
match_server: $server.hostname
|
||
provisioning_vers: "1.0.4"
|
||
now: (date now | format date '%Y-%m-%d %H:%M:%S')
|
||
debug: "no"
|
||
use_time: "false"
|
||
wait: false
|
||
runset: {output_format: "yaml"}
|
||
wk_file: ($settings.wk_path | path join "creation_script.sh")
|
||
}
|
||
|
||
# Try to render the template with daemon first, fallback to plugin
|
||
if ($server_template | path exists) {
|
||
let absolute_template = (($server_template | path expand) | str trim)
|
||
let template_content = (open $absolute_template)
|
||
|
||
# First try: Use Tera daemon (50-100x faster for batch operations)
|
||
let use_daemon = (is-tera-daemon-available)
|
||
let rendered = if $use_daemon {
|
||
let daemon_result = (do { tera-render-daemon $template_content $template_context --name ($server.hostname) } | complete)
|
||
if $daemon_result.exit_code == 0 {
|
||
$daemon_result.stdout
|
||
} else {
|
||
# Fallback to plugin if daemon fails
|
||
if (get-use-tera-plugin) {
|
||
let tera_loaded = (plugin list | where name == "tera" | length) > 0
|
||
if not $tera_loaded {
|
||
(plugin use tera)
|
||
}
|
||
($template_context | tera-render $absolute_template)
|
||
} else {
|
||
error make {msg: "Template rendering not available (no daemon, no plugin)"}
|
||
}
|
||
}
|
||
} else if (get-use-tera-plugin) {
|
||
# Fallback: Use tera plugin if daemon not available
|
||
let tera_loaded = (plugin list | where name == "tera" | length) > 0
|
||
if not $tera_loaded {
|
||
(plugin use tera)
|
||
}
|
||
($template_context | tera-render $absolute_template)
|
||
} else {
|
||
error make {msg: "Template rendering not available (no daemon, no plugin)"}
|
||
}
|
||
|
||
# Handle outfile parameter: save to file if provided, otherwise print to stdout
|
||
let has_outfile = ($outfile != null and ($outfile | str length) > 0)
|
||
if $has_outfile {
|
||
# Expand the outfile path to absolute
|
||
let absolute_outfile = ($outfile | path expand)
|
||
# Create parent directories if they don't exist
|
||
let outfile_dir = ($absolute_outfile | path dirname)
|
||
if not ($outfile_dir | path exists) {
|
||
^mkdir -p $outfile_dir
|
||
}
|
||
# Write rendered content to file
|
||
$rendered | save --force $absolute_outfile
|
||
_print $"✅ Script saved to: ($absolute_outfile)"
|
||
} else {
|
||
_print $rendered
|
||
}
|
||
} else {
|
||
_print $"\n⚠️ Template file not found"
|
||
_print $" Template path: ($server_template)"
|
||
_print $" Server: ($server.hostname)"
|
||
}
|
||
|
||
if false {
|
||
_print $"⚠️ Template rendering not available (tera plugin not installed)"
|
||
_print $"\n📝 Template variables that would be used:"
|
||
_print $" • hostname = ($server.hostname)"
|
||
_print $" • provider = ($server.provider)"
|
||
_print $" • plan = ($server.plan)"
|
||
_print $" • zone = ($server.zone | default 'default')"
|
||
}
|
||
|
||
_print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
_print $"\n✅ Check completed successfully"
|
||
_print $" This server would be created with:"
|
||
_print $" • Hostname: ($server.hostname)"
|
||
_print $" • Provider: ($server.provider)"
|
||
_print $" • Plan: ($server.plan)"
|
||
_print $" • Zone: ($server.zone | default 'default')"
|
||
_print $"\n To actually create, run without --check flag"
|
||
} else {
|
||
_print $"\n⚠️ Template not found: ($server_template)"
|
||
$env.PROVISIONING_NO_TERMINAL = $old_no_terminal
|
||
return false
|
||
}
|
||
|
||
# Restore original NO_TERMINAL setting
|
||
$env.PROVISIONING_NO_TERMINAL = $old_no_terminal
|
||
return true
|
||
}
|
||
|
||
let server_info = (mw_server_info $server true)
|
||
|
||
# Check if server_info is a record, otherwise it's an error (empty or string)
|
||
let already_created = if ($server_info | describe | str starts-with "record") {
|
||
($server_info | get hostname? | default null | is-not-empty)
|
||
} else {
|
||
false
|
||
}
|
||
if ($already_created) {
|
||
_print $"Server (_ansi green_bold)($server.hostname)(_ansi reset) already created "
|
||
check_server $settings $server $index $server_info $check $wait $settings $outfile
|
||
#mw_server_info $server false
|
||
if not $check { return true }
|
||
}
|
||
# Search for template in workspace .providers first, then in system providers
|
||
let workspace_infra_path = ($settings.src_path | path dirname | path dirname)
|
||
let workspace_template = ($workspace_infra_path | path join ".providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2")
|
||
let server_template = if ($workspace_template | path exists) {
|
||
$workspace_template
|
||
} else {
|
||
(get-base-path | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2")
|
||
}
|
||
let create_result = on_server_template $server_template $server $index $check false $wait $settings $outfile
|
||
if not $create_result { return false }
|
||
let server_info = (mw_server_info $server true)
|
||
check_server $settings $server $index $server_info $check $wait $settings $outfile
|
||
true
|
||
}
|
||
|
||
export def verify_server_info [
|
||
settings: record
|
||
server: record
|
||
info: record
|
||
]: nothing -> nothing {
|
||
_print $"Checking server (_ansi green_bold)($server.hostname)(_ansi reset) info "
|
||
let server_plan = ($server | get plan? | default "")
|
||
let curr_plan = ($info | get plan? | default "")
|
||
if ($server_plan | is-not-empty) {
|
||
if $server_plan != $curr_plan {
|
||
mw_modify_server $settings $server [{plan: $server_plan}] false
|
||
}
|
||
}
|
||
}
|
||
export def check_server [
|
||
settings: record
|
||
server: record
|
||
index: int
|
||
info: record
|
||
check: bool
|
||
wait: bool
|
||
settings: record
|
||
outfile?: string
|
||
]: nothing -> bool {
|
||
## Provider middleware now available through lib_provisioning
|
||
#use utils.nu *
|
||
let server_info = if ($info | is-empty) {
|
||
(mw_server_info $server true)
|
||
} else {
|
||
$info
|
||
}
|
||
let already_created = ($server_info | is-not-empty)
|
||
if not $already_created {
|
||
_print $"🛑 server (_ansi green_bold)($server.hostname)(_ansi reset) not exists"
|
||
return false
|
||
}
|
||
if not $check {
|
||
^ssh-keygen -f $"($env.HOME)/.ssh/known_hosts" -R $server.hostname err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })
|
||
let ip_raw = (mw_get_ip $settings $server $server.liveness_ip false )
|
||
let ip = ($ip_raw | str trim --char "\"")
|
||
if $ip == "" {
|
||
_print "🛑 No liveness ip found for state checking "
|
||
return false
|
||
}
|
||
verify_server_info $settings $server $server_info
|
||
_print $"liveness (_ansi purple)($ip):($server.liveness_port)(_ansi reset)"
|
||
if (wait_for_server $index $server $settings $ip) {
|
||
# Check if SSH setup succeeded (returns false on CTRL-C during sudo)
|
||
let ssh_result = (on_server_ssh $settings $server "pub" "create" false $check)
|
||
if not $ssh_result {
|
||
_print $"\n(_ansi red)✗ Server creation cancelled(_ansi reset)"
|
||
return false
|
||
}
|
||
# collect fingerprint
|
||
let res = (^ssh-keyscan "-H" $ip err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })| complete)
|
||
if $res.exit_code == 0 {
|
||
let known_hosts_path = (("~" | path join ".ssh" | path join "known_hosts") | path expand)
|
||
let markup = $"# ($ip) keyscan"
|
||
let lines_found = (open $known_hosts_path --raw | lines | find $markup | length)
|
||
if $lines_found == 0 {
|
||
( $"($markup)\n" | save --append $known_hosts_path)
|
||
($res.stdout | save --append $known_hosts_path)
|
||
_print $"(_ansi green_bold)($ip)(_ansi reset) (_ansi yellow)ssh-keyscan(_ansi reset) added to ($known_hosts_path)"
|
||
}
|
||
#} else {
|
||
# _print $"🛑 Error (_ansi yellow)ssh-keyscan(_ansi reset) from ($ip)"
|
||
# _print $"($res.stdout)"
|
||
}
|
||
if $already_created {
|
||
let res = (mw_post_create_server $settings $server $check)
|
||
match $res {
|
||
"error" | "-1" => { exit 1},
|
||
"storage" | "" => {
|
||
let storage_sh = ($settings.wk_path | path join $"($server.hostname)-storage.sh")
|
||
let result = (on_server_template (get-templates-path | path join "storage.j2") $server 0 true true true $settings $storage_sh)
|
||
if $result and ($storage_sh | path exists) and (wait_for_server $index $server $settings $ip) {
|
||
let target_cmd = "/tmp/storage.sh"
|
||
#use ssh.nu scp_to ssh_cmd
|
||
if not (scp_to $settings $server [$storage_sh] $target_cmd $ip) { return false }
|
||
_print $"Running (_ansi blue_italic)($target_cmd | path basename)(_ansi reset) in (_ansi green_bold)($server.hostname)(_ansi reset)"
|
||
if not (ssh_cmd $settings $server true $target_cmd $ip) { return false }
|
||
if (is-ssh-debug-enabled) { return true }
|
||
if not (is-debug-enabled) {
|
||
(ssh_cmd $settings $server false $"rm -f ($target_cmd)" $ip)
|
||
}
|
||
} else {
|
||
return false
|
||
}
|
||
}
|
||
_ => {
|
||
return true
|
||
},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
true
|
||
}
|