prvng_core/nulib/servers/create.nu

use std
# REMOVED: use lib_provisioning * - causes circular import
use utils.nu *
use ../images/state.nu *
use delete.nu [sync-servers-state-post-op]
#use utils.nu on_server_template
use ssh.nu *
use ../lib_provisioning/utils/ssh.nu *
# Provider middleware now available through lib_provisioning
use ../lib_provisioning/plugins/auth.nu *
use ../lib_provisioning/utils/hints.nu *
use ../lib_provisioning/utils/init.nu *
use ../lib_provisioning/utils/logging.nu *
use ../lib_provisioning/utils/script-compression.nu *
use ../lib_provisioning/platform/service-manager.nu [load-service-config get-service-port]
# COMMENTED OUT: tera_daemon.nu has parse errors - will use fallback tera plugin
# use ../lib_provisioning/tera_daemon.nu *

use ../lib_provisioning/providers.nu [mw_enrich_template_context]
use ../lib_provisioning/utils/undefined.nu [invalid_task]
use ../lib_provisioning/utils/nickel_processor.nu [ncl-eval-soft]
use ../lib_provisioning/utils/settings.nu *
use ../lib_provisioning/utils/interface.nu [set-provisioning-no-terminal set-provisioning-out _ansi _print end_run desktop_run_notify]

# ─────────────────────────────────────────────────────────────────
# Multi-Template Orchestration Helpers (Phase 1)
# Enables conditional template rendering based on server configuration
# ─────────────────────────────────────────────────────────────────

# Determine if template should be rendered based on server config
def should_render_template [
    server: record
    template_name: string
]: nothing -> bool {
    match $template_name {
        "common_vals" => true,                                       # Always first: shared header
        "ssh_keys" => true,                                          # Always required
        "networks" => ($server.networking?.private_network? != null), # Conditional: only if networking.private_network defined
        "volumes" => (  # top-level volumes OR schema-nested storage.additional_volumes
            ($server.volumes? | default [] | length) > 0 or
            ($server.storage?.additional_volumes? | default [] | length) > 0
        ),
        "servers" => true,                                          # Always required
        "firewalls" => true,                                        # Always required
        _ => false
    }
}

# Build template-specific context for each template type
def build_template_context [
    base_context: record
    server: record
    template_name: string
]: nothing -> record {
    let context = $base_context

    match $template_name {
        "ssh_keys" => {
            let ssh_key_config = if ($server.ssh_keys? | default [] | is-not-empty) {
                {
                    name: ($server.ssh_keys | first),
                    public_key_path: $"~/.ssh/(($server.ssh_keys | first)).pub"
                }
            } else {
                # Default to htz_ops (Hetzner operations SSH key)
                # This should be present in ~/.ssh/htz_ops.pub
                # CRITICAL: This is the fallback when ssh_keys is not properly exported from Nickel
                { name: "htz_ops", public_key_path: "~/.ssh/htz_ops.pub" }
            }
            ($context | merge { ssh_key: $ssh_key_config })
        }
        "networks" => {
            if ($server.networking?.private_network? != null) {
                # Map server location to Hetzner network zone (must match server zone)
                let location = ($server.location? | default "nbg1")
                let network_zone = match ($location | str downcase) {
                    "ash" | "ash1" | "as-south" => "ap-southeast",  # Ashburn → Singapur
                    "sjc" | "sjc1" | "us-west" => "us-west",         # San Jose
                    "fsn" | "fsn1" | "eu-central" => "eu-central",   # Falkenstein
                    "hel" | "hel1" | "eu-central" => "eu-central",   # Helsinki
                    "nbg" | "nbg1" | "eu-central" => "eu-central",   # Nuremberg
                    _ => "eu-central"                                  # Default
                }

                # Build subnet with /22 (supports 1024 IPs instead of 256)
                let ip_range = ($server.networking.ip_range? | default "10.0.0.0/16")
                let subnet_range = ($server.networking.subnet_range? | default "10.0.0.0/24")

                let network_config = {
                    name: $server.networking.private_network,
                    ip_range: $ip_range,
                    subnet_range: $subnet_range,
                    zone: $network_zone
                }
                ($context | merge { network: $network_config })
            } else {
                $context
            }
        }
        "volumes" => {
            let declared = ($server.volumes? | default [])
            let from_storage = (
                $server.storage?.additional_volumes? | default []
                | each {|v| {
                    name:     $v.name
                    size:     ($v.size_gb? | default 20)
                    location: ($server.location? | default "nbg1")
                    format:   ($v.type? | default "ext4")
                    mount_path:     ($v.mount_path? | default "")
                    permanent_mount: ($v.permanent_mount? | default true)
                    volume_state:   ($v.volume_state? | default "new")
                }}
            )
            let all_vols = ($declared | append $from_storage)
            # Expose both `server` (singular) and `servers` so the template can reference
            # server.hostname for the attach step
            ($context | merge { volumes: $all_vols, server: $server })
        }
        "firewalls" => $context
        "servers" => {
            # Enrich server record: resolve floating_ip_address from state if not set in NCL.
            # Priority: NCL explicit value > .servers-state.json > .provisioning-state.json (bootstrap FIPs)
            let fip_name = ($server.floating_ip? | default "")
            let fip_addr = ($server.floating_ip_address? | default "")
            if ($fip_name | is-not-empty) and ($fip_addr | is-empty) {
                let ws_root = ($env.PROVISIONING_WORKSPACE_PATH? | default "")
                let infra_name = ($server.infra? | default "")

                # Try .servers-state.json first
                let srv_state_path = ($ws_root | path join "infra" | path join $infra_name | path join ".servers-state.json")
                let srv_cached_fip = if ($srv_state_path | path exists) {
                    open $srv_state_path | get -o ($server.hostname? | default "") | get -o floating_ip_address | default ""
                } else { "" }

                # Fallback: bootstrap state FIP lookup by name
                let resolved_ip = if ($srv_cached_fip | is-not-empty) {
                    $srv_cached_fip
                } else {
                    let bs_path = ($ws_root | path join ".provisioning-state.json")
                    if ($bs_path | path exists) {
                        let fip_key = ($fip_name | str replace --all "librecloud-fip-" "" | str replace --all "-" "_")
                        open $bs_path | get -o $"bootstrap.floating_ips.($fip_key).ip" | default ""
                    } else { "" }
                }

                let enriched_server = ($server | upsert floating_ip_address $resolved_ip)
                ($context | upsert servers [$enriched_server])
            } else {
                $context
            }
        }
        _ => $context
    }
}

# Concatenate multi-template sections into single atomic bash script
def concatenate_script_sections [
    sections: list
]: nothing -> string {
    let sorted = ($sections | sort-by priority)

    # common_vals (priority 0) MUST be first and without a delimiter so #!/bin/bash is line 1
    let body = (
        $sorted
        | each { |section|
            if ($section.priority == 0) {
                # Header section: raw content first, no delimiter
                $"($section.content)\n"
            } else {
                let delimiter = $"\n# ========== (($section.name | str upcase)) ==========\n"
                let state_load = "[ -f \"\$STATE_DIR/.env\" ] && source \"\$STATE_DIR/.env\"\n"
                $"($delimiter)($state_load)($section.content)\n"
            }
        }
        | str join ""
    )

    let footer = "\n# ========== COMPLETE ==========\n"

    [$body, $footer] | str join ""
}

# Get orchestrator URL from platform config/env
# Priority:
#   1. PROVISIONING_ORCHESTRATOR_URL env var (explicit override)
#   2. Load from ~/Library/Application Support/provisioning/platform/config/orchestrator.ncl
#   3. Extract server.port and construct http://localhost:PORT
# Errors if truly unavailable
def get-orchestrator-url-strict [] {
    # Priority 1: Environment variable (explicit override)
    let env_url = ($env.PROVISIONING_ORCHESTRATOR_URL? | default "")
    if ($env_url | is-not-empty) {
        return $env_url
    }

    # Priority 2: Load from platform service config
    let orch_config = (load-service-config "orchestrator")

    if ($orch_config != null) {
        # Check for explicit full URL in config
        if ($orch_config.orchestrator? != null) {
            if ($orch_config.orchestrator | get --optional "url") != null {
                let config_url = ($orch_config.orchestrator.url)
                if ($config_url | is-not-empty) {
                    return $config_url
                }
            }
        }

        # Extract port from orchestrator.server.port and construct URL
        if ($orch_config.orchestrator? != null) {
            if ($orch_config.orchestrator | get --optional "server") != null {
                if ($orch_config.orchestrator.server | get --optional "port") != null {
                    let port = ($orch_config.orchestrator.server.port)
                    return $"http://localhost:($port)"
                }
            }
        }
    }

    # No configuration found - error with guidance
    error make {
        msg: "Orchestrator URL not available. Configure via:
  1. Environment: PROVISIONING_ORCHESTRATOR_URL=http://localhost:9011
  2. User config: ~/Library/Application Support/provisioning/platform/config/orchestrator.ncl
     with structure: { orchestrator: { server: { port: 9011 } } }
  3. Command flag: --orchestrator http://localhost:9011"
    }
}

# Helper: Compress workflow for orchestrator transmission
# Combines template path, context variables, and rendered script into auditable compressed unit
def prepare_compressed_workflow_payload [] {
    # Get captured values from environment (set during template rendering)
    let template_path = ($env.LAST_TEMPLATE_PATH? | default "")
    let template_context = ($env.LAST_TEMPLATE_CONTEXT? | default {})
    let rendered_script = ($env.LAST_RENDERED_SCRIPT? | default "")

    if ($template_path | is-empty) or ($rendered_script | is-empty) {
        return null
    }

    # Compress all three as atomic unit
    compress-workflow $template_path $template_context $rendered_script
}

# > Server create
export def "main create" [
    name?: string       # Server hostname in settings
    ...args                 # Args for create command
    --infra (-i): string     # Infra directory
    --settings (-s): string # Settings path
    --outfile (-o): string  # Output file
    --serverpos (-p): int   # Server position in settings
    --check (-c)     # Only check mode no servers will be created
    --wait (-w)      # Wait servers to be created
    --select: string # Select with task as option
    --debug (-x)     # Use Debug mode
    --xm             # Debug with PROVISIONING_METADATA
    --xc             # Debuc for task and services locally PROVISIONING_DEBUG_CHECK
    --xr             # Debug for remote servers PROVISIONING_DEBUG_REMOTE
    --xld            # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug
    --metadata       # Error with metadata (-xm)
    --notitles       # not tittles
    --helpinfo (-h)  # For more details use options "help" (no dashes)
    --out: string    #  Print Output format: json, yaml, text (default)
    --orchestrated   # Use orchestrator workflow instead of direct execution
    --orchestrator: string = ""  # Orchestrator URL (empty = use config/service discovery)
] {
    if ($out | is-not-empty) {
        set-provisioning-out $out
        set-provisioning-no-terminal true
    }
    # Activate debug flags BEFORE provisioning_init
    if $debug { set-debug-enabled true }
    if $metadata { set-metadata-enabled true }
    if $xm { set-debug-enabled true; set-metadata-enabled true }
    if $xc { $env.PROVISIONING_DEBUG_CHECK = "true" }
    if $xr { $env.PROVISIONING_DEBUG_REMOTE = "true" }
    if $xld { $env.PROVISIONING_LOG_LEVEL = "debug" }
    # Convert args to list of strings for provisioning_init
    let string_args = ($args | each { $in | into string })
    provisioning_init $helpinfo "servers create" $string_args
    if $name != null and $name != "h" and $name != "help" {
        let infra_arg = if ($infra | is-empty) { null } else { $infra }
        let settings_arg = if ($settings | is-empty) { null } else { $settings }

        # Get infrastructure path (explicit or from workspace)
        let actual_infra = if ($infra_arg == null) {
            let ws_path = (get-workspace-path)
            if ($ws_path | is-empty) {
                # Workspace not found - try local detection or require explicit path
                null
            } else {
                $ws_path | path join "infra" | path join "main"
            }
        } else {
            $infra_arg
        }

        let curr_settings = (find_get_settings --infra $actual_infra --settings $settings_arg true true)

        # Guard: Check that settings loaded successfully
        if ($curr_settings == null or ($curr_settings | is-empty)) {
            _print "🛑 Failed to load settings"
            _print ""
            _print "Possible causes:"
            _print "  1. Infrastructure path not specified: use --infra <path>"
            _print "  2. No settings.ncl/main.ncl in infrastructure directory"
            _print "  3. Invalid infrastructure path"
            _print ""
            _print "Usage examples:"
            _print "  # From workspace root:"
            _print "  prvng server create --infra infra/main <server_name>"
            _print ""
            _print "  # From project root:"
            _print "  prvng server create --infra workspaces/librecloud_hetzner/infra/main <server_name>"
            _print ""
            _print "Available workspaces:"
            _print "  provisioning workspace list"
            exit 1
        }

        # Validate server name exists (skip if no servers loaded)
        let servers_list = ($curr_settings.data.servers? | default [])
        if ($servers_list | length) > 0 {
          if ($servers_list | find $name | length) == 0 {
              _print $"🛑 invalid name ($name)"
              exit 1
          }
        } else {
          # No servers loaded - proceed with check anyway for demonstration
          if $check {
            _print $"⚠️  Warning: Could not load servers from settings, proceeding with check mode anyway"
          }
        }
    }
    let task = if ($args | length) > 0 {
        ($args| get 0)
    } else {
        let str_task = (((get-provisioning-args) | str replace "create " " " ))
        let str_task = if  $name != null {
            ($str_task | str replace $name "")
        } else {
            $str_task
        }
        ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim)
    }
    let other = if ($args | length) > 0 { ($args| skip 1) } else { "" }
    let ops = $"((get-provisioning-args)) " | str replace $" ($task) " "" | str trim

    match $task {
        "" if $name == "h" => {
            ^$"(get-provisioning-name)" -mod server create help --notitles
        },
        "" if $name == "help" => {
            ^$"(get-provisioning-name)" -mod server create --help
            _print (provisioning_options "create")
        },
        "" | "c" | "create" => {
            # Guard: Validate settings before proceeding
            let infra_arg = if ($infra | is-empty) { null } else { $infra }
            let settings_arg = if ($settings | is-empty) { null } else { $settings }
            let curr_settings = (find_get_settings --infra $infra_arg --settings $settings_arg true true)
            if ($curr_settings | is-empty) or ($curr_settings.wk_path? | is-empty) {
                _print "🛑 Failed to load settings"
                _print ""
                _print "Possible causes:"
                _print "  1. No settings.yaml found in infrastructure directory"
                _print "  2. Invalid infrastructure path: use --infra /path/to/infra"
                _print "  3. No workspace configured. Use 'prvng workspace list' to see available workspaces"
                _print ""
                _print "Usage:"
                _print "  prvng server create --infra <path> <server_name>"
                exit 1
            }

            # Main logic: Create servers
            set-wk-cnprov $curr_settings.wk_path
            # Server name: null/empty = all servers, provided = only that server
            let match_name = if $name == null or $name == "" { "" } else { $name}
            let run_create = {
                on_create_servers $curr_settings $check $wait $outfile $match_name $serverpos --notitles=$notitles --orchestrator=$orchestrator
            }
            let result = desktop_run_notify $"(get-provisioning-name) servers create" "-> " $run_create --timeout 11sec
            if not ($result | get status? | default true) { exit 1 }

            # Sync .servers-state.json so server list reflects the new server immediately
            if not $check {
                let sync_infra = if ($infra | is-not-empty) { $infra | path basename } else { "" }
                let sync_ws = $curr_settings.src_path? | default ""
                if ($sync_ws | is-not-empty) and ($sync_infra | is-not-empty) {
                    _print "\n[state sync]"
                    sync-servers-state-post-op $sync_ws $sync_infra
                }
            }
        },
        _ => {
            invalid_task "servers create" $task --end
        }
    }
    if not $notitles and not (is-debug-enabled) { end_run "" }
}
export def on_create_servers [
    settings: record   # Settings record
    check: bool        # Check mode only: validate without creating
    wait: bool         # Wait for orchestrator completion
    outfile?: string   # Output file for check mode (save rendered script)
    hostname?: string  # Server hostname in settings
    serverpos?: int    # Server position in settings
    --notitles         # Don't show titles
    --orchestrator: string = ""  # Orchestrator URL (REQUIRED for production - error if unresolvable)
] {
    # CRITICAL: Verify daemon availability FIRST (before ANY output or processing)
    use ../lib_provisioning/utils/service-check.nu verify-daemon-or-block
    let daemon_check = (verify-daemon-or-block "create server")
    if $daemon_check.status == "error" {
        return {status: false, error: "provisioning_daemon not available"}
    }

    # All creation delegates to orchestrator (no fallback to local execution)
    # Orchestrator is mandatory - errors if unavailable

    use ../workflows/server_create.nu *

    # Resolve orchestrator URL - REQUIRED, NO FALLBACK
    let resolved_orchestrator = if ($orchestrator | is-not-empty) {
        $orchestrator
    } else {
        let discovered = (do { get-orchestrator-url-strict } catch { null })
        if ($discovered | is-empty) {
            _print $"\n❌ Orchestrator REQUIRED for server creation"
            _print $"   No orchestrator available via:"
            _print $"   • --orchestrator flag"
            _print $"   • service-endpoint discovery"
            _print $"   • config orchestrator.url"
            _print $"\n   Configure via:"
            _print $"   1. Environment: PROVISIONING_ORCHESTRATOR_URL"
            _print $"   2. Config: ~/.config/provisioning/config.yaml"
            _print $"   3. Service: Platform service registry"
            exit 1
        } else {
            $discovered
        }
    }

    # In check mode: validate server configuration by rendering templates
    if $check {
        let target_servers = (get-target-servers $settings $hostname $serverpos)
        mut check_failed = false

        for it in ($target_servers | enumerate) {
            if not (create_server $it.item $it.index true $wait $settings $outfile) {
                $check_failed = true
                break
            }
            _print $"\n(_ansi blue_reverse)----🌥 ----🌥 ----🌥 ---- oOo ----🌥 ----🌥 ----🌥 ---- (_ansi reset)\n"
        }

        if $check_failed {
            return { status: false, error: "Server check failed" }
        }
        return { status: true, error: "" }
    }

    # Production flow: delegate to orchestrator — one workflow per server
    let target_servers = (get-target-servers $settings $hostname $serverpos)
    let server_count = ($target_servers | length)

    # Query live servers first — needed by both bootstrap check and categorization
    let hcloud_srv_res = (do { ^hcloud server list -o json } | complete)
    let live_servers = if $hcloud_srv_res.exit_code == 0 and ($hcloud_srv_res.stdout | str trim | is-not-empty) {
        $hcloud_srv_res.stdout | from json | each {|s| $s.name}
    } else { [] }

    # Pre-flight: bootstrap validation — verify L1 resources exist before submitting
    let bootstrap_errors = (
        $target_servers | each {|srv|
            mut errs = []

            let net = ($srv.networking?.private_network? | default "")
            if ($net | is-not-empty) {
                let res = (do { ^hcloud network describe $net } | complete)
                if $res.exit_code != 0 {
                    $errs = ($errs | append $"network '($net)' not found — run: prvng bootstrap")
                }
            }

            let fw = ($srv.firewall? | default "")
            if ($fw | is-not-empty) {
                let res = (do { ^hcloud firewall describe $fw } | complete)
                if $res.exit_code != 0 {
                    $errs = ($errs | append $"firewall '($fw)' not found — run: prvng bootstrap")
                }
            }

            let fip = ($srv.floating_ip? | default "")
            let srv_exists = ($live_servers | any {|n| $n == $srv.hostname})
            if ($fip | is-not-empty) and not $srv_exists {
                let res = (do { ^hcloud floating-ip describe $fip } | complete)
                if $res.exit_code != 0 {
                    $errs = ($errs | append $"floating-ip '($fip)' not found — run: prvng bootstrap")
                }
            }

            if ($errs | is-not-empty) { { host: $srv.hostname, errors: $errs } } else { null }
        }
        | where { $in != null }
    )

    if ($bootstrap_errors | is-not-empty) {
        _print "\n❌ Bootstrap pre-flight failed:"
        for e in $bootstrap_errors {
            for msg in $e.errors { _print $"   ($e.host): ($msg)" }
        }
        _print ""
        return { status: false, error: "Bootstrap resources missing" }
    }

    # Pre-flight: categorize servers — full create / volumes-only / nothing to do

    let hcloud_vol_res = (do { ^hcloud volume list -o json } | complete)
    # Keep full volume records to check attachment state, not just names
    let live_volumes_full = if $hcloud_vol_res.exit_code == 0 and ($hcloud_vol_res.stdout | str trim | is-not-empty) {
        $hcloud_vol_res.stdout | from json
    } else { [] }
    let live_volumes = ($live_volumes_full | each {|v| $v.name})

    # Classify each server — per-volume state: new | exists_unattached | exists_attached
    let classified = ($target_servers | each {|srv|
        let srv_exists = ($live_servers | any {|n| $n == $srv.hostname})
        let declared_vols = ($srv.storage?.additional_volumes? | default [])

        let vol_states = ($declared_vols | each {|v|
            let live = ($live_volumes_full | where {|lv| $lv.name == $v.name} | first | default null)
            if $live == null {
                { vol: $v, state: "new" }          # create + format + attach + mount
            } else if ($live.server? | default null) != null {
                { vol: $v, state: "exists_attached" }  # nothing to do
            } else {
                { vol: $v, state: "exists_unattached" }  # attach + mount only — NO format
            }
        })

        let needs_work = ($vol_states | where {|vs| $vs.state != "exists_attached"} | length) > 0

        if not $srv_exists {
            { srv: $srv, mode: "full", vol_states: $vol_states }
        } else if $needs_work {
            let pending = ($vol_states | where {|vs| $vs.state != "exists_attached"} | each {|vs| $"($vs.vol.name)=($vs.state)"} | str join ', ')
            _print $"ℹ️  Server (_ansi cyan_bold)($srv.hostname)(_ansi reset) exists — pending volumes: ($pending)"
            { srv: $srv, mode: "volumes_only", vol_states: $vol_states }
        } else {
            _print $"ℹ️  Server (_ansi cyan_bold)($srv.hostname)(_ansi reset) — all volumes attached"
            { srv: $srv, mode: "skip", vol_states: $vol_states }
        }
    })

    let to_create       = ($classified | where mode == "full"          | get srv)
    let to_create_vols  = ($classified | where mode == "volumes_only"  | get srv)
    let skipped         = ($classified | where mode == "skip"          | get srv)

    # Annotate servers with per-volume state so templates can act correctly:
    #   new               → hcloud create + attach + vol-prepare (format + mount persistent)
    #   exists_unattached → hcloud attach only + mount if mount_path declared (no format)
    #   exists_attached   → nothing
    # permanent_mount (default true): adds fstab entry; false = attach without fstab
    let annotate_vols = {|srv classified_entry|
        let vols = ($srv.storage?.additional_volumes? | default [] | each {|v|
            let vs = ($classified_entry.vol_states | where {|x| $x.vol.name == $v.name} | first | default null)
            let state = if $vs != null { $vs.state } else { "new" }
            let permanent = ($v.permanent_mount? | default true)
            $v | merge { volume_state: $state, permanent_mount: $permanent }
        })
        if ($vols | is-not-empty) {
            $srv | upsert storage ($srv.storage | upsert additional_volumes $vols)
        } else { $srv }
    }

    let full_entries    = ($classified | where mode == "full")
    let vol_only_entries = ($classified | where mode == "volumes_only")

    let to_create_annotated  = ($full_entries    | each {|e| do $annotate_vols $e.srv $e})
    let to_create_vols_annotated = ($vol_only_entries | each {|e| do $annotate_vols $e.srv $e})

    if ($to_create | is-empty) and ($to_create_vols | is-empty) {
        _print "\nNothing to do — all servers and volumes already exist."
        return { status: true, error: "" }
    }

    let submit_list = ($to_create_annotated | append $to_create_vols_annotated)
    _print $"\nCreate (_ansi blue_bold)($submit_list | length)(_ansi reset) servers (_ansi blue_bold)>>> 🌥 → Orchestrator(_ansi reset)\n"
    _print $"✓ Submitting to orchestrator: (_ansi cyan)($resolved_orchestrator)(_ansi reset)"
    _print $"Servers to create:"
    $to_create | each { |srv| _print $"  - ($srv.hostname) [($srv.provider)]" }
    _print ""

    # Phase 1: Render + compress SEQUENTIALLY — tera plugin reads JSON context files
    # from disk; compress-workflow writes to /tmp and returns base64 payload immediately.
    # Both are safe to run sequentially. Each server gets its own compressed archive.
    let rendered = ($to_create | enumerate | each {|it|
        let srv = $it.item
        let render_result = (create_server $srv $it.index false $wait $settings)
        let render_ok = (
            ($render_result | describe | str starts-with "record") and
            ($render_result | get success? | default false)
        )
        let script = if $render_ok { ($render_result | get rendered_script? | default "") } else { "" }
        let tpl_path = if $render_ok { ($render_result | get template_path? | default "") } else { "" }
        let tpl_ctx = if $render_ok { ($render_result | get template_context? | default {}) } else { {} }
        let ok = ($render_ok and ($script | is-not-empty))
        let compression = if $ok {
            compress-workflow $tpl_path $tpl_ctx $script
        } else { {} }
        {
            hostname: $srv.hostname,
            compression: $compression,
            ok: $ok
        }
    })

    let render_failures = ($rendered | where ok == false)
    if ($render_failures | length) > 0 {
        $render_failures | each { |r| _print $"\n❌ Template render failed for ($r.hostname)" }
        return { status: false, error: "Template rendering failed" }
    }

    # Phase 2: Submit + wait in parallel — each closure carries its own compressed archive.
    # No shared env state. HTTP POST + polling are thread-safe.
    let results = ($rendered | par-each {|r|
        let c = $r.compression
        let wf = (on_create_servers_workflow $settings false $wait $outfile $r.hostname
            --orchestrator $resolved_orchestrator
            --script-compressed ($c | get script_compressed? | default "")
            --template-path ($c | get template_path? | default "")
            --compression-ratio ($c | get compression_ratio? | default 0.0)
            --original-size ($c | get original_size? | default 0)
            --compressed-size ($c | get compressed_size? | default 0)
        )
        if not $wf.status {
            { hostname: $r.hostname, status: "failed", task_id: "", error: ($wf.error? | default "submit failed") }
        } else {
            { hostname: $r.hostname, status: "ok", task_id: ($wf | get task_id? | default ""), error: "" }
        }
    })

    let failed = ($results | where status != "ok")
    let succeeded = ($results | where status == "ok")

    $succeeded | each { |r| _print $"  ✓ ($r.hostname) submitted" }
    $failed | each { |r| _print $"\n❌ ($r.hostname): ($r.error)" }

    if ($failed | length) > 0 {
        return { status: false, error: "One or more servers failed to submit" }
    }

    let task_ids = ($succeeded | get task_id | where { $in | is-not-empty })

    if $wait {
        _print $"\n✅ Server creation completed successfully"
        show-next-step "server_create" {infra: $settings.infra_path}
    } else {
        _print $"\n📋 Server creation workflows submitted to orchestrator"
        $task_ids | each { |tid| _print $"   (_ansi green)($tid)(_ansi reset)" }
        _print ""
        _print $"(_ansi cyan)Monitor execution:(_ansi reset)"
        $task_ids | each { |tid| _print $"   provisioning workflow status ($tid)" }
    }

    { status: true, error: "" }
}

# Helper: Get target servers based on filters
def get-target-servers [settings: record, hostname?: string, serverpos?: int] {
    let match_hostname = if $hostname != null {
        $hostname
    } else if $serverpos != null {
        let total = ($settings.data.servers | length)
        if $serverpos > 0 and $serverpos <= $total {
            ($settings.data.servers | get ($serverpos - 1)).hostname
        } else {
            null
        }
    } else {
        null
    }

    $settings.data.servers | where {|srv|
        if $match_hostname == null or $match_hostname == "" {
            true
        } else if $srv.hostname == $match_hostname {
            true
        } else {
            $srv.hostname | str starts-with $match_hostname
        }
    }
}

# Helper: Get server hostnames as list
def get-target-servers-list [settings: record, hostname?: string, serverpos?: int] {
    get-target-servers $settings $hostname $serverpos | each {|srv| $srv.hostname}
}
# Pre-flight check for servers that reference a role image.
# Returns {ok: bool, severity: string, message: string}.
# severity "stop" aborts creation; "warn" prints and continues.
def preflight_image_check [server: record]: nothing -> record {
    let role = ($server | get -o image_role | default null)
    if ($role | is-empty) { return { ok: true, severity: "", message: "" } }

    let provider = $server.provider
    let state    = (image-state-read $provider $role)

    if $state.snapshot_id == "SNAPSHOT_PENDING" {
        return {
            ok:       false,
            severity: "stop",
            message:  $"Image role '($role)' has no snapshot. Run: provisioning build image create ($role)",
        }
    }

    let fresh = (do { image-state-is-fresh $provider $role } catch { false })
    if not $fresh {
        return {
            ok:       true,
            severity: "warn",
            message:  $"Image role '($role)' snapshot ($state.snapshot_id) may be stale. Consider: provisioning build image update ($role)",
        }
    }

    { ok: true, severity: "", message: "" }
}

export def create_server [
    server: record
    index: int
    check: bool
    wait: bool
    settings: record
    outfile?: string
] {
    ## Provider middleware now available through lib_provisioning
    #use utils.nu *

    # Generate state directory with timestamp for provisioning state management
    # Format: provisioning-{cluster}-{YYYYMMDD}-{HHMMSS}
    # This is done before check mode so state_dir is available for templates
    let now_date = (date now)
    let timestamp = ($now_date | format date '%Y%m%d-%H%M%S')
    let cluster_name = (
        # Try to extract cluster name from infra path or settings
        if ($settings.data.cluster? | is-not-empty) {
            $settings.data.cluster
        } else if ($settings.infra_path | str contains "librecloud") {
            "librecloud"
        } else if ($settings.infra_path | str contains "wuji") {
            "wuji"
        } else {
            # Extract from last path component of infra path
            $settings.infra_path | path basename
        }
    )
    let state_dir = ($settings.wk_path | path join ".provisioning-tmp" | path join $"provisioning-($cluster_name)-($timestamp)")

    # Pre-flight: verify provider is declared in the server config
    if ($server.provider? | is-empty) {
        error make { msg: $"Server '($server.hostname?)' is missing required field 'provider'. Declare it explicitly in your infra servers.ncl." }
    }

    # Pre-flight: verify role image exists and is fresh before any template work
    let image_check = (preflight_image_check $server)
    if not $image_check.ok {
        _print $"🛑 ($image_check.message)"
        return false
    }
    if ($image_check.severity == "warn") {
        _print $"⚠️  ($image_check.message)"
    }

    # In check mode, show what would be created
    if $check {
        # Multi-template orchestration: Determine which templates to render
        # Template priority (execution order):
        # 1. ssh_keys  (always)
        # 2. networks  (if private_network defined)
        # 3. firewalls (always — must exist before server so attach works)
        # 4. volumes   (if volumes array not empty)
        # 5. servers   (always — creates server + attaches to firewall)

        let templates_config = [
            { name: "common_vals", priority: 0 }
            { name: "ssh_keys",    priority: 1 }
            { name: "networks",    priority: 2 }
            { name: "firewalls",   priority: 3 }
            { name: "servers",     priority: 4 }
            { name: "volumes",     priority: 5 }
        ]

        # Build template list with file paths
        let workspace_infra_path = ($settings.src_path | path dirname | path dirname)

        mut to_render = []
        for tpl in $templates_config {
            # Check if this template should be rendered
            if not (should_render_template $server $tpl.name) {
                continue
            }

            # Resolve path: workspace → system
            let template_filename = $"($server.provider)_($tpl.name).j2"
            let workspace_path = ($workspace_infra_path | path join ".providers" | path join $server.provider | path join "templates" | path join $template_filename)
            let system_path = ($env.PROVISIONING | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $template_filename)

            let template_path = if ($workspace_path | path exists) { $workspace_path } else { $system_path }

            if ($template_path | path exists) {
                $to_render = ($to_render | append { name: $tpl.name, path: $template_path, priority: $tpl.priority })
            }
        }

        # Verify critical templates exist
        if (($to_render | where name == "servers" | length) == 0) {
            _print "❌ Critical: servers template not found"
            return false
        }

        let server_template = ($to_render | where name == "servers" | first | get path)

        # Temporarily disable NO_TERMINAL to ensure check output is displayed
        let old_no_terminal = ($env.PROVISIONING_NO_TERMINAL? | default false)
        $env.PROVISIONING_NO_TERMINAL = false

        _print $"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
        _print $"Check: Create server (_ansi cyan_bold)($server.hostname)(_ansi reset) with provider (_ansi green_bold)($server.provider)(_ansi reset)"
        _print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

        if ($server_template | path exists) {
            _print $"\n📋 Template: ($server_template)"

            # Show template rendering info
            _print "\n🔧 Generated script:"
            _print "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

            # Build complete context record with all variables the template expects
            # Augment server object with default fields that template expects
            let server_with_defaults = ($server | merge {
                ssh_keys: ($server.ssh_keys? | default [])
                labels: ($server.labels? | default {})
                volumes: ($server.volumes? | default [])
                location: ($server.location? | default "nbg1")
            })

            # Load cluster-level firewalls from workspace Nickel config
            let firewalls_ncl = ($settings.infra_path | path join "firewalls.ncl")
            let firewalls = if ($firewalls_ncl | path exists) {
                ncl-eval-soft $firewalls_ncl [] [] | get -o firewalls | default []
            } else { [] }

            let template_context = {
                servers: [$server_with_defaults]
                firewalls: $firewalls
                defaults: {}
                match_server: $server.hostname
                cluster_name: $cluster_name
                state_dir: $state_dir
                provisioning_version: "1.0.4"
                now: ($now_date | format date '%Y-%m-%d %H:%M:%S')
                debug: (if ($env.PROVISIONING_DEBUG? | is-not-empty) { "yes" } else { "no" })
                use_time: "false"
                wait: false
                runset: {output_format: "yaml"}
                wk_file: ($settings.wk_path | path join "creation_script.sh")
            }

            # Capture template and context for compression/orchestrator transmission
            $env.LAST_TEMPLATE_PATH = $server_template
            $env.LAST_TEMPLATE_CONTEXT = $template_context

            # DEBUG: Save context to file for inspection
            ($template_context | to json) | save -f /tmp/tpl_context.json
            print $"ℹ️  Template context saved to /tmp/tpl_context.json"

            # Ensure tera plugin is loaded
            let tera_loaded = (plugin list | where name == "tera" | length) > 0
            if not $tera_loaded {
                (plugin use tera)
            }

            # Phase 1: Enrich template context via provider (cache management is provider's responsibility)
            let rendering_context = (mw_enrich_template_context $settings $server $template_context)

            # Render all selected templates with appropriate context
            mut sections = []
            for tpl in $to_render {
                # Build template-specific context with cached resources
                let tpl_context = (build_template_context $rendering_context $server $tpl.name)

                # Save context to temp file for this template
                let ctx_file = $"/tmp/tpl_($server.hostname)_($tpl.name)_ctx.json"
                ($tpl_context | to json) | save -f $ctx_file

                # Render template
                let absolute_template = (($tpl.path | path expand) | str trim)
                let render_result = (do {
                    let rendered = (tera-render $absolute_template $ctx_file)
                    {success: true, content: $rendered, error: null}
                } catch { |e|
                    {success: false, content: null, error: $"Error rendering ($tpl.name): $($e)"}
                })

                if not $render_result.success {
                    print $"❌ ($render_result.error)"
                    $env.PROVISIONING_NO_TERMINAL = $old_no_terminal
                    exit 1
                }

                # Collect rendered section
                $sections = ($sections | append {
                    name: $tpl.name,
                    content: $render_result.content,
                    priority: $tpl.priority
                })
            }

            # Concatenate all sections into single atomic script
            let final_script = (concatenate_script_sections $sections)

            # Capture rendered script for compression/orchestrator transmission
            $env.LAST_RENDERED_SCRIPT = $final_script

            # Handle outfile parameter: save to file if provided, otherwise print to stdout
            let has_outfile = ($outfile != null and ($outfile | str length) > 0)
            if $has_outfile {
                # Expand the outfile path to absolute
                let absolute_outfile = ($outfile | path expand)
                # Create parent directories if they don't exist
                let outfile_dir = ($absolute_outfile | path dirname)
                if not ($outfile_dir | path exists) {
                    ^mkdir -p $outfile_dir
                }
                # Write rendered content to file
                $final_script | save --force $absolute_outfile
                print $"✅ Script saved to: ($absolute_outfile)"
                print $"   State directory: ($state_dir)"
            } else {
                # Pipe through bat for syntax highlighting and paging
                let bat_available = (which bat | is-not-empty)
                if $bat_available {
                    $final_script | ^bat --language bash --style plain --paging auto
                } else {
                    # Fallback to plain print if bat not available
                    print $final_script
                }
            }

            print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

            print $"\n✅ Check completed successfully"
            print $"   Server configuration:"
            print $"   • Hostname: ($server.hostname? | default '')"
            print $"   • Provider: ($server.provider)"
            print $"   • Type: ($server.server_type?| default '')"
            print $"   • Location: ($server.location? | default '')"
            print $"   • Cluster: ($cluster_name | default '')"

            # Show what's included in the atomic script
            print "\n📋 Atomic script includes:"
            print "   ✓ Server creation"
            print "   ✓ Firewall setup:"
            #print "     - SSH (TCP 22) from 0.0.0.0/0 and ::/0"
            #print "     - ICMP from 0.0.0.0/0 and ::/0"
            #print "     - Outbound TCP, UDP, ICMP to anywhere"
            print "   ✓ Idempotent checks (safe to retry)"

            print ""
            print "   (Check mode - nothing executed)"
            print ""
            print "   Next steps:"
            print ("   ▶ Execute locally:  provisioning create server " + $server.hostname + " --infra " + $settings.infra_path)
            print ("   ▶ Save script:      provisioning create server " + $server.hostname + " --infra " + $settings.infra_path + " --outfile ~/provisioning-script.sh")
            print ("   ▶ Via orchestrator: provisioning create server " + $server.hostname + " --infra " + $settings.infra_path + " --orchestrated")
            print ""
            print "   Note: Orchestrator receives metadata (infra, settings), then regenerates and executes script"

            # Restore original NO_TERMINAL setting and exit immediately in check mode
            # Exit directly to avoid any cleanup code that might hang with bat/pager
            $env.PROVISIONING_NO_TERMINAL = $old_no_terminal
            exit 0
        } else {
            _print $"\n⚠️  Template not found: ($server_template)"
            $env.PROVISIONING_NO_TERMINAL = $old_no_terminal
            return false
        }
    }

    # PRODUCTION MODE: Render template first (before any server checks)
    # In production, we MUST capture the script for orchestrator transmission
    if not $check {
        # Production flow: render template immediately
    } else {
        # Check mode already handled above (line 426)
        # If we reach here in check mode, something is wrong
        _print "🛑 Unexpected state: check mode not handled"
        return false
    }

    # Production mode: Multi-template orchestration (same as check mode)
    # Build template list with file paths
    let templates_config = [
        { name: "common_vals", priority: 0 }  # shebang + STATE_DIR + set -euo pipefail
        { name: "ssh_keys",    priority: 1 }
        { name: "networks",    priority: 2 }
        { name: "firewalls",   priority: 3 }
        { name: "servers",     priority: 4 }
        { name: "volumes",     priority: 5 }
    ]

    let workspace_infra_path = ($settings.src_path | path dirname | path dirname)

    mut to_render = []
    for tpl in $templates_config {
        # Check if this template should be rendered
        if not (should_render_template $server $tpl.name) {
            continue
        }

        # Resolve path: workspace → system
        let template_filename = $"($server.provider)_($tpl.name).j2"
        let workspace_path = ($workspace_infra_path | path join ".providers" | path join $server.provider | path join "templates" | path join $template_filename)
        let system_path = ($env.PROVISIONING | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $template_filename)

        let template_path = if ($workspace_path | path exists) { $workspace_path } else { $system_path }

        if ($template_path | path exists) {
            $to_render = ($to_render | append { name: $tpl.name, path: $template_path, priority: $tpl.priority })
        }
    }

    # Verify critical templates exist
    if (($to_render | where name == "servers" | length) == 0) {
        _print "❌ Critical: servers template not found"
        return false
    }

    # Build template context (same as check mode)
    let now_date = (date now)
    let cluster_name = (
        if ($settings.data.cluster? | is-not-empty) {
            $settings.data.cluster
        } else if ($settings.infra_path | str contains "librecloud") {
            "librecloud"
        } else if ($settings.infra_path | str contains "wuji") {
            "wuji"
        } else {
            $settings.infra_path | path basename
        }
    )
    let timestamp = ($now_date | format date '%Y%m%d-%H%M%S')
    let state_dir = ($settings.wk_path | path join ".provisioning-tmp" | path join $"provisioning-($cluster_name)-($timestamp)")

    let server_with_defaults = ($server | merge {
        ssh_keys: ($server.ssh_keys? | default [])
        labels: ($server.labels? | default {})
        volumes: ($server.volumes? | default [])
        location: ($server.location? | default "nbg1")
    })

    let template_context = {
        servers: [$server_with_defaults]
        defaults: {}
        match_server: $server.hostname
        cluster_name: $cluster_name
        state_dir: $state_dir
        provisioning_version: "1.0.4"
        now: ($now_date | format date '%Y-%m-%d %H:%M:%S')
        debug: (if ($env.PROVISIONING_DEBUG? | is-not-empty) { "yes" } else { "no" })
        use_time: "false"
        wait: false
        runset: {output_format: "yaml"}
        wk_file: ($settings.wk_path | path join "creation_script.sh")
    }

    # Ensure tera plugin is loaded
    let tera_loaded = (plugin list | where name == "tera" | length) > 0
    if not $tera_loaded {
        (plugin use tera)
    }

    # Render all selected templates
    mut sections = []
    for tpl in $to_render {
        # Build template-specific context
        let tpl_context = (build_template_context $template_context $server $tpl.name)

        # Save context to temp file — include hostname to avoid races in par-each
        let ctx_file = $"/tmp/tpl_prod_($server.hostname)_($tpl.name)_ctx.json"
        ($tpl_context | to json) | save -f $ctx_file

        # Render template
        let absolute_template = (($tpl.path | path expand) | str trim)
        let render_result = (do {
            let rendered = (tera-render $absolute_template $ctx_file)
            {success: true, content: $rendered, error: null}
        } catch { |e|
            {success: false, content: null, error: $"Error rendering ($tpl.name): $($e)"}
        })

        if not $render_result.success {
            _print $"❌ ($render_result.error)"
            return false
        }

        # Collect rendered section
        $sections = ($sections | append {
            name: $tpl.name,
            content: $render_result.content,
            priority: $tpl.priority
        })
    }

    # Concatenate all sections into single atomic script
    let final_script = (concatenate_script_sections $sections)

    if ($final_script | is-empty) or ($final_script | str length) == 0 {
        _print $"❌ Template rendering failed: empty output"
        return false
    }

    # Capture for compression/orchestrator transmission
    $env.LAST_TEMPLATE_PATH = ($to_render | first | get path)
    $env.LAST_TEMPLATE_CONTEXT = $template_context
    $env.LAST_RENDERED_SCRIPT = $final_script

    # Return both success and rendered script for orchestrator
    {
        success: true,
        rendered_script: $final_script,
        template_path: ($to_render | first | get path),
        template_context: $template_context
    }
}

export def verify_server_info [
    settings: record
    server: record
    info: record
] {
    _print $"Checking server (_ansi green_bold)($server.hostname)(_ansi reset) info "
    let server_plan = ($server | get plan? | default "")
    let curr_plan = ($info | get plan? | default "")
    if ($server_plan | is-not-empty) {
        if $server_plan != $curr_plan {
            mw_modify_server $settings $server [{plan: $server_plan}] false
        }
    }
}
export def check_server [
    settings: record
    server: record
    index: int
    info: record
    check: bool
    wait: bool
    settings: record
    outfile?: string
] {
    ## Provider middleware now available through lib_provisioning
    #use utils.nu *
    let server_info = if ($info | is-empty) {
        (mw_server_info $server true)
    } else {
        $info
    }
    let already_created = ($server_info | is-not-empty)
    if not $already_created {
        _print $"🛑 server (_ansi green_bold)($server.hostname)(_ansi reset) not exists"
        return false
    }
    if not $check {
        ^ssh-keygen -f $"($env.HOME)/.ssh/known_hosts" -R $server.hostname err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })
        let ip_raw = (mw_get_ip $settings $server $server.liveness_ip false )
        let ip = ($ip_raw | str trim --char "\"")
        if $ip == "" {
            _print "🛑 No liveness ip found for state checking "
            return false
        }
        verify_server_info $settings $server $server_info
        _print $"liveness (_ansi purple)($ip):($server.liveness_port)(_ansi reset)"
        if (wait_for_server $index $server $settings $ip) {
            # Check if SSH setup succeeded (returns false on CTRL-C during sudo)
            let ssh_result = (on_server_ssh $settings $server "pub" "create" false $check)
            if not $ssh_result {
                _print $"\n(_ansi red)✗ Server creation cancelled(_ansi reset)"
                return false
            }
            # collect fingerprint
            let res = (^ssh-keyscan "-H" $ip err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })| complete)
            if $res.exit_code == 0 {
                let known_hosts_path = (("~" | path join ".ssh" | path join "known_hosts") | path expand)
                let markup = $"# ($ip) keyscan"
                let lines_found = (open $known_hosts_path --raw | lines |  find $markup | length)
                if $lines_found == 0 {
                    ( $"($markup)\n" | save --append $known_hosts_path)
                    ($res.stdout | save --append $known_hosts_path)
                    _print $"(_ansi green_bold)($ip)(_ansi reset) (_ansi yellow)ssh-keyscan(_ansi reset) added to ($known_hosts_path)"
                }
            #} else {
            #   _print $"🛑 Error (_ansi yellow)ssh-keyscan(_ansi reset) from ($ip)"
            #   _print $"($res.stdout)"
            }
            if $already_created {
                let res = (mw_post_create_server $settings $server $check)
                match $res {
                    "error" | "-1" => { exit 1},
                    "storage" | "" => {
                        let storage_sh = ($settings.wk_path | path join $"($server.hostname)-storage.sh")
                        let result = (on_server_template (get-templates-path | path join "storage.j2") $server 0 true true true $settings $storage_sh)
                        if $result and ($storage_sh | path exists) and (wait_for_server $index $server $settings $ip) {
                            let target_cmd = "/tmp/storage.sh"
                            #use ssh.nu scp_to ssh_cmd
                            if not (scp_to $settings $server [$storage_sh] $target_cmd $ip) { return false }
                            _print $"Running (_ansi blue_italic)($target_cmd | path basename)(_ansi reset) in (_ansi green_bold)($server.hostname)(_ansi reset)"
                            if not (ssh_cmd $settings $server true $target_cmd $ip) { return false }
                            if (is-ssh-debug-enabled) { return true }
                            if not (is-debug-enabled) {
                                (ssh_cmd $settings $server false $"rm -f ($target_cmd)" $ip)
                            }
                        } else {
                            return false
                        }
                    }
                    _  => {
                        return true
                    },
                }
            }
        }
    }
    true
}