diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6cc555d --- /dev/null +++ b/.gitignore @@ -0,0 +1,112 @@ +.p +.claude +.vscode +.shellcheckrc +.coder +.migration +.zed +ai_demo.nu +CLAUDE.md +.cache +.coder +wrks +ROOT +OLD +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ +# Encryption keys and related files (CRITICAL - NEVER COMMIT) +.k +.k.backup +*.k +*.key.backup + +config.*.toml +config.*back + +# where book is written +_book + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +node_modules/ + +**/output.css +**/input.css + +# Environment files +.env +.env.local +.env.production +.env.development +.env.staging + +# Keep example files +!.env.example + +# Configuration files (may contain sensitive data) +config.prod.toml +config.production.toml +config.local.toml +config.*.local.toml + +# Keep example configuration files +!config.toml +!config.dev.toml +!config.example.toml + +# Log files +logs/ +*.log + +# TLS certificates and keys +certs/ +*.pem +*.crt +*.key +*.p12 +*.pfx + +# Database files +*.db +*.sqlite +*.sqlite3 + +# Backup files +*.bak +*.backup +*.tmp +*~ + +# Encryption and security related files +*.encrypted +*.enc +secrets/ +private/ +security/ + +# Configuration backups that may contain secrets +config.*.backup +config.backup.* + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db +# Documentation build output +book-output/ +# Generated setup report +SETUP_COMPLETE.md diff --git a/README.md b/README.md index 4b869ca..90e39c5 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,16 @@ # Core Engine -The **Core Engine** is the foundation of the Provisioning platform—a modular, high-performance infrastructure automation system built on **Nushell** and **KCL**. It provides unified CLI tools, core libraries, and extensible architecture for managing cloud infrastructure, Kubernetes clusters, and infrastructure-as-code workflows. +The **Core Engine** is the foundational component of the [Provisioning project](PRIOVISIONING.md), providing the unified CLI interface, core Nushell libraries, and essential utility scripts. Built on **Nushell** and **KCL**, it serves as the primary entry point for all infrastructure operations. ## Overview -The Core Engine serves as the central orchestration layer, providing: +The Core Engine provides: -- **Unified CLI Interface** - Single command-line interface for all infrastructure operations -- **Core Libraries** - Reusable Nushell modules for configuration, validation, deployment, and workflow management +- **Unified CLI Interface** - Single command-line tool for all infrastructure operations +- **Core Libraries** - Reusable Nushell modules for configuration, validation, and utilities - **Provider Abstraction** - Cloud-agnostic interface supporting UpCloud, AWS, and local providers -- **Workflow Orchestration** - Batch operations, dependency resolution, and state management +- **Workflow Integration** - Commands for submitting and managing workflows (executed by the orchestrator) - **Configuration System** - Hierarchical, config-driven architecture with 476+ configuration accessors ## Project Structure diff --git a/cli/.gitkeep b/cli/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cli/cfssl-install.sh b/cli/cfssl-install.sh new file mode 100755 index 0000000..f2740e3 --- /dev/null +++ b/cli/cfssl-install.sh @@ -0,0 +1,17 @@ +#!/bin/bash +VERSION="1.6.4" + +# shellcheck disable=SC2006 +OS=$(uname | tr '[:upper:]' '[:lower:]') +ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" + +wget https://github.com/cloudflare/cfssl/releases/download/v${VERSION}/cfssl_${VERSION}_${OS}_${ARCH} +if [ -r "cfssl_${VERSION}_${OS}_${ARCH}" ] ; then + chmod +x "cfssl_${VERSION}_${OS}_${ARCH}" + sudo mv "cfssl_${VERSION}_${OS}_${ARCH}" /usr/local/bin/cfssl +fi +wget https://github.com/cloudflare/cfssl/releases/download/v${VERSION}/cfssljson_${VERSION}_${OS}_${ARCH} +if [ -r "cfssljson_${VERSION}_${OS}_${ARCH}" ] ; then + chmod +x "cfssljson_${VERSION}_${OS}_${ARCH}" + sudo mv "cfssljson_${VERSION}_${OS}_${ARCH}" /usr/local/bin/cfssljson +fi diff --git a/cli/install_config.sh b/cli/install_config.sh new file mode 100755 index 0000000..2570ecb --- /dev/null +++ b/cli/install_config.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Info: Script to install Provisioning config +# Author: JesusPerezLorenzo +# Release: 1.0.4 +# Date: 15-04-2024 + + +NU_FILES=" +core/nulib/libremote.nu +core/nulib/lib_provisioning/setup/config.nu +" + +WK_FILE=/tmp/make_config_provisioning.nu + +[ -r "$WK_FILE" ] && rm -f "$WK_FILE" + +set -o allexport +## shellcheck disable=SC1090 +[ -n "$PROVISIONING_ENV" ] && [ -r "$PROVISIONING_ENV" ] && source "$PROVISIONING_ENV" +set +o allexport + +export NU=$(type -P nu) +[ -z "$NU" ] && echo "Nu shell not found" && exit 1 + +export PROVISIONING=${PROVISIONING:-/usr/local/provisioning} +export PROVISIONING_DEBUG=false + +for it in $NU_FILES +do + [ -r "$PROVISIONING/$it" ] && cat $PROVISIONING/$it >> $WK_FILE +done + +echo " +install_config \"reset\" --context +" >> $WK_FILE + +NU_ARGS="" +CMD_ARGS="" + +DEFAULT_CONTEXT_TEMPLATE="default_context.yaml" +case "$(uname | tr '[:upper:]' '[:lower:]')" in + linux) PROVISIONING_USER_CONFIG="$HOME/.config/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/.config/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; + darwin) PROVISIONING_USER_CONFIG="$HOME/Library/Application\ Support/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/Library/Application\ Support/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; + *) PROVISIONING_USER_CONFIG="$HOME/.config/provisioning/nushell" + PROVISIONING_CONTEXT_PATH="$HOME/.config/provisioning/$DEFAULT_CONTEXT_TEMPLATE" + ;; +esac + +[ -d "$PROVISIONING_USER_CONFIG" ] && rm -r "$PROVISIONING_USER_CONFIG" +[ -r "$PROVISIONING_CONTEXT_PATH" ] && rm -f "$PROVISIONING_CONTEXT_PATH" + +nu $NU_ARGS $WK_FILE $CMD_ARGS + +rm -f $WK_FILE diff --git a/cli/install_nu.sh b/cli/install_nu.sh new file mode 100755 index 0000000..6b0b817 --- /dev/null +++ b/cli/install_nu.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Info: Script to instal NUSHELL for Provisioning +# Author: JesusPerezLorenzo +# Release: 1.0.5 +# Date: 8-03-2024 + +test_runner() { + echo -e "\nTest installation ... " + RUNNER_PATH=$(type -P $RUNNER) + [ -z "$RUNNER_PATH" ] && echo "🛑 Error $RUNNER not found in PATH ! " && exit 1 + if $RUNNER ; then + echo -e "\n✅ Installation completed successfully ! Use \"$RUNNER\"" + else + echo -e "\n🛑 Error $RUNNER ! Review installation " && exit 1 + fi +} +register_plugins() { + local source=$1 + local warn=$2 + [ ! -d "$source" ] && echo "🛑 Error path $source is not a directory" && exit 1 + [ -z "$(ls $source/nu_plugin_* 2> /dev/null)" ] && echo "🛑 Error no 'nu_plugin_*' found in $source to register" && exit 1 + echo -e "Nushell $NU_VERSION plugins registration \n" + if [ -n "$warn" ] ; then + echo -e $"❗Warning: Be sure Nushell plugins are compiled for same Nushell version $NU_VERSION\n otherwise will probably not work and will break installation !\n" + fi + for plugin in ${source}/nu_plugin_* + do + if $source/nu -c "register \"${plugin}\" " 2>/dev/null ; then + echo -en "$(basename $plugin)" + if [[ "$plugin" == *_notifications ]] ; then + echo -e " registred " + else + echo -e "\t\t registred " + fi + fi + done + + # Install nu_plugin_tera if available + if command -v cargo >/dev/null 2>&1; then + echo -e "Installing nu_plugin_tera..." + if cargo install nu_plugin_tera; then + if $source/nu -c "register ~/.cargo/bin/nu_plugin_tera" 2>/dev/null; then + echo -e "nu_plugin_tera\t\t registred" + else + echo -e "❗ Failed to register nu_plugin_tera" + fi + else + echo -e "❗ Failed to install nu_plugin_tera" + fi + + # Install nu_plugin_kcl if available + echo -e "Installing nu_plugin_kcl..." + if cargo install nu_plugin_kcl; then + if $source/nu -c "register ~/.cargo/bin/nu_plugin_kcl" 2>/dev/null; then + echo -e "nu_plugin_kcl\t\t registred" + else + echo -e "❗ Failed to register nu_plugin_kcl" + fi + else + echo -e "❗ Failed to install nu_plugin_kcl" + fi + else + echo -e "❗ Cargo not found - nu_plugin_tera and nu_plugin_kcl not installed" + fi +} + +install_mode() { + local mode=$1 + case "$mode" in + ui| desktop) + if cp $PROVISIONING_MODELS_SRC/plugins_defs.nu $PROVISIONING_MODELS_TARGET/plugins_defs.nu ; then + echo "Mode $mode installed" + fi + ;; + *) + NC_PATH=$(type -P nc) + if [ -z "$NC_PATH" ] ; then + echo "'nc' command not found in PATH. Install 'nc' (netcat) command." + exit 1 + fi + if cp $PROVISIONING_MODELS_SRC/no_plugins_defs.nu $PROVISIONING_MODELS_TARGET/plugins_defs.nu ; then + echo "Mode 'no plugins' installed" + fi + esac +} +install_from_url() { + local target_path=$1 + local lib_mode + local url_source + local download_path + local download_url + local tar_file + + [ ! -d "$target_path" ] && echo "🛑 Error path $target_path is not a directory" && exit 1 + lib_mode=$(grep NU_LIB $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') + url_source=$(grep NU_SOURCE $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') + download_path="nu-${NU_VERSION}-${ARCH_ORG}-${OS}" + case "$OS" in + linux) download_path="nu-${NU_VERSION}-${ARCH_ORG}-unknown-${OS}-gnu" + ;; + esac + download_url="$url_source/${NU_VERSION}/$download_path.tar.gz" + tar_file=$download_path.tar.gz + echo -e "Nushell $NU_VERSION downloading ..." + if ! curl -sSfL $download_url -o $tar_file ; then + echo "🛑 Error download $download_url " && exit 1 + return 1 + fi + echo -e "Nushell $NU_VERSION extracting ..." + if ! tar xzf $tar_file ; then + echo "🛑 Error download $download_url " && exit 1 + return 1 + fi + rm -f $tar_file + if [ ! -d "$download_path" ] ; then + echo "🛑 Error $download_path not found " && exit 1 + return 1 + fi + echo -e "Nushell $NU_VERSION installing ..." + if [ -r "$download_path/nu" ] ; then + chmod +x $download_path/nu + if ! sudo cp $download_path/nu $target_path ; then + echo "🛑 Error installing \"nu\" in $target_path" + rm -rf $download_path + return 1 + fi + fi + rm -rf $download_path + echo "✅ Nushell and installed in $target_path" + [[ ! "$PATH" =~ $target_path ]] && echo "❗ Warning: \"$target_path\" is not in your PATH for $(basename $SHELL) ! Fix your PATH settings " + echo "" + # TDOO install plguins via cargo ?? + # TODO a NU version without PLUGINS + # register_plugins $target_path +} + +install_from_local() { + local source=$1 + local target=$2 + local tmpdir + + [ ! -d "$target" ] && echo "🛑 Error path $target is not a directory" && exit 1 + [ ! -r "$source/nu.gz" ] && echo "🛑 Error command 'nu' not found in $source/nu.gz" && exit 1 + + echo -e "Nushell $NU_VERSION self installation guarantees consistency with plugins and settings \n" + tmpdir=$(mktemp -d) + cp $source/*gz $tmpdir + for file in $tmpdir/*gz ; do gunzip $file ; done + if ! sudo mv $tmpdir/* $target ; then + echo -e "🛑 Errors to install Nushell and plugins in \"${target}\"" + rm -rf $tmpdir + return 1 + fi + rm -rf $tmpdir + echo "✅ Nushell and plugins installed in $target" + [[ ! "$PATH" =~ $target ]] && echo "❗ Warning: \"$target\" is not in your PATH for $(basename $SHELL) ! Fix your PATH settings " + echo "" + register_plugins $target +} + +message_install() { + local ask=$1 + local msg + local answer + [ -r "$PROVISIONING/resources/ascii.txt" ] && cat "$PROVISIONING/resources/ascii.txt" && echo "" + if [ -z "$NU" ] ; then + echo -e "🛑 Nushell $NU_VERSION not installed is mandatory for \"${RUNNER}\"" + echo -e "Check PATH or https://www.nushell.sh/book/installation.html with version $NU_VERSION" + else + echo -e "Nushell $NU_VERSION update for \"${RUNNER}\"" + fi + echo "" + if [ -n "$ask" ] && [ -d "$(dirname $0)/nu/${ARCH}-${OS}" ] ; then + echo -en "Install Nushell $(uname -m) $(uname) in \"$INSTALL_PATH\" now (yes/no) ? : " + read -r answer + if [ "$answer" != "yes" ] && [ "$answer" != "y" ] ; then + return 1 + fi + fi + if [ -d "$(dirname $0)/nu/${ARCH}-${OS}" ] ; then + install_from_local $(dirname $0)/nu/${ARCH}-${OS} $INSTALL_PATH + install_mode "ui" + else + install_from_url $INSTALL_PATH + install_mode "" + fi +} + +set +o errexit +set +o pipefail + +RUNNER="provisioning" +export NU=$(type -P nu) + +[ -n "$PROVISIONING_ENV" ] && [ -r "$PROVISIONING_ENV" ] && source "$PROVISIONING_ENV" +[ -r "../env-provisioning" ] && source ../env-provisioning +[ -r "env-provisioning" ] && source ./env-provisioning +#[ -r ".env" ] && source .env set +set +o allexport + +if [ -n "$1" ] && [ -d "$1" ] && [ -d "$1/core" ] ; then + export PROVISIONING=$1 +else + export PROVISIONING=${PROVISIONING:-/usr/local/provisioning} +fi + +TASK=${1:-check} +shift +if [ "$TASK" == "mode" ] && [ -n "$1" ] ; then + INSTALL_MODE=$1 + shift +else + INSTALL_MODE="ui" +fi + +ASK_MESSAGE="ask" +[ -n "$1" ] && [ "$1" == "no-ask" ] && ASK_MESSAGE="" && shift +[ -n "$1" ] && [ "$1" == "mode-ui" ] && INSTALL_MODE="ui" && shift +[ -n "$1" ] && [[ "$1" == mode-* ]] && INSTALL_MODE="" && shift + +INSTALL_PATH=${1:-/usr/local/bin} + +NU_VERSION=$(grep NU_VERSION $PROVISIONING/core/versions | cut -f2 -d"=" | sed 's/"//g') +#ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" +ARCH="$(uname -m | sed -e 's/amd64/x86_64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" +ARCH_ORG="$(uname -m | tr '[:upper:]' '[:lower:]')" +OS="$(uname | tr '[:upper:]' '[:lower:]')" + +PROVISIONING_MODELS_SRC=$PROVISIONING/core/nulib/models +PROVISIONING_MODELS_TARGET=$PROVISIONING/core/nulib/lib_provisioning + +USAGE="$(basename $0) [install | reinstall | mode | check] no-ask mode-?? " +case $TASK in + install) + message_install $ASK_MESSAGE + ;; + reinstall | update) + INSTALL_PATH=$(dirname $NU) + if message_install ; then + test_runner + fi + ;; + mode) + install_mode $INSTALL_MODE + ;; + check) + $PROVISIONING/core/bin/tools-install check nu + ;; + help|-h) + echo "$USAGE" + ;; + *) echo "Option $TASK not defined" +esac diff --git a/cli/module-loader b/cli/module-loader new file mode 100755 index 0000000..316c79c --- /dev/null +++ b/cli/module-loader @@ -0,0 +1,981 @@ +#!/usr/bin/env nu + +# Enhanced Module Loader CLI +# Unified CLI for discovering and loading taskservs, providers, and clusters +# Includes template and layer support from enhanced version + +use ../nulib/taskservs/discover.nu * +use ../nulib/taskservs/load.nu * +use ../nulib/providers/discover.nu * +use ../nulib/providers/load.nu * +use ../nulib/clusters/discover.nu * +use ../nulib/clusters/load.nu * +use ../nulib/lib_provisioning/kcl_module_loader.nu * +use ../nulib/lib_provisioning/config/accessor.nu config-get + +# Main module loader command with enhanced features +def main [subcommand?: string] { + if ($subcommand | is-empty) { + print_enhanced_help + return + } + + match $subcommand { + "help" => print_enhanced_help + "discover" => print_discover_help + "load" => print_load_help + "list" => print_list_help + "unload" => print_unload_help + "init" => print_init_help + "validate" => print_validate_help + "info" => print_info_help + "template" => print_template_help + "layer" => print_layer_help + "override" => print_override_help + _ => { + print $"Unknown command: ($subcommand)" + print_enhanced_help + } + } +} + +# === DISCOVERY COMMANDS === + +# Discover available modules +export def "main discover" [ + type: string, # Module type: taskservs, providers, clusters + query?: string, # Search query + --format: string = "table", # Output format: table, yaml, json, names + --category: string = "", # Filter by category (for taskservs) + --group: string = "" # Filter by group (for taskservs) +] { + match $type { + "taskservs" => { + let taskservs = if ($query | is-empty) { + discover-taskservs + } else { + search-taskservs $query + } + + let filtered = if ($category | is-empty) and ($group | is-empty) { + $taskservs + } else if not ($category | is-empty) { + $taskservs | where group == $category + } else if not ($group | is-empty) { + $taskservs | where group == $group + } else { + $taskservs + } + + format_output $filtered $format + } + "providers" => { + print "Provider discovery not implemented yet" + } + "clusters" => { + print "Cluster discovery not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + print "Available types: taskservs, providers, clusters" + } + } +} + +# Sync KCL dependencies for infrastructure workspace +export def "main sync-kcl" [ + infra: string, # Infrastructure name or path + --manifest: string = "providers.manifest.yaml", # Manifest file name + --kcl # Show KCL module info after sync +] { + # Resolve infrastructure path + let infra_path = if ($infra | path exists) { + $infra + } else { + # Try workspace path + let workspace_path = $"workspace/infra/($infra)" + if ($workspace_path | path exists) { + $workspace_path + } else { + print $"❌ Infrastructure not found: ($infra)" + return + } + } + + # Sync KCL dependencies using library function + sync-kcl-dependencies $infra_path --manifest $manifest + + # Show KCL module info if requested + if $kcl { + print "" + print "📋 KCL Modules:" + let modules_dir = (get-config-value "kcl" "modules_dir") + let modules_path = ($infra_path | path join $modules_dir) + + if ($modules_path | path exists) { + ls $modules_path | each {|entry| + print $" • ($entry.name | path basename) → ($entry.name)" + } + } + } +} + +# === LOAD/UNLOAD COMMANDS === + +# Load modules into workspace +export def "main load" [ + type: string, # Module type: taskservs, providers, clusters + workspace: string, # Workspace path + ...modules: string, # Module names to load + --layer: string = "workspace", # Layer to load into: workspace, infra + --validate # Validate after loading + --force (-f) # Force overwrite existing files +] { + if ($modules | is-empty) { + print $"No modules specified for loading" + return + } + + print $"Loading ($modules | length) ($type) into ($workspace) at layer ($layer)" + + match $type { + "taskservs" | "providers" | "clusters" | "workflows" => { + load_extension_to_workspace $type $workspace $modules $layer $force + } + _ => { + print $"Unknown module type: ($type)" + } + } + + if $validate { + main validate $workspace + } +} + +# Enhanced load with template support +export def "main load enhanced" [ + type: string, # Module type + workspace: string, # Workspace path + infra: string, # Infrastructure name + modules: list, # Module names + --layer: string = "workspace", # Target layer + --template-base # Use template as base +] { + print $"🚀 Enhanced loading ($modules | length) ($type) for infra ($infra)" + + for module in $modules { + print $" 📦 Loading ($module)..." + + # Check if template exists for this module + let template_path = $"provisioning/workspace/templates/taskservs/*/($module).k" + let has_template = (glob $template_path | length) > 0 + + if $has_template and $template_base { + print $" ✓ Using template base for ($module)" + # Template-based loading would go here + } else { + print $" ✓ Direct loading for ($module)" + # Direct loading + } + } + + print "✅ Enhanced loading completed" +} + +# Unload module from workspace +export def "main unload" [ + type: string, # Module type + workspace: string, # Workspace path + module: string, # Module name to unload + --layer: string = "workspace" # Layer to unload from +] { + print $"Unloading ($module) from ($workspace) at layer ($layer)" + + match $type { + "taskservs" => { + unload_taskserv_from_workspace $workspace $module $layer + } + "providers" => { + print "Provider unloading not implemented yet" + } + "clusters" => { + print "Cluster unloading not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + } + } +} + +# === LIST COMMANDS === + +# List modules in workspace +export def "main list" [ + type: string, # Module type + workspace: string, # Workspace path + --layer: string = "all", # Layer to list: workspace, infra, all + --format: string = "table" # Output format +] { + print $"Listing ($type) in ($workspace) for layer ($layer)" + + match $type { + "taskservs" => { + list_workspace_taskservs $workspace $layer $format + } + "providers" => { + print "Provider listing not implemented yet" + } + "clusters" => { + print "Cluster listing not implemented yet" + } + _ => { + print $"Unknown module type: ($type)" + } + } +} + +# === TEMPLATE COMMANDS === + +# List available templates +export def "main template list" [ + --template-type: string = "all", # Template type: taskservs, providers, servers, clusters + --format: string = "table" # Output format +] { + print $"📋 Available templates type: ($template_type)" + + let template_base = "provisioning/workspace/templates" + + match $template_type { + "taskservs" | "all" => { + let taskserv_templates = if (($template_base | path join "taskservs") | path exists) { + glob ($template_base | path join "taskservs" "*" "*.k") + | each { |path| + let category = ($path | path dirname | path basename) + let name = ($path | path basename | str replace ".k" "") + { type: "taskserv", category: $category, name: $name, path: $path } + } + } else { [] } + + format_output $taskserv_templates $format + } + "providers" => { + print "Provider templates not implemented yet" + } + "servers" => { + let server_templates = if (($template_base | path join "servers") | path exists) { + ls ($template_base | path join "servers") | get name + | each { |path| { type: "server", name: ($path | path basename), path: $path } } + } else { [] } + + format_output $server_templates $format + } + _ => { + print $"Unknown template type: ($template_type)" + } + } +} + +# Extract template from existing infrastructure +export def "main template extract" [ + source_infra: string, # Source infrastructure path + template_name: string, # Name for the new template + --type: string = "taskserv", # Template type + --output: string = "provisioning/workspace/templates" # Output directory +] { + print $"📤 Extracting template ($template_name) from ($source_infra)" + + # Implementation would analyze the source infra and create template + print "Template extraction not yet implemented" +} + +# Apply template to infrastructure +export def "main template apply" [ + template_name: string, # Template to apply + target_infra: string, # Target infrastructure + --override-file: string = "", # Override file path + --dry-run # Show what would be done +] { + if $dry_run { + print $"🔍 [DRY RUN] Would apply template ($template_name) to ($target_infra)" + } else { + print $"📥 Applying template ($template_name) to ($target_infra)" + } + + # Implementation would apply template with overrides + print "Template application not yet implemented" +} + +# === LAYER COMMANDS === + +# Show layer information +export def "main layer show" [ + workspace: string, # Workspace path + --module: string = "", # Specific module to show + --type: string = "taskservs" # Module type +] { + print $"📊 Layer information for ($workspace)" + + if not ($module | is-empty) { + # Use existing layer utilities + try { + nu -c $"use provisioning/workspace/tools/layer-utils.nu *; test_layer_resolution ($module) ($workspace) upcloud" + } catch { + print $"Could not test layer resolution for ($module)" + } + } else { + print "Showing overall layer structure..." + try { + nu -c "use provisioning/workspace/tools/layer-utils.nu *; show_layer_stats" + } catch { + print "Could not show layer statistics" + } + } +} + +# Test layer resolution +export def "main layer test" [ + module: string, # Module to test + workspace: string, # Workspace/infra name + provider: string = "upcloud" # Provider for testing +] { + print $"🧪 Testing layer resolution: ($module) in ($workspace) with ($provider)" + + try { + nu -c $"use provisioning/workspace/tools/layer-utils.nu *; test_layer_resolution ($module) ($workspace) ($provider)" + } catch { + print $"❌ Layer resolution test failed for ($module)" + } +} + +# === OVERRIDE COMMANDS === + +# Create configuration override +export def "main override create" [ + type: string, # Type: taskservs, providers, clusters + infra: string, # Infrastructure name + module: string, # Module name + --from: string = "", # Template to base override on + --layer: string = "infra" # Layer for override +] { + print $"⚙️ Creating override for ($module) in ($infra) at layer ($layer)" + + let override_path = match $layer { + "infra" => $"workspace/infra/($infra)/overrides/($module).k" + "workspace" => $"provisioning/workspace/templates/($type)/($module).k" + _ => { + print $"Unknown layer: ($layer)" + return + } + } + + print $"📝 Override will be created at: ($override_path)" + + if not ($from | is-empty) { + print $"📋 Based on template: ($from)" + } + + # Create directory if needed + mkdir ($override_path | path dirname) + + # Create basic override file + let content = if not ($from | is-empty) { + $"# Override for ($module) in ($infra) +# Based on template: ($from) + +import ($type).*.($module).kcl.($module) as base +import provisioning.workspace.templates.($type).($from) as template + +# Infrastructure-specific overrides +($module)_($infra)_override: base.($module | str capitalize) = template.($from)_template { + # Add your overrides here + # Example: + # replicas = 3 + # resources.memory = \"1Gi\" +} +" + } else { + $"# Override for ($module) in ($infra) + +import ($type).*.($module).kcl.($module) as base + +# Infrastructure-specific overrides +($module)_($infra)_override: base.($module | str capitalize) = base.($module)_config { + # Add your overrides here + # Example: + # replicas = 3 + # resources.memory = \"1Gi\" +} +" + } + + $content | save $override_path + print $"✅ Override created: ($override_path)" +} + +# === WORKSPACE MANAGEMENT === + +# Initialize workspace with modules +export def "main init" [ + workspace: string, # Workspace path + --modules: list = [], # Initial modules to load + --template: string = "", # Workspace template + --provider: string = "upcloud" # Default provider +] { + print $"🚀 Initializing workspace: ($workspace)" + + # Create workspace structure + let workspace_dirs = [ + $"($workspace)/config" + $"($workspace)/taskservs" + $"($workspace)/overrides" + $"($workspace)/defs" + $"($workspace)/clusters" + ] + + for dir in $workspace_dirs { + mkdir $dir + print $" 📁 Created: ($dir)" + } + + # Create basic configuration + let config_content = $"# Workspace configuration for ($workspace) +# Provider: ($provider) +# Initialized: (date now) + +provider = "($provider)" +workspace = "($workspace)" +" + $config_content | save $"($workspace)/config/workspace.toml" + print $" 📄 Created: ($workspace)/config/workspace.toml" + + # Load initial modules + if ($modules | length) > 0 { + print $"📦 Loading initial modules: (($modules | str join ', '))" + main load taskservs $workspace ...$modules + } + + print $"✅ Workspace ($workspace) initialized successfully" +} + +# Validate workspace integrity +export def "main validate" [workspace: string] { + print $"🔍 Validating workspace: ($workspace)" + + let required_dirs = ["config", "taskservs", "overrides", "defs"] + mut validation_errors = [] + + for dir in $required_dirs { + let full_path = ($workspace | path join $dir) + if not ($full_path | path exists) { + $validation_errors = ($validation_errors | append $"Missing directory: ($full_path)") + } + } + + # Check configuration file + let config_file = ($workspace | path join "config" "workspace.toml") + if not ($config_file | path exists) { + $validation_errors = ($validation_errors | append $"Missing configuration: ($config_file)") + } + + # Report results + if ($validation_errors | is-empty) { + print "✅ Workspace validation passed" + return true + } else { + print "❌ Workspace validation failed:" + for error in $validation_errors { + print $" • ($error)" + } + return false + } +} + +# Show workspace information +export def "main info" [workspace: string] { + print $"📊 Workspace Information: ($workspace)" + + if not (($workspace | path join "config" "workspace.toml") | path exists) { + print "❌ Workspace not found or not initialized" + return + } + + # Show basic info + let config = try { open ($workspace | path join "config" "workspace.toml") | from toml } catch { {} } + + print $" Provider: (($config.provider? | default 'unknown'))" + print $" Path: ($workspace)" + + # Count modules + let taskserv_count = try { + ls ($workspace | path join "taskservs") | length + } catch { 0 } + + let override_count = try { + ls ($workspace | path join "overrides") | length + } catch { 0 } + + print $" Task Services: ($taskserv_count)" + print $" Overrides: ($override_count)" + + # Show recent activity + let recent_files = try { + ls $workspace | where type == file | sort-by modified | last 3 | get name + } catch { [] } + + if ($recent_files | length) > 0 { + print " Recent activity:" + for file in $recent_files { + print $" • ($file | path basename)" + } + } +} + +# === HELPER FUNCTIONS === + +# Generic extension loading function (taskservs, providers, clusters, workflows) +def load_extension_to_workspace [ + extension_type: string, # taskservs, providers, clusters, workflows + workspace: string, + modules: list, + layer: string, + force: bool = false +] { + # Get extension-specific info function based on type + let get_info_fn = match $extension_type { + "taskservs" => { |name| get-taskserv-info $name } + "providers" => { |name| get-provider-info $name } + "clusters" => { |name| get-cluster-info $name } + _ => { |name| {name: $name, group: "", type: $extension_type} } + } + + # Get source path from config + let source_base_path = (config-get $"paths.($extension_type)" | path expand) + + # Get template base path from config + let provisioning_base = (config-get "paths.base" | path expand) + let template_base_path = ($provisioning_base | path join "workspace" "templates" $extension_type) + + for module in $modules { + print $" 📦 Loading ($extension_type): ($module)" + + # Get module info + let module_info = try { + do $get_info_fn $module + } catch { + print $" ❌ Module not found: ($module)" + continue + } + + print $" ✓ Found: ($module_info.name) (($module_info.group? | default ""))" + + # Resolve workspace paths + let workspace_abs = ($workspace | path expand) + let workspace_root = if ($workspace_abs | str contains "/infra/") { + let parts = ($workspace_abs | split row "/infra/") + $parts.0 + } else { + $workspace_abs + } + + # Build source path (handle optional group, "root" means no category) + let group_path = ($module_info.group? | default "") + let group_path = if ($group_path == "root") { "" } else { $group_path } + let source_module_path = if ($group_path | is-not-empty) { + $source_base_path | path join $group_path $module + } else { + $source_base_path | path join $module + } + + # STEP 1: Copy schemas to workspace/.{extension_type} + let target_schemas_dir = ($workspace_root | path join $".($extension_type)") + let target_module_path = if ($group_path | is-not-empty) { + $target_schemas_dir | path join $group_path $module + } else { + $target_schemas_dir | path join $module + } + + # Config file directory + let config_dir = ($workspace_abs | path join $extension_type) + let config_file_path = ($config_dir | path join $"($module).k") + + # Check if already loaded + if ($config_file_path | path exists) and ($target_module_path | path exists) { + if not $force { + print $" ✅ Module already loaded: ($module)" + print $" Config: ($config_file_path)" + print $" Source: ($target_module_path)" + print $" 💡 Use --force to overwrite existing files" + continue + } else { + print $" 🔄 Overwriting existing module: ($module)" + } + } + + # Copy schemas from system extensions to workspace + let parent_dir = ($target_module_path | path dirname) + mkdir $parent_dir + + if ($source_module_path | path exists) { + print $" 📦 Copying schemas to workspace .($extension_type)..." + print $" From: ($source_module_path)" + print $" To: ($target_module_path)" + + if ($target_module_path | path exists) { + rm -rf $target_module_path + } + + cp -r $source_module_path $parent_dir + print $" ✓ Schemas copied to workspace .($extension_type)/" + + # STEP 2a: Update individual module's kcl.mod with correct workspace paths + # Calculate relative paths based on categorization depth + let provisioning_path = if ($group_path | is-not-empty) { + # Categorized: .{ext}/{category}/{module}/kcl/ -> ../../../../.kcl/packages/provisioning + "../../../../.kcl/packages/provisioning" + } else { + # Non-categorized: .{ext}/{module}/kcl/ -> ../../../.kcl/packages/provisioning + "../../../.kcl/packages/provisioning" + } + + let parent_path = if ($group_path | is-not-empty) { + # Categorized: .{ext}/{category}/{module}/kcl/ -> ../../.. + "../../.." + } else { + # Non-categorized: .{ext}/{module}/kcl/ -> ../.. + "../.." + } + + # Update the module's kcl.mod file with workspace-relative paths + let module_kcl_mod_path = ($target_module_path | path join "kcl" "kcl.mod") + if ($module_kcl_mod_path | path exists) { + print $" 🔧 Updating module kcl.mod with workspace paths" + let module_kcl_mod_content = $"[package] +name = \"($module)\" +edition = \"v0.11.3\" +version = \"0.0.1\" + +[dependencies] +provisioning = { path = \"($provisioning_path)\", version = \"0.0.1\" } +($extension_type) = { path = \"($parent_path)\", version = \"0.1.0\" } +" + $module_kcl_mod_content | save -f $module_kcl_mod_path + print $" ✓ Updated kcl.mod: ($module_kcl_mod_path)" + } + } else { + print $" ⚠️ Warning: Source not found at ($source_module_path)" + } + + # STEP 2b: Create kcl.mod in workspace/.{extension_type} + let extension_kcl_mod = ($target_schemas_dir | path join "kcl.mod") + if not ($extension_kcl_mod | path exists) { + print $" 📦 Creating kcl.mod for .($extension_type) package" + let kcl_mod_content = $"[package] +name = \"($extension_type)\" +edition = \"v0.11.3\" +version = \"0.1.0\" +description = \"Workspace-level ($extension_type) schemas\" +" + $kcl_mod_content | save $extension_kcl_mod + } + + # Ensure config directory exists + mkdir $config_dir + + # STEP 4: Generate config from template + let template_path = if ($group_path | is-not-empty) { + $template_base_path | path join $group_path $"($module).k" + } else { + $template_base_path | path join $"($module).k" + } + + # Build import statement with "as {module}" alias + let import_stmt = if ($group_path | is-not-empty) { + $"import ($extension_type).($group_path).($module).kcl.($module) as ($module)" + } else { + $"import ($extension_type).($module).kcl.($module) as ($module)" + } + + # Get relative paths for comments + let workspace_name = ($workspace_root | path basename) + let relative_schema_path = if ($group_path | is-not-empty) { + $"($workspace_name)/.($extension_type)/($group_path)/($module)" + } else { + $"($workspace_name)/.($extension_type)/($module)" + } + + let config_content = if ($template_path | path exists) { + print $" 📄 Using template from: ($template_path)" + let template_body = (open $template_path) + $"# Configuration for ($module) +# Workspace: ($workspace_name) +# Schemas from: ($relative_schema_path) +($import_stmt) + +($template_body)" + } else { + $"# Configuration for ($module) +# Workspace: ($workspace_name) +# Schemas from: ($relative_schema_path) +($import_stmt) + +# TODO: Configure your ($module) instance +# See available schemas at: ($relative_schema_path)/kcl/ +" + } + + $config_content | save -f $config_file_path + print $" ✓ Config created: ($config_file_path)" + print $" 📝 Edit ($extension_type)/($module).k to configure settings" + + # STEP 3: Update infra kcl.mod + if ($workspace_abs | str contains "/infra/") { + let kcl_mod_path = ($workspace_abs | path join "kcl.mod") + if ($kcl_mod_path | path exists) { + let kcl_mod_content = (open $kcl_mod_path) + if not ($kcl_mod_content | str contains $"($extension_type) =") { + print $" 🔧 Updating kcl.mod to include ($extension_type) dependency" + let new_dependency = $"\n# Workspace-level ($extension_type) \(shared across infras\)\n($extension_type) = { path = \"../../.($extension_type)\" }\n" + $"($kcl_mod_content)($new_dependency)" | save -f $kcl_mod_path + } + } + } + } +} + +# Unload taskserv from workspace +def unload_taskserv_from_workspace [workspace: string, module: string, layer: string] { + let target_path = match $layer { + "workspace" => ($workspace | path join "taskservs" $"($module).k") + "infra" => ($workspace | path join "overrides" $"($module).k") + _ => ($workspace | path join "taskservs" $"($module).k") + } + + if ($target_path | path exists) { + rm $target_path + print $" ✓ Removed: ($target_path)" + } else { + print $" ❌ Not found: ($target_path)" + } +} + +# List workspace taskservs +def list_workspace_taskservs [workspace: string, layer: string, format: string] { + let paths = match $layer { + "workspace" => [($workspace | path join "taskservs")] + "infra" => [($workspace | path join "overrides")] + "all" => [($workspace | path join "taskservs"), ($workspace | path join "overrides")] + _ => [($workspace | path join "taskservs")] + } + + mut all_taskservs = [] + + for path in $paths { + if ($path | path exists) { + let taskservs = ls $path + | where type == file + | where name =~ '\\.k$' + | each { |file| + { + name: ($file.name | path basename | str replace ".k" "") + layer: ($path | path basename) + path: $file.name + modified: $file.modified + } + } + $all_taskservs = ($all_taskservs | append $taskservs) + } + } + + format_output $all_taskservs $format +} + +# Format output based on requested format +def format_output [data: any, format: string] { + match $format { + "json" => ($data | to json) + "yaml" => ($data | to yaml) + "names" => ($data | get name | str join "\n") + "table" | _ => ($data | table) + } +} + +# === HELP FUNCTIONS === + +def print_enhanced_help [] { + print "Enhanced Module Loader CLI - Discovery and loading with template support" + print "" + print "Usage: module-loader [options]" + print "" + print "CORE COMMANDS:" + print " discover [query] [--format ] [--category ] - Discover available modules" + print " sync-kcl [--manifest ] [--kcl] - Sync KCL dependencies for infrastructure" + print " load [--layer ] - Load modules into workspace" + print " list [--layer ] - List loaded modules" + print " unload [--layer ] - Unload module from workspace" + print "" + print "WORKSPACE COMMANDS:" + print " init [--modules ] [--template ] - Initialize workspace" + print " validate - Validate workspace integrity" + print " info - Show workspace information" + print "" + print "TEMPLATE COMMANDS:" + print " template list [--type ] [--format ] - List available templates" + print " template extract [--type ] - Extract template from infra" + print " template apply