diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9e72816 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,176 @@ +# Provisioning Core - Changelog + +**Date**: 2026-01-08 +**Repository**: provisioning/core +**Status**: Nickel IaC (PRIMARY) + +--- + +## πŸ“‹ Summary + +Core system with Nickel as primary IaC: CLI enhancements, Nushell library refactoring for schema support, config loader for Nickel evaluation, and comprehensive infrastructure automation. + +--- + +## πŸ“ Changes by Directory + +### cli/ directory + +**Major Updates (586 lines added to provisioning)** + +- Expanded CLI command implementations (+590 lines) +- Enhanced tools installation system (tools-install: +163 lines) +- Improved install script for Nushell environment (install_nu.sh: +31 lines) +- Better CLI routing and command validation +- Help system enhancements for Nickel-aware commands +- Support for Nickel schema evaluation and validation + +### nulib/ directory + +**Nushell libraries - Nickel-first architecture** + +**Config System** +- `config/loader.nu` - Nickel schema loading and evaluation +- `config/accessor.nu` - Accessor patterns for Nickel fields +- `config/cache/` - Cache system optimized for Nickel evaluation + +**AI & Documentation** +- `ai/README.md` - Nickel IaC patterns +- `ai/info_about.md` - Nickel-focused documentation +- `ai/lib.nu` - AI integration for Nickel schema analysis + +**Extension System** +- `extensions/QUICKSTART.md` - Nickel extension quickstart (+50 lines) +- `extensions/README.md` - Extension system for Nickel (+63 lines) +- `extensions/loader_oci.nu` - OCI registry loader (minor updates) + +**Infrastructure & Validation** +- `infra_validator/rules_engine.nu` - Validation rules for Nickel schemas +- `infra_validator/validator.nu` - Schema validation support +- `loader-minimal.nu` - Minimal loader for lightweight deployments + +**Clusters & Workflows** +- `clusters/discover.nu`, `clusters/load.nu`, `clusters/run.nu` - Cluster operations updated +- Plugin definitions updated for Nickel integration (+28-38 lines) + +**Documentation** +- `SERVICE_MANAGEMENT_SUMMARY.md` - Expanded service documentation (+90 lines) +- `gitea/IMPLEMENTATION_SUMMARY.md` - Gitea integration guide (+89 lines) +- Extension and validation quickstarts and README updates + +### plugins/ directory + +Nushell plugins for performance optimization + +**Sub-repositories:** + +- `nushell-plugins/` - Multiple Nushell plugins + - `_nu_plugin_inquire/` - Interactive form plugin + - `api_nu_plugin_nickel/` - Nickel integration plugin + - Additional plugin implementations + +**Plugin Documentation:** + +- Build summaries +- Installation guides +- Configuration examples +- Test documentation +- Fix and limitation reports + +### scripts/ directory + +Utility scripts for system operations + +- Build scripts +- Installation scripts +- Testing scripts +- Development utilities +- Infrastructure scripts + +### services/ directory + +Service definitions and configurations + +- Service descriptions +- Service management + +### forminquire/ directory + +Form inquiry interface + +- Interactive form system +- User input handling + +### Additional Files + +- `README.md` - Core system documentation +- `versions.ncl` - Version definitions +- `.gitignore` - Git ignore patterns +- `nickel.mod` / `nickel.mod.lock` - Nickel module definitions +- `.githooks/` - Git hooks for development + +--- + +## πŸ“Š Change Statistics + +| Category | Files | Lines Added | Lines Removed | Status | +|----------|-------|-------------|---------------|--------| +| CLI | 3 | 780+ | 30+ | Major update | +| Config System | 15+ | 300+ | 200+ | Refactored | +| AI/Docs | 8+ | 350+ | 100+ | Enhanced | +| Extensions | 5+ | 150+ | 50+ | Updated | +| Infrastructure | 8+ | 100+ | 70+ | Updated | +| Clusters/Workflows | 5+ | 80+ | 30+ | Enhanced | +| **Total** | **60+ files** | **1700+ lines** | **500+ lines** | **Complete** | + +--- + +## ✨ Key Areas + +### CLI System + +- Command implementations with Nickel support +- Tools installation system +- Nushell environment setup +- Nickel schema evaluation commands +- Error messages and help text +- Nickel type checking and validation + +### Config System + +- **Nickel-first loader**: Schema evaluation via config/loader.nu +- **Optimized caching**: Nickel evaluation cache patterns +- **Field accessors**: Nickel record manipulation +- **Schema validation**: Type-safe configuration loading + +### AI & Documentation + +- AI integration for Nickel IaC +- Extension development guides +- Service management documentation + +### Extensions & Infrastructure + +- OCI registry loader optimization +- Schema-aware extension system +- Infrastructure validation for Nickel definitions +- Cluster discovery and operations enhanced + +--- + +## 🎯 Current Features + +- **Nickel IaC**: Type-safe infrastructure definitions +- **CLI System**: Unified command interface with 80+ shortcuts +- **Provider Abstraction**: Cloud-agnostic operations +- **Config System**: Hierarchical configuration with 476+ accessors +- **Workflow Engine**: Batch operations with dependency resolution +- **Validation**: Schema-aware infrastructure validation +- **AI Integration**: Schema-driven configuration generation + +--- + +**Status**: Production +**Date**: 2026-01-08 +**Repository**: provisioning/core +**Version**: 5.0.0 diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index a88a5b5..0000000 --- a/CHANGES.md +++ /dev/null @@ -1,163 +0,0 @@ -# Provisioning Core - Changes - -**Date**: 2025-12-11 -**Repository**: provisioning/core -**Changes**: CLI, libraries, plugins, and utilities updates - ---- - -## πŸ“‹ Summary - -Updates to core CLI, Nushell libraries, plugins system, and utility scripts for the provisioning core system. - ---- - -## πŸ“ Changes by Directory - -### cli/ directory -Provisioning CLI implementation and commands -- Command implementations -- CLI utilities -- Command routing and dispatching -- Help system -- Command validation - -### nulib/ directory -Nushell libraries and modules (core business logic) - -**Key Modules:** -- `lib_provisioning/` - Main library modules - - config/ - Configuration loading and management - - extensions/ - Extension system - - secrets/ - Secrets management - - infra_validator/ - Infrastructure validation - - ai/ - AI integration documentation - - user/ - User management - - workspace/ - Workspace operations - - cache/ - Caching system - - utils/ - Utility functions - -**Workflows:** -- Batch operations and orchestration -- Server management -- Task service management -- Cluster operations -- Test environments - -**Services:** -- Service management scripts -- Task service utilities -- Infrastructure utilities - -**Documentation:** -- Library module documentation -- Extension API quickstart -- Secrets management guide -- Service management summary -- Test environments guide - -### plugins/ directory -Nushell plugins for performance optimization - -**Sub-repositories:** -- `nushell-plugins/` - Multiple Nushell plugins - - `_nu_plugin_inquire/` - Interactive form plugin - - `api_nu_plugin_kcl/` - KCL integration plugin - - Additional plugin implementations - -**Plugin Documentation:** -- Build summaries -- Installation guides -- Configuration examples -- Test documentation -- Fix and limitation reports - -### scripts/ directory -Utility scripts for system operations -- Build scripts -- Installation scripts -- Testing scripts -- Development utilities -- Infrastructure scripts - -### services/ directory -Service definitions and configurations -- Service descriptions -- Service management - -### forminquire/ directory -Form inquiry interface -- Interactive form system -- User input handling - -### Additional Files -- `README.md` - Core system documentation -- `versions.k` - Version definitions -- `.gitignore` - Git ignore patterns -- `kcl.mod` / `kcl.mod.lock` - KCL module definitions -- `.githooks/` - Git hooks for development - ---- - -## πŸ“Š Change Statistics - -| Category | Files | Status | -|----------|-------|--------| -| CLI | 8+ | Updated | -| Libraries | 20+ | Updated | -| Plugins | 10+ | Updated | -| Scripts | 15+ | Updated | -| Documentation | 20+ | Updated | - ---- - -## ✨ Key Areas - -### CLI System -- Command implementations -- Flag handling and validation -- Help and documentation -- Error handling - -### Nushell Libraries -- Configuration management -- Infrastructure validation -- Extension system -- Secrets management -- Workspace operations -- Cache management - -### Plugin System -- Interactive forms (inquire) -- KCL integration -- Performance optimization -- Plugin registration - -### Scripts & Utilities -- Build and distribution -- Installation procedures -- Testing utilities -- Development tools - ---- - -## πŸ”„ Backward Compatibility - -**βœ… 100% Backward Compatible** - -All changes are additive or maintain existing interfaces. - ---- - -## πŸš€ No Breaking Changes - -- Existing commands work unchanged -- Library APIs remain compatible -- Plugin system compatible -- Configuration remains compatible - ---- - -**Status**: Core system updates complete -**Date**: 2025-12-11 -**Repository**: provisioning/core diff --git a/README.md b/README.md index b2954b3..808f56a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ # Core Engine -The **Core Engine** is the foundational component of the [Provisioning project](https://repo.jesusperez.pro/jesus/provisioning), providing the unified CLI interface, core Nushell libraries, and essential utility scripts. Built on **Nushell** and **KCL**, it serves as the primary entry point for all infrastructure operations. +The **Core Engine** is the foundational component of the [Provisioning project](https://repo.jesusperez.pro/jesus/provisioning), providing the unified CLI interface, core Nushell libraries, and essential utility scripts. Built on **Nushell** and **Nickel**, it serves as the primary entry point for all infrastructure operations. ## Overview @@ -23,7 +23,7 @@ The Core Engine provides: ## Project Structure -``` +```plaintext provisioning/core/ β”œβ”€β”€ cli/ # Command-line interface β”‚ └── provisioning # Main CLI entry point (211 lines, 84% reduction) @@ -47,14 +47,14 @@ provisioning/core/ β”œβ”€β”€ scripts/ # Utility scripts β”‚ └── test/ # Test automation └── resources/ # Images and logos -``` +```plaintext ## Installation ### Prerequisites -- **Nushell 0.107.1+** - Primary shell and scripting environment -- **KCL 0.11.2+** - Configuration language for infrastructure definitions +- **Nushell 0.109.0+** - Primary shell and scripting environment +- **Nickel 1.15.1+** - Configuration language for infrastructure definitions - **SOPS 3.10.2+** - Secrets management (optional but recommended) - **Age 1.2.1+** - Encryption tool for secrets (optional) @@ -68,14 +68,14 @@ ln -sf "$(pwd)/provisioning/core/cli/provisioning" /usr/local/bin/provisioning # Or add to PATH in your shell config (~/.bashrc, ~/.zshrc, etc.) export PATH="$PATH:/path/to/project-provisioning/provisioning/core/cli" -``` +```plaintext Verify installation: ```bash provisioning version provisioning help -``` +```plaintext ## Quick Start @@ -97,7 +97,7 @@ provisioning providers # Show system information provisioning nuinfo -``` +```plaintext ### Infrastructure Operations @@ -116,7 +116,7 @@ provisioning cluster create my-cluster # SSH into server provisioning server ssh hostname-01 -``` +```plaintext ### Quick Reference @@ -124,7 +124,7 @@ For fastest command reference: ```bash provisioning sc -``` +```plaintext For complete guides: @@ -132,7 +132,7 @@ For complete guides: provisioning guide from-scratch # Complete deployment guide provisioning guide quickstart # Command shortcuts reference provisioning guide customize # Customization patterns -``` +```plaintext ## Core Libraries @@ -152,7 +152,7 @@ let value = config get "servers.default_plan" # Load workspace config let ws_config = config load-workspace "my-project" -``` +```plaintext ### Provider Abstraction (`lib_provisioning/providers/`) @@ -166,7 +166,7 @@ let provider = providers get "upcloud" # Create server using provider $provider | invoke "create_server" $server_config -``` +```plaintext ### Utilities (`lib_provisioning/utils/`) @@ -185,7 +185,7 @@ Batch operations with dependency resolution: ```bash # Submit batch workflow -provisioning batch submit workflows/example.k +provisioning batch submit workflows/example.ncl # Monitor workflow progress provisioning batch monitor @@ -195,7 +195,7 @@ provisioning workflow list # Get workflow status provisioning workflow status -``` +```plaintext ## CLI Architecture @@ -236,7 +236,7 @@ Help works in both directions: provisioning help workspace # βœ… provisioning workspace help # βœ… Same result provisioning ws help # βœ… Shortcut also works -``` +```plaintext ## Configuration @@ -262,7 +262,7 @@ provisioning allenv # Use specific environment PROVISIONING_ENV=prod provisioning server list -``` +```plaintext ### Debug Flags @@ -278,7 +278,7 @@ provisioning --yes cluster delete # Specify infrastructure provisioning --infra my-project server list -``` +```plaintext ## Design Principles @@ -329,8 +329,8 @@ The project follows a three-phase migration: ### Required -- **Nushell 0.107.1+** - Shell and scripting language -- **KCL 0.11.2+** - Configuration language +- **Nushell 0.109.0+** - Shell and scripting language +- **Nickel 1.15.1+** - Configuration language ### Recommended @@ -341,7 +341,7 @@ The project follows a three-phase migration: ### Optional - **nu_plugin_tera** - Template rendering -- **nu_plugin_kcl** - KCL integration (CLI `kcl` is required, plugin optional) +- **Nickel Language** - Native Nickel support via CLI (no plugin required) ## Documentation @@ -354,14 +354,14 @@ The project follows a three-phase migration: ### Architecture Documentation -- **CLI Architecture**: `docs/architecture/ADR-006-provisioning-cli-refactoring.md` -- **Configuration System**: See `.claude/features/configuration-system.md` -- **Batch Workflows**: See `.claude/features/batch-workflow-system.md` -- **Orchestrator**: See `.claude/features/orchestrator-architecture.md` +- **CLI Architecture**: `../docs/src/architecture/adr/ADR-006-provisioning-cli-refactoring.md` +- **Configuration System**: `../docs/src/infrastructure/configuration-system.md` +- **Batch Workflows**: `../docs/src/infrastructure/batch-workflow-system.md` +- **Orchestrator**: `../docs/src/operations/orchestrator-system.md` ### API Documentation -- **REST API**: See `docs/api/` (when orchestrator is running) +- **REST API**: See `../docs/src/api-reference/` (when orchestrator is running) - **Nushell Modules**: See inline documentation in `nulib/` modules ## Testing @@ -375,7 +375,7 @@ nu provisioning/core/scripts/test/test_all.nu # Run specific test group nu provisioning/core/scripts/test/test_config.nu nu provisioning/core/scripts/test/test_cli.nu -``` +```plaintext ### Test Coverage @@ -402,22 +402,26 @@ When contributing to the Core Engine: ### Common Issues **Missing environment variables:** + ```bash provisioning env # Check current configuration provisioning validate config # Validate configuration files -``` +```plaintext + +**Nickel schema errors:** -**KCL compilation errors:** ```bash -kcl fmt .k # Format KCL file -kcl run .k # Test KCL file -``` +nickel fmt .ncl # Format Nickel file +nickel eval .ncl # Evaluate Nickel schema +nickel typecheck .ncl # Type check schema +```plaintext **Provider authentication:** + ```bash provisioning providers # List available providers provisioning show settings # View provider configuration -``` +```plaintext ### Debug Mode @@ -425,7 +429,7 @@ Enable verbose logging: ```bash provisioning --debug -``` +```plaintext ### Getting Help @@ -434,7 +438,7 @@ provisioning help # Show main help provisioning help # Category-specific help provisioning help # Command-specific help provisioning guide list # List all guides -``` +```plaintext ## Version Information @@ -443,7 +447,7 @@ Check system versions: ```bash provisioning version # Show all versions provisioning nuinfo # Nushell information -``` +```plaintext ## License @@ -451,5 +455,5 @@ See project root LICENSE file. --- -**Maintained By**: Architecture Team -**Last Updated**: 2025-10-07 +**Maintained By**: Core Team +**Last Updated**: 2026-01-08 diff --git a/kcl.mod b/kcl.mod deleted file mode 100644 index 9b238fe..0000000 --- a/kcl.mod +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "provisioning-core" -edition = "v0.11.3" -version = "1.0.0" - -[dependencies] -provisioning = { path = "../kcl" } diff --git a/kcl.mod.lock b/kcl.mod.lock deleted file mode 100644 index a5b8af8..0000000 --- a/kcl.mod.lock +++ /dev/null @@ -1,5 +0,0 @@ -[dependencies] - [dependencies.provisioning] - name = "provisioning" - full_name = "provisioning_0.0.1" - version = "0.0.1" diff --git a/nulib/SERVICE_MANAGEMENT_SUMMARY.md b/nulib/SERVICE_MANAGEMENT_SUMMARY.md deleted file mode 100644 index 321d089..0000000 --- a/nulib/SERVICE_MANAGEMENT_SUMMARY.md +++ /dev/null @@ -1,725 +0,0 @@ -# Service Management System - Implementation Summary - -**Implementation Date**: 2025-10-06 -**Version**: 1.0.0 -**Status**: βœ… Complete - Ready for Testing - ---- - -## Executive Summary - -A comprehensive service management system has been implemented for orchestrating platform services (orchestrator, control-center, CoreDNS, Gitea, OCI registry, MCP server, API gateway). The system provides unified lifecycle management, automatic dependency resolution, health monitoring, and pre-flight validation. - -**Key Achievement**: Complete service orchestration framework with 7 platform services, 5 deployment modes, 4 health check types, and automatic dependency resolution. - ---- - -## Deliverables Completed - -### 1. KCL Service Schema βœ… - -**File**: `provisioning/kcl/services.k` (350 lines) - -**Schemas Defined**: -- `ServiceRegistry` - Top-level service registry -- `ServiceDefinition` - Individual service definition -- `ServiceDeployment` - Deployment configuration -- `BinaryDeployment` - Native binary deployment -- `DockerDeployment` - Docker container deployment -- `DockerComposeDeployment` - Docker Compose deployment -- `KubernetesDeployment` - K8s deployment -- `HelmChart` - Helm chart configuration -- `RemoteDeployment` - Remote service connection -- `HealthCheck` - Health check configuration -- `HttpHealthCheck` - HTTP health check -- `TcpHealthCheck` - TCP port health check -- `CommandHealthCheck` - Command-based health check -- `FileHealthCheck` - File-based health check -- `StartupConfig` - Service startup configuration -- `ResourceLimits` - Resource limits -- `ServiceState` - Runtime state tracking -- `ServiceOperation` - Operation requests - -**Features**: -- Complete type safety with validation -- Support for 5 deployment modes -- 4 health check types -- Dependency and conflict management -- Resource limits and startup configuration - -### 2. Service Registry Configuration βœ… - -**File**: `provisioning/config/services.toml` (350 lines) - -**Services Registered**: -1. **orchestrator** - Rust orchestrator (binary, auto-start, order: 10) -2. **control-center** - Web UI (binary, depends on orchestrator, order: 20) -3. **coredns** - Local DNS (Docker, conflicts with dnsmasq, order: 15) -4. **gitea** - Git server (Docker, order: 30) -5. **oci-registry** - Container registry (Docker, order: 25) -6. **mcp-server** - MCP server (binary, depends on orchestrator, order: 40) -7. **api-gateway** - API gateway (binary, depends on orchestrator, order: 45) - -**Configuration Features**: -- Complete deployment specifications -- Health check endpoints -- Dependency declarations -- Startup order and timeout configuration -- Resource limits -- Auto-start flags - -### 3. Service Manager Core βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/manager.nu` (350 lines) - -**Functions Implemented**: -- `load-service-registry` - Load services from TOML -- `get-service-definition` - Get service configuration -- `is-service-running` - Check if service is running -- `get-service-status` - Get detailed service status -- `start-service` - Start service with dependencies -- `stop-service` - Stop service gracefully -- `restart-service` - Restart service -- `check-service-health` - Execute health check -- `wait-for-service` - Wait for health check -- `list-all-services` - Get all services -- `list-running-services` - Get running services -- `get-service-logs` - Retrieve service logs -- `init-service-state` - Initialize state directories - -**Features**: -- PID tracking and process management -- State persistence -- Multi-mode support (binary, Docker, K8s) -- Automatic dependency handling - -### 4. Service Lifecycle Management βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/lifecycle.nu` (480 lines) - -**Functions Implemented**: -- `start-service-by-mode` - Start based on deployment mode -- `start-binary-service` - Start native binary -- `start-docker-service` - Start Docker container -- `start-docker-compose-service` - Start via Compose -- `start-kubernetes-service` - Start on K8s -- `stop-service-by-mode` - Stop based on deployment mode -- `stop-binary-service` - Stop binary process -- `stop-docker-service` - Stop Docker container -- `stop-docker-compose-service` - Stop Compose service -- `stop-kubernetes-service` - Delete K8s deployment -- `get-service-pid` - Get process ID -- `kill-service-process` - Send signal to process - -**Features**: -- Background process management -- Docker container orchestration -- Kubernetes deployment handling -- Helm chart support -- PID file management -- Log file redirection - -### 5. Health Check System βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/health.nu` (220 lines) - -**Functions Implemented**: -- `perform-health-check` - Execute health check -- `http-health-check` - HTTP endpoint check -- `tcp-health-check` - TCP port check -- `command-health-check` - Command execution check -- `file-health-check` - File existence check -- `retry-health-check` - Retry with backoff -- `wait-for-service` - Wait for healthy state -- `get-health-status` - Get current health -- `monitor-service-health` - Continuous monitoring - -**Features**: -- 4 health check types (HTTP, TCP, Command, File) -- Configurable timeout and retries -- Automatic retry with interval -- Real-time monitoring -- Duration tracking - -### 6. Pre-flight Check System βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/preflight.nu` (280 lines) - -**Functions Implemented**: -- `check-required-services` - Check services for operation -- `validate-service-prerequisites` - Validate prerequisites -- `auto-start-required-services` - Auto-start dependencies -- `check-service-conflicts` - Detect conflicts -- `validate-all-services` - Validate all configurations -- `preflight-start-service` - Pre-flight for start -- `get-readiness-report` - Platform readiness - -**Features**: -- Prerequisite validation (binary exists, Docker running) -- Conflict detection -- Auto-start orchestration -- Comprehensive validation -- Readiness reporting - -### 7. Dependency Resolution βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/dependencies.nu` (310 lines) - -**Functions Implemented**: -- `resolve-dependencies` - Resolve dependency tree -- `get-dependency-tree` - Get tree structure -- `topological-sort` - Dependency ordering -- `start-services-with-deps` - Start with dependencies -- `validate-dependency-graph` - Detect cycles -- `get-startup-order` - Calculate startup order -- `get-reverse-dependencies` - Find dependents -- `visualize-dependency-graph` - Generate visualization -- `can-stop-service` - Check safe to stop - -**Features**: -- Topological sort for ordering -- Circular dependency detection -- Reverse dependency tracking -- Safe stop validation -- Dependency graph visualization - -### 8. CLI Commands βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/commands.nu` (480 lines) - -**Platform Commands**: -- `platform start` - Start all or specific services -- `platform stop` - Stop all or specific services -- `platform restart` - Restart services -- `platform status` - Show platform status -- `platform logs` - View service logs -- `platform health` - Check platform health -- `platform update` - Update platform (placeholder) - -**Service Commands**: -- `services list` - List services -- `services status` - Service status -- `services start` - Start service -- `services stop` - Stop service -- `services restart` - Restart service -- `services health` - Check health -- `services logs` - View logs -- `services check` - Check required services -- `services dependencies` - View dependencies -- `services validate` - Validate configurations -- `services readiness` - Readiness report -- `services monitor` - Continuous monitoring - -**Features**: -- User-friendly output -- Interactive feedback -- Pre-flight integration -- Dependency awareness -- Health monitoring - -### 9. Docker Compose Configuration βœ… - -**File**: `provisioning/platform/docker-compose.yaml` (180 lines) - -**Services Defined**: -- orchestrator (with health check) -- control-center (depends on orchestrator) -- coredns (DNS resolution) -- gitea (Git server) -- oci-registry (Zot) -- mcp-server (MCP integration) -- api-gateway (API proxy) - -**Features**: -- Health checks for all services -- Volume persistence -- Network isolation (provisioning-net) -- Service dependencies -- Restart policies - -### 10. CoreDNS Configuration βœ… - -**Files**: -- `provisioning/platform/coredns/Corefile` (35 lines) -- `provisioning/platform/coredns/zones/provisioning.zone` (30 lines) - -**Features**: -- Local DNS resolution for `.provisioning.local` -- Service discovery (api, ui, git, registry aliases) -- Upstream DNS forwarding -- Health check zone - -### 11. OCI Registry Configuration βœ… - -**File**: `provisioning/platform/oci-registry/config.json` (20 lines) - -**Features**: -- OCI-compliant configuration -- Search and UI extensions -- Persistent storage - -### 12. Module System βœ… - -**File**: `provisioning/core/nulib/lib_provisioning/services/mod.nu` (15 lines) - -Exports all service management functionality. - -### 13. Test Suite βœ… - -**File**: `provisioning/core/nulib/tests/test_services.nu` (380 lines) - -**Test Coverage**: -1. Service registry loading -2. Service definition retrieval -3. Dependency resolution -4. Dependency graph validation -5. Startup order calculation -6. Prerequisites validation -7. Conflict detection -8. Required services check -9. All services validation -10. Readiness report -11. Dependency tree generation -12. Reverse dependencies -13. Can-stop-service check -14. Service state initialization - -**Total Tests**: 14 comprehensive test cases - -### 14. Documentation βœ… - -**File**: `docs/user/SERVICE_MANAGEMENT_GUIDE.md` (1,200 lines) - -**Content**: -- Complete overview and architecture -- Service registry documentation -- Platform commands reference -- Service commands reference -- Deployment modes guide -- Health monitoring guide -- Dependency management guide -- Pre-flight checks guide -- Troubleshooting guide -- Advanced usage examples - -### 15. KCL Integration βœ… - -**Updated**: `provisioning/kcl/main.k` - -Added services schema import to main module. - ---- - -## Architecture Overview - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Service Management CLI β”‚ -β”‚ (platform/services commands) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ β”‚ - β–Ό β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Manager β”‚ β”‚ Lifecycle β”‚ -β”‚ (Registry, β”‚ β”‚ (Start, Stop, β”‚ -β”‚ Status, β”‚ β”‚ Multi-mode) β”‚ -β”‚ State) β”‚ β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β–Ό β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Health β”‚ β”‚ Dependencies β”‚ -β”‚ (4 check β”‚ β”‚ (Topological β”‚ -β”‚ types) β”‚ β”‚ sort) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Pre-flight β”‚ - β”‚ (Validation, β”‚ - β”‚ Auto-start) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - ---- - -## Key Features - -### 1. Unified Service Management -- Single interface for all platform services -- Consistent commands across all services -- Centralized configuration - -### 2. Automatic Dependency Resolution -- Topological sort for startup order -- Automatic dependency starting -- Circular dependency detection -- Safe stop validation - -### 3. Health Monitoring -- HTTP endpoint checks -- TCP port checks -- Command execution checks -- File existence checks -- Continuous monitoring -- Automatic retry - -### 4. Multiple Deployment Modes -- **Binary**: Native process management -- **Docker**: Container orchestration -- **Docker Compose**: Multi-container apps -- **Kubernetes**: K8s deployments with Helm -- **Remote**: Connect to remote services - -### 5. Pre-flight Checks -- Prerequisite validation -- Conflict detection -- Dependency verification -- Automatic error prevention - -### 6. State Management -- PID tracking (`~/.provisioning/services/pids/`) -- State persistence (`~/.provisioning/services/state/`) -- Log aggregation (`~/.provisioning/services/logs/`) - ---- - -## Usage Examples - -### Start Platform - -```bash -# Start all auto-start services -provisioning platform start - -# Start specific services with dependencies -provisioning platform start control-center - -# Check platform status -provisioning platform status - -# Check platform health -provisioning platform health -``` - -### Manage Individual Services - -```bash -# List all services -provisioning services list - -# Start service (with pre-flight checks) -provisioning services start orchestrator - -# Check service health -provisioning services health orchestrator - -# View service logs -provisioning services logs orchestrator --follow - -# Stop service (with dependent check) -provisioning services stop orchestrator -``` - -### Dependency Management - -```bash -# View dependency graph -provisioning services dependencies - -# View specific service dependencies -provisioning services dependencies control-center - -# Check if service can be stopped safely -nu -c "use lib_provisioning/services/mod.nu *; can-stop-service orchestrator" -``` - -### Health Monitoring - -```bash -# Continuous health monitoring -provisioning services monitor orchestrator --interval 30 - -# One-time health check -provisioning services health orchestrator -``` - -### Validation - -```bash -# Validate all services -provisioning services validate - -# Check readiness -provisioning services readiness - -# Check required services for operation -provisioning services check server -``` - ---- - -## Integration Points - -### 1. Command Dispatcher - -Pre-flight checks integrated into dispatcher: - -```nushell -# Before executing operation, check required services -let preflight = (check-required-services $task) - -if not $preflight.all_running { - if $preflight.can_auto_start { - auto-start-required-services $task - } else { - error "Required services not running" - } -} -``` - -### 2. Workflow System - -Orchestrator automatically starts when workflows are submitted: - -```bash -provisioning workflow submit my-workflow -# Orchestrator auto-starts if not running -``` - -### 3. Test Environments - -Orchestrator required for test environment operations: - -```bash -provisioning test quick kubernetes -# Orchestrator auto-starts if needed -``` - ---- - -## File Structure - -``` -provisioning/ -β”œβ”€β”€ kcl/ -β”‚ β”œβ”€β”€ services.k # KCL schemas (350 lines) -β”‚ └── main.k # Updated with services import -β”œβ”€β”€ config/ -β”‚ └── services.toml # Service registry (350 lines) -β”œβ”€β”€ core/nulib/ -β”‚ β”œβ”€β”€ lib_provisioning/services/ -β”‚ β”‚ β”œβ”€β”€ mod.nu # Module exports (15 lines) -β”‚ β”‚ β”œβ”€β”€ manager.nu # Core manager (350 lines) -β”‚ β”‚ β”œβ”€β”€ lifecycle.nu # Lifecycle mgmt (480 lines) -β”‚ β”‚ β”œβ”€β”€ health.nu # Health checks (220 lines) -β”‚ β”‚ β”œβ”€β”€ preflight.nu # Pre-flight checks (280 lines) -β”‚ β”‚ β”œβ”€β”€ dependencies.nu # Dependency resolution (310 lines) -β”‚ β”‚ └── commands.nu # CLI commands (480 lines) -β”‚ └── tests/ -β”‚ └── test_services.nu # Test suite (380 lines) -β”œβ”€β”€ platform/ -β”‚ β”œβ”€β”€ docker-compose.yaml # Docker Compose (180 lines) -β”‚ β”œβ”€β”€ coredns/ -β”‚ β”‚ β”œβ”€β”€ Corefile # CoreDNS config (35 lines) -β”‚ β”‚ └── zones/ -β”‚ β”‚ └── provisioning.zone # DNS zone (30 lines) -β”‚ └── oci-registry/ -β”‚ └── config.json # Registry config (20 lines) -└── docs/user/ - └── SERVICE_MANAGEMENT_GUIDE.md # Complete guide (1,200 lines) -``` - -**Total Implementation**: ~4,700 lines of code + documentation - ---- - -## Technical Capabilities - -### Process Management -- Background process spawning -- PID tracking and verification -- Signal handling (TERM, KILL) -- Graceful shutdown - -### Docker Integration -- Container lifecycle management -- Image pulling and building -- Port mapping and volumes -- Network configuration -- Health checks - -### Kubernetes Integration -- Deployment management -- Helm chart support -- Namespace handling -- Manifest application - -### Health Monitoring -- Multiple check protocols -- Configurable timeouts and retries -- Real-time monitoring -- Duration tracking - -### State Persistence -- JSON state files -- PID tracking -- Log rotation support -- Uptime calculation - ---- - -## Testing - -Run test suite: - -```bash -nu provisioning/core/nulib/tests/test_services.nu -``` - -**Expected Output**: -``` -=== Service Management System Tests === - -Testing: Service registry loading -βœ… Service registry loads correctly - -Testing: Service definition retrieval -βœ… Service definition retrieval works - -... - -=== Test Results === -Passed: 14 -Failed: 0 -Total: 14 - -βœ… All tests passed! -``` - ---- - -## Next Steps - -### 1. Integration Testing - -Test with actual services: - -```bash -# Build orchestrator -cd provisioning/platform/orchestrator -cargo build --release - -# Install binary -cp target/release/provisioning-orchestrator ~/.provisioning/bin/ - -# Test service management -provisioning platform start orchestrator -provisioning services health orchestrator -provisioning platform status -``` - -### 2. Docker Compose Testing - -```bash -cd provisioning/platform -docker-compose up -d -docker-compose ps -docker-compose logs -f orchestrator -``` - -### 3. End-to-End Workflow - -```bash -# Start platform -provisioning platform start - -# Create server (orchestrator auto-starts) -provisioning server create --check - -# Check all services -provisioning platform health - -# Stop platform -provisioning platform stop -``` - -### 4. Future Enhancements - -- [ ] Metrics collection (Prometheus integration) -- [ ] Alert integration (email, Slack, PagerDuty) -- [ ] Service discovery integration -- [ ] Load balancing support -- [ ] Rolling updates -- [ ] Blue-green deployments -- [ ] Service mesh integration - ---- - -## Performance Characteristics - -- **Service start time**: 5-30 seconds (depends on service) -- **Health check latency**: 5-100ms (depends on check type) -- **Dependency resolution**: <100ms for 10 services -- **State persistence**: <10ms per operation - ---- - -## Security Considerations - -- PID files in user-specific directory -- No hardcoded credentials -- TLS support for remote services -- Token-based authentication -- Docker socket access control -- Kubernetes RBAC integration - ---- - -## Compatibility - -- **Nushell**: 0.107.1+ -- **KCL**: 0.11.3+ -- **Docker**: 20.10+ -- **Docker Compose**: v2.0+ -- **Kubernetes**: 1.25+ -- **Helm**: 3.0+ - ---- - -## Success Metrics - -βœ… **Complete Implementation**: All 15 deliverables implemented -βœ… **Comprehensive Testing**: 14 test cases covering all functionality -βœ… **Production-Ready**: Error handling, logging, state management -βœ… **Well-Documented**: 1,200-line user guide with examples -βœ… **Idiomatic Code**: Follows Nushell and KCL best practices -βœ… **Extensible Architecture**: Easy to add new services and modes - ---- - -## Summary - -A complete, production-ready service management system has been implemented with: - -- **7 platform services** registered and configured -- **5 deployment modes** (binary, Docker, Docker Compose, K8s, remote) -- **4 health check types** (HTTP, TCP, command, file) -- **Automatic dependency resolution** with topological sorting -- **Pre-flight validation** preventing failures -- **Comprehensive CLI** with 15+ commands -- **Complete documentation** with troubleshooting guide -- **Full test coverage** with 14 test cases - -The system is ready for testing and integration with the existing provisioning infrastructure. - ---- - -**Implementation Status**: βœ… COMPLETE -**Ready for**: Integration Testing -**Documentation**: βœ… Complete -**Tests**: βœ… 14/14 Passing (expected) diff --git a/nulib/ai/query_processor.nu b/nulib/ai/query_processor.nu index 9356186..d67b64a 100644 --- a/nulib/ai/query_processor.nu +++ b/nulib/ai/query_processor.nu @@ -716,4 +716,4 @@ export def get_query_capabilities []: nothing -> record { troubleshooting: "Why is the web service responding slowly?" } } -} \ No newline at end of file +} diff --git a/nulib/api/routes.nu b/nulib/api/routes.nu index 8cb7ee8..5e0dd32 100644 --- a/nulib/api/routes.nu +++ b/nulib/api/routes.nu @@ -363,4 +363,4 @@ export def validate_routes []: nothing -> record { path_conflicts: $path_conflicts validation_passed: ($path_conflicts | length) == 0 } -} \ No newline at end of file +} diff --git a/nulib/api/server.nu b/nulib/api/server.nu index 3649886..399abc8 100644 --- a/nulib/api/server.nu +++ b/nulib/api/server.nu @@ -442,4 +442,4 @@ export def check_api_health [ response: $response } } -} \ No newline at end of file +} diff --git a/nulib/clusters/create.nu b/nulib/clusters/create.nu index 9592bd6..1ad8def 100644 --- a/nulib/clusters/create.nu +++ b/nulib/clusters/create.nu @@ -6,76 +6,76 @@ use utils.nu * # > Clusters services export def "main create" [ name?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # infra directory - --settings (-s): string # Settings path + ...args # Args for create command + --infra (-i): string # infra directory + --settings (-s): string # Settings path --outfile (-o): string # Output file - --cluster_pos (-p): int # Server position in settings - --check (-c) # Only check mode no clusters will be created - --wait (-w) # Wait clusters to be created - --select: string # Select with task as option + --cluster_pos (-p): int # Server position in settings + --check (-c) # Only check mode no clusters will be created + --wait (-w) # Wait clusters to be created + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "cluster create" $args #parse_help_command "cluster create" $name --ismod --end # print "on cluster main create" - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } if $name != null and $name != "h" and $name != "help" { let curr_settings = (find_get_settings --infra $infra --settings $settings) if ($curr_settings.data.clusters | find $name| length) == 0 { - _print $"πŸ›‘ invalid name ($name)" + _print $"πŸ›‘ invalid name ($name)" exit 1 } } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "create " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "create " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $"($task) " "" | str trim - let run_create = { + let run_create = { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.WK_CNPROV = $curr_settings.wk_path - let match_name = if $name == null or $name == "" { "" } else { $name} - on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos + let match_name = if $name == null or $name == "" { "" } else { $name} + on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos } match $task { - "" if $name == "h" => { + "" if $name == "h" => { ^$"($env.PROVISIONING_NAME)" -mod cluster create help --notitles }, - "" if $name == "help" => { + "" if $name == "help" => { ^$"($env.PROVISIONING_NAME)" -mod cluster create --help print (provisioning_options "create") }, - "" => { + "" => { let result = desktop_run_notify $"($env.PROVISIONING_NAME) clusters create" "-> " $run_create --timeout 11sec #do $run_create }, - _ => { - if $task != "" { print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { print $"πŸ›‘ invalid_option ($task)" } print $"\nUse (_ansi blue_bold)($env.PROVISIONING_NAME) -h(_ansi reset) for help on commands and options" } - } + } # "" | "create" - if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/clusters/create.nu.bak2 b/nulib/clusters/create.nu.bak2 index 2a7bd30..cf9357e 100644 --- a/nulib/clusters/create.nu.bak2 +++ b/nulib/clusters/create.nu.bak2 @@ -6,76 +6,76 @@ use utils.nu * # > Clusters services export def "main create" [ name?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # infra directory - --settings (-s): string # Settings path + ...args # Args for create command + --infra (-i): string # infra directory + --settings (-s): string # Settings path --outfile (-o): string # Output file - --cluster_pos (-p): int # Server position in settings - --check (-c) # Only check mode no clusters will be created - --wait (-w) # Wait clusters to be created - --select: string # Select with task as option + --cluster_pos (-p): int # Server position in settings + --check (-c) # Only check mode no clusters will be created + --wait (-w) # Wait clusters to be created + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "cluster create" $args #parse_help_command "cluster create" $name --ismod --end # print "on cluster main create" - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } if $name != null and $name != "h" and $name != "help" { let curr_settings = (find_get_settings --infra $infra --settings $settings) if ($curr_settings.data.clusters | find $name| length) == 0 { - _print $"πŸ›‘ invalid name ($name)" + _print $"πŸ›‘ invalid name ($name)" exit 1 } } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "create " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "create " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ( | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $"($task) " "" | str trim - let run_create = { + let run_create = { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.WK_CNPROV = $curr_settings.wk_path - let match_name = if $name == null or $name == "" { "" } else { $name} - on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos + let match_name = if $name == null or $name == "" { "" } else { $name} + on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos } match $task { - "" if $name == "h" => { + "" if $name == "h" => { ^$"($env.PROVISIONING_NAME)" -mod cluster create help --notitles }, - "" if $name == "help" => { + "" if $name == "help" => { ^$"($env.PROVISIONING_NAME)" -mod cluster create --help print (provisioning_options "create") }, - "" => { + "" => { let result = desktop_run_notify $"($env.PROVISIONING_NAME) clusters create" "-> " $run_create --timeout 11sec #do $run_create }, - _ => { - if $task != "" { print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { print $"πŸ›‘ invalid_option ($task)" } print $"\nUse (_ansi blue_bold)($env.PROVISIONING_NAME) -h(_ansi reset) for help on commands and options" } - } + } # "" | "create" - if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/clusters/discover.nu b/nulib/clusters/discover.nu index 71dcc93..f19f059 100644 --- a/nulib/clusters/discover.nu +++ b/nulib/clusters/discover.nu @@ -14,29 +14,29 @@ export def discover-clusters []: nothing -> list { error make { msg: $"Clusters path not found: ($clusters_path)" } } - # Find all cluster directories with KCL modules + # Find all cluster directories with Nickel modules ls $clusters_path | where type == "dir" | each { |dir| let cluster_name = ($dir.name | path basename) - let kcl_path = ($dir.name | path join "kcl") - let kcl_mod_path = ($kcl_path | path join "kcl.mod") + let schema_path = ($dir.name | path join "nickel") + let mod_path = ($schema_path | path join "nickel.mod") - if ($kcl_mod_path | path exists) { - extract_cluster_metadata $cluster_name $kcl_path + if ($mod_path | path exists) { + extract_cluster_metadata $cluster_name $schema_path } } | compact | sort-by name } -# Extract metadata from a cluster's KCL module -def extract_cluster_metadata [name: string, kcl_path: string]: nothing -> record { - let kcl_mod_path = ($kcl_path | path join "kcl.mod") - let mod_content = (open $kcl_mod_path | from toml) +# Extract metadata from a cluster's Nickel module +def extract_cluster_metadata [name: string, schema_path: string]: nothing -> record { + let mod_path = ($schema_path | path join "nickel.mod") + let mod_content = (open $mod_path | from toml) - # Find KCL schema files - let schema_files = (glob ($kcl_path | path join "*.k")) + # Find Nickel schema files + let schema_files = (glob ($schema_path | path join "*.ncl")) let main_schema = ($schema_files | where ($it | str contains $name) | first | default "") # Extract dependencies @@ -60,17 +60,17 @@ def extract_cluster_metadata [name: string, kcl_path: string]: nothing -> record type: "cluster" cluster_type: $cluster_type version: $mod_content.package.version - kcl_path: $kcl_path + schema_path: $schema_path main_schema: $main_schema dependencies: $dependencies components: $components description: $description available: true - last_updated: (ls $kcl_mod_path | get 0.modified) + last_updated: (ls $mod_path | get 0.modified) } } -# Extract description from KCL schema file +# Extract description from Nickel schema file def extract_schema_description [schema_file: string]: nothing -> string { if not ($schema_file | path exists) { return "" @@ -187,4 +187,4 @@ export def list-cluster-types []: nothing -> list { | get cluster_type | uniq | sort -} \ No newline at end of file +} diff --git a/nulib/clusters/generate.nu b/nulib/clusters/generate.nu index 67a96b3..7779f34 100644 --- a/nulib/clusters/generate.nu +++ b/nulib/clusters/generate.nu @@ -6,76 +6,76 @@ use utils.nu * # > Clusters services export def "main generate" [ name?: string # Server hostname in settings - ...args # Args for generate command - --infra (-i): string # Infra directory - --settings (-s): string # Settings path + ...args # Args for generate command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path --outfile (-o): string # Output file - --cluster_pos (-p): int # Server position in settings - --check (-c) # Only check mode no clusters will be generated - --wait (-w) # Wait clusters to be generated - --select: string # Select with task as option + --cluster_pos (-p): int # Server position in settings + --check (-c) # Only check mode no clusters will be generated + --wait (-w) # Wait clusters to be generated + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "cluster generate" $args #parse_help_command "cluster generate" $name --ismod --end # print "on cluster main generate" - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } # if $name != null and $name != "h" and $name != "help" { # let curr_settings = (find_get_settings --infra $infra --settings $settings) # if ($curr_settings.data.clusters | find $name| length) == 0 { - # _print $"πŸ›‘ invalid name ($name)" + # _print $"πŸ›‘ invalid name ($name)" # exit 1 # } # } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "generate " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "generate " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $"($task) " "" | str trim - let run_generate = { + let run_generate = { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.WK_CNPROV = $curr_settings.wk_path - let match_name = if $name == null or $name == "" { "" } else { $name} - # on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos + let match_name = if $name == null or $name == "" { "" } else { $name} + # on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos } match $task { - "" if $name == "h" => { + "" if $name == "h" => { ^$"($env.PROVISIONING_NAME)" -mod cluster generate help --notitles }, - "" if $name == "help" => { + "" if $name == "help" => { ^$"($env.PROVISIONING_NAME)" -mod cluster generate --help print (provisioning_options "generate") }, - "" => { + "" => { let result = desktop_run_notify $"($env.PROVISIONING_NAME) clusters generate" "-> " $run_generate --timeout 11sec #do $run_generate }, - _ => { - if $task != "" { print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { print $"πŸ›‘ invalid_option ($task)" } print $"\nUse (_ansi blue_bold)($env.PROVISIONING_NAME) -h(_ansi reset) for help on commands and options" } - } + } # "" | "generate" - if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/clusters/generate.nu.bak2 b/nulib/clusters/generate.nu.bak2 index c22851a..cf83a36 100644 --- a/nulib/clusters/generate.nu.bak2 +++ b/nulib/clusters/generate.nu.bak2 @@ -6,76 +6,76 @@ use utils.nu * # > Clusters services export def "main generate" [ name?: string # Server hostname in settings - ...args # Args for generate command - --infra (-i): string # Infra directory - --settings (-s): string # Settings path + ...args # Args for generate command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path --outfile (-o): string # Output file - --cluster_pos (-p): int # Server position in settings - --check (-c) # Only check mode no clusters will be generated - --wait (-w) # Wait clusters to be generated - --select: string # Select with task as option + --cluster_pos (-p): int # Server position in settings + --check (-c) # Only check mode no clusters will be generated + --wait (-w) # Wait clusters to be generated + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote clusters PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "cluster generate" $args #parse_help_command "cluster generate" $name --ismod --end # print "on cluster main generate" - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } # if $name != null and $name != "h" and $name != "help" { # let curr_settings = (find_get_settings --infra $infra --settings $settings) # if ($curr_settings.data.clusters | find $name| length) == 0 { - # _print $"πŸ›‘ invalid name ($name)" + # _print $"πŸ›‘ invalid name ($name)" # exit 1 # } # } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "generate " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = (($env.PROVISIONING_ARGS? | default "") | str replace "generate " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ( | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $"($task) " "" | str trim - let run_generate = { + let run_generate = { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.WK_CNPROV = $curr_settings.wk_path - let match_name = if $name == null or $name == "" { "" } else { $name} - # on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos + let match_name = if $name == null or $name == "" { "" } else { $name} + # on_clusters $curr_settings $check $wait $outfile $match_name $cluster_pos } match $task { - "" if $name == "h" => { + "" if $name == "h" => { ^$"($env.PROVISIONING_NAME)" -mod cluster generate help --notitles }, - "" if $name == "help" => { + "" if $name == "help" => { ^$"($env.PROVISIONING_NAME)" -mod cluster generate --help print (provisioning_options "generate") }, - "" => { + "" => { let result = desktop_run_notify $"($env.PROVISIONING_NAME) clusters generate" "-> " $run_generate --timeout 11sec #do $run_generate }, - _ => { - if $task != "" { print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { print $"πŸ›‘ invalid_option ($task)" } print $"\nUse (_ansi blue_bold)($env.PROVISIONING_NAME) -h(_ansi reset) for help on commands and options" } - } + } # "" | "generate" - if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/clusters/handlers.nu b/nulib/clusters/handlers.nu index 698d0dc..c457e73 100644 --- a/nulib/clusters/handlers.nu +++ b/nulib/clusters/handlers.nu @@ -8,7 +8,7 @@ def install_from_server [ wk_server: string ]: nothing -> bool { _print $"($defs.cluster.name) on ($defs.server.hostname) install (_ansi purple_bold)from ($defs.cluster_install_mode)(_ansi reset)" - run_cluster $defs ((get-run-clusters-path) | path join $defs.cluster.name | path join $server_cluster_path) + run_cluster $defs ((get-run-clusters-path) | path join $defs.cluster.name | path join $server_cluster_path) ($wk_server | path join $defs.cluster.name) } def install_from_library [ @@ -17,35 +17,35 @@ def install_from_library [ wk_server: string ]: nothing -> bool { _print $"($defs.cluster.name) on ($defs.server.hostname) installed (_ansi purple_bold)from library(_ansi reset)" - run_cluster $defs ((get-clusters-path) |path join $defs.cluster.name | path join $defs.cluster_profile) + run_cluster $defs ((get-clusters-path) |path join $defs.cluster.name | path join $defs.cluster_profile) ($wk_server | path join $defs.cluster.name) } export def on_clusters [ settings: record - match_cluster: string - match_server: string + match_cluster: string + match_server: string iptype: string check: bool ]: nothing -> bool { - # use ../../../providers/prov_lib/middleware.nu mw_get_ip + # use ../../../providers/prov_lib/middleware.nu mw_get_ip _print $"Running (_ansi yellow_bold)clusters(_ansi reset) ..." if (get-provisioning-use-sops) == "" { - # A SOPS load env + # A SOPS load env $env.CURRENT_INFRA_PATH = $"($settings.infra_path)/($settings.infra)" - use sops_env.nu + use sops_env.nu } let ip_type = if $iptype == "" { "public" } else { $iptype } mut server_pos = -1 mut cluster_pos = -1 mut curr_cluster = 0 - let created_clusters_dirpath = ( $settings.data.created_clusters_dirpath | default "/tmp" | - str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME | str replace "NOW" $env.NOW + let created_clusters_dirpath = ( $settings.data.created_clusters_dirpath | default "/tmp" | + str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME | str replace "NOW" $env.NOW ) let root_wk_server = ($created_clusters_dirpath | path join "on-server") if not ($root_wk_server | path exists ) { ^mkdir "-p" $root_wk_server } - let dflt_clean_created_clusters = ($settings.data.defaults_servers.clean_created_clusters? | default $created_clusters_dirpath | - str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME + let dflt_clean_created_clusters = ($settings.data.defaults_servers.clean_created_clusters? | default $created_clusters_dirpath | + str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME ) let run_ops = if (is-debug-enabled) { "bash -x" } else { "" } for srvr in $settings.data.servers { @@ -54,20 +54,20 @@ export def on_clusters [ $server_pos += 1 $cluster_pos = -1 _print $"On server ($srvr.hostname) pos ($server_pos) ..." - if $match_server != "" and $srvr.hostname != $match_server { continue } + if $match_server != "" and $srvr.hostname != $match_server { continue } let clean_created_clusters = (($settings.data.servers | try { get $server_pos).clean_created_clusters? } catch { $dflt_clean_created_clusters ) } - let ip = if (is-debug-check-enabled) { + let ip = if (is-debug-check-enabled) { "127.0.0.1" - } else { + } else { let curr_ip = (mw_get_ip $settings $srvr $ip_type false | default "") - if $curr_ip == "" { - _print $"πŸ›‘ No IP ($ip_type) found for (_ansi green_bold)($srvr.hostname)(_ansi reset) ($server_pos) " + if $curr_ip == "" { + _print $"πŸ›‘ No IP ($ip_type) found for (_ansi green_bold)($srvr.hostname)(_ansi reset) ($server_pos) " continue } - #use utils.nu wait_for_server - if not (wait_for_server $server_pos $srvr $settings $curr_ip) { - print $"πŸ›‘ server ($srvr.hostname) ($curr_ip) (_ansi red_bold)not in running state(_ansi reset)" - continue + #use utils.nu wait_for_server + if not (wait_for_server $server_pos $srvr $settings $curr_ip) { + print $"πŸ›‘ server ($srvr.hostname) ($curr_ip) (_ansi red_bold)not in running state(_ansi reset)" + continue } $curr_ip } @@ -75,36 +75,36 @@ export def on_clusters [ let wk_server = ($root_wk_server | path join $server.hostname) if ($wk_server | path exists ) { rm -rf $wk_server } ^mkdir "-p" $wk_server - for cluster in $server.clusters { - $cluster_pos += 1 + for cluster in $server.clusters { + $cluster_pos += 1 if $cluster_pos > $curr_cluster { break } $curr_cluster += 1 - if $match_cluster != "" and $match_cluster != $cluster.name { continue } - if not ((get-clusters-path) | path join $cluster.name | path exists) { - print $"cluster path: ((get-clusters-path) | path join $cluster.name) (_ansi red_bold)not found(_ansi reset)" + if $match_cluster != "" and $match_cluster != $cluster.name { continue } + if not ((get-clusters-path) | path join $cluster.name | path exists) { + print $"cluster path: ((get-clusters-path) | path join $cluster.name) (_ansi red_bold)not found(_ansi reset)" continue } if not ($wk_server | path join $cluster.name| path exists) { ^mkdir "-p" ($wk_server | path join $cluster.name) } let $cluster_profile = if $cluster.profile == "" { "default" } else { $cluster.profile } let $cluster_install_mode = if $cluster.install_mode == "" { "library" } else { $cluster.install_mode } let server_cluster_path = ($server.hostname | path join $cluster_profile) - let defs = { - settings: $settings, server: $server, cluster: $cluster, + let defs = { + settings: $settings, server: $server, cluster: $cluster, cluster_install_mode: $cluster_install_mode, cluster_profile: $cluster_profile, pos: { server: $"($server_pos)", cluster: $cluster_pos}, ip: $ip } match $cluster.install_mode { - "server" | "getfile" => { + "server" | "getfile" => { (install_from_server $defs $server_cluster_path $wk_server ) }, - "library-server" => { + "library-server" => { (install_from_library $defs $server_cluster_path $wk_server) (install_from_server $defs $server_cluster_path $wk_server ) }, - "server-library" => { + "server-library" => { (install_from_server $defs $server_cluster_path $wk_server ) (install_from_library $defs $server_cluster_path $wk_server) }, - "library" => { + "library" => { (install_from_library $defs $server_cluster_path $wk_server) }, } @@ -119,4 +119,4 @@ export def on_clusters [ #use utils.nu servers_selector servers_selector $settings $ip_type false true -} \ No newline at end of file +} diff --git a/nulib/clusters/load.nu b/nulib/clusters/load.nu index 468a952..2ebc5f7 100644 --- a/nulib/clusters/load.nu +++ b/nulib/clusters/load.nu @@ -70,8 +70,8 @@ def load-single-cluster [target_path: string, name: string, force: bool, layer: } } - # Copy KCL files and directories - cp -r $cluster_info.kcl_path $target_dir + # Copy Nickel files and directories + cp -r $cluster_info.schema_path $target_dir print $"βœ… Loaded cluster: ($name) (type: ($cluster_info.cluster_type))" { @@ -96,12 +96,12 @@ def load-single-cluster [target_path: string, name: string, force: bool, layer: } } -# Generate clusters.k import file +# Generate clusters.ncl import file def generate-clusters-imports [target_path: string, clusters: list, layer: string] { # Generate individual imports for each cluster let imports = ($clusters | each { |name| # Check if the cluster main file exists - let main_file = ($target_path | path join ".clusters" $name ($name + ".k")) + let main_file = ($target_path | path join ".clusters" $name ($name + ".ncl")) if ($main_file | path exists) { $"import .clusters.($name).($name) as ($name)_cluster" } else { @@ -130,7 +130,7 @@ clusters = { clusters" # Save the imports file - $content | save -f ($target_path | path join "clusters.k") + $content | save -f ($target_path | path join "clusters.ncl") # Also create individual alias files for easier direct imports for $name in $clusters { @@ -142,7 +142,7 @@ import .clusters.($name) as ($name) # Re-export for convenience ($name)" - $alias_content | save -f ($target_path | path join $"cluster_($name).k") + $alias_content | save -f ($target_path | path join $"cluster_($name).ncl") } } @@ -166,7 +166,7 @@ def update-clusters-manifest [target_path: string, clusters: list, layer components: $info.components layer: $layer loaded_at: (date now | format date '%Y-%m-%d %H:%M:%S') - source_path: $info.kcl_path + source_path: $info.schema_path } }) @@ -198,7 +198,7 @@ export def unload-cluster [workspace: string, name: string]: nothing -> record { if ($updated_clusters | is-empty) { rm $manifest_path - rm ($workspace | path join "clusters.k") + rm ($workspace | path join "clusters.ncl") } else { let updated_manifest = ($manifest | update loaded_clusters $updated_clusters) $updated_manifest | to yaml | save $manifest_path @@ -256,7 +256,7 @@ export def clone-cluster [ cp -r $source_dir $target_dir # Update cluster name in schema files - let schema_files = (ls ($target_dir | path join "*.k") | get name) + let schema_files = (ls ($target_dir | path join "*.ncl") | get name) for $file in $schema_files { let content = (open $file) let updated = ($content | str replace $source_name $target_name) @@ -280,4 +280,4 @@ export def clone-cluster [ status: "cloned" workspace: $workspace } -} \ No newline at end of file +} diff --git a/nulib/clusters/run.nu b/nulib/clusters/run.nu index 6e1df44..7d3de70 100644 --- a/nulib/clusters/run.nu +++ b/nulib/clusters/run.nu @@ -57,11 +57,11 @@ export def run_cluster_library [ if not ($cluster_path | path exists) { return false } let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace "~" $env.HOME) let cluster_server_name = $defs.server.hostname - rm -rf ($cluster_env_path | path join "*.k") ($cluster_env_path | path join "kcl") - mkdir ($cluster_env_path | path join "kcl") + rm -rf ($cluster_env_path | path join "*.ncl") ($cluster_env_path | path join "nickel") + mkdir ($cluster_env_path | path join "nickel") let err_out = ($cluster_env_path | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".err") | path basename) - let kcl_temp = ($cluster_env_path | path join "kcl" | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".k" ) | path basename) + let nickel_temp = ($cluster_env_path | path join "nickel" | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".ncl" ) | path basename) let wk_format = if $env.PROVISIONING_WK_FORMAT == "json" { "json" } else { "yaml" } let wk_data = { defs: $defs.settings.data, pos: $defs.pos, server: $defs.server } @@ -70,28 +70,28 @@ export def run_cluster_library [ } else { $wk_data | to yaml | save --force $wk_vars } - if $env.PROVISIONING_USE_KCL { + if $env.PROVISIONING_USE_nickel { cd ($defs.settings.infra_path | path join $defs.settings.infra) - let kcl_cluster_path = if ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) { - ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k") - } else if (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) { - (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k") + let nickel_cluster_path = if ($cluster_path | path join "nickel"| path join $"($defs.cluster.name).ncl" | path exists) { + ($cluster_path | path join "nickel"| path join $"($defs.cluster.name).ncl") + } else if (($cluster_path | path dirname) | path join "nickel"| path join $"($defs.cluster.name).ncl" | path exists) { + (($cluster_path | path dirname) | path join "nickel"| path join $"($defs.cluster.name).ncl") } else { "" } - if ($kcl_temp | path exists) { rm -f $kcl_temp } - let res = (^kcl import -m $wk_format $wk_vars -o $kcl_temp | complete) + if ($nickel_temp | path exists) { rm -f $nickel_temp } + let res = (^nickel import -m $wk_format $wk_vars -o $nickel_temp | complete) if $res.exit_code != 0 { - print $"❗KCL import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found " + print $"❗Nickel import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found " print $res.stdout - rm -f $kcl_temp + rm -f $nickel_temp cd $env.PWD return false } # Very important! Remove external block for import and re-format it - # ^sed -i "s/^{//;s/^}//" $kcl_temp - open $kcl_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $kcl_temp - ^kcl fmt $kcl_temp - if $kcl_cluster_path != "" and ($kcl_cluster_path | path exists) { cat $kcl_cluster_path | save --append $kcl_temp } - # } else { print $"❗ No cluster kcl ($defs.cluster.k) path found " ; return false } + # ^sed -i "s/^{//;s/^}//" $nickel_temp + open $nickel_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $nickel_temp + ^nickel fmt $nickel_temp + if $nickel_cluster_path != "" and ($nickel_cluster_path | path exists) { cat $nickel_cluster_path | save --append $nickel_temp } + # } else { print $"❗ No cluster nickel ($defs.cluster.ncl) path found " ; return false } if $env.PROVISIONING_KEYS_PATH != "" { #use sops on_sops let keys_path = ($defs.settings.src_path | path join $env.PROVISIONING_KEYS_PATH) @@ -103,23 +103,23 @@ export def run_cluster_library [ } return false } - (on_sops d $keys_path) | save --append $kcl_temp - if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.server.hostname | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.server.hostname| path join $"($defs.cluster.name).k" ) | save --append $kcl_temp - } else if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp - } else if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp + (on_sops d $keys_path) | save --append $nickel_temp + if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.server.hostname | path join $"($defs.cluster.name).ncl" | path exists ) { + cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.server.hostname| path join $"($defs.cluster.name).ncl" ) | save --append $nickel_temp + } else if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).ncl" | path exists ) { + cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).ncl" ) | save --append $nickel_temp + } else if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).ncl" | path exists ) { + cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).ncl" ) | save --append $nickel_temp } - let res = (^kcl $kcl_temp -o $wk_vars | complete) + let res = (^nickel $nickel_temp -o $wk_vars | complete) if $res.exit_code != 0 { - print $"❗KCL errors (_ansi red_bold)($kcl_temp)(_ansi reset) found " + print $"❗Nickel errors (_ansi red_bold)($nickel_temp)(_ansi reset) found " print $res.stdout rm -f $wk_vars cd $env.PWD return false } - rm -f $kcl_temp $err_out + rm -f $nickel_temp $err_out } else if ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).yaml" | path exists) { cat ($defs.settings.src_path | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).yaml" ) | tee { save -a $wk_vars } | ignore } @@ -147,7 +147,7 @@ export def run_cluster_library [ } } } - rm -f ($cluster_env_path | path join "kcl") ($cluster_env_path | path join "*.k") + rm -f ($cluster_env_path | path join "nickel") ($cluster_env_path | path join "*.ncl") on_template_path $cluster_env_path $wk_vars true true if ($cluster_env_path | path join $"env-($defs.cluster.name)" | path exists) { ^sed -i 's,\t,,g;s,^ ,,g;/^$/d' ($cluster_env_path | path join $"env-($defs.cluster.name)") @@ -159,7 +159,7 @@ export def run_cluster_library [ } } if not (is-debug-enabled) { - rm -f ($cluster_env_path | path join "*.j2") $err_out $kcl_temp + rm -f ($cluster_env_path | path join "*.j2") $err_out $nickel_temp } true } @@ -181,7 +181,7 @@ export def run_cluster [ if not ( $created_clusters_dirpath | path exists) { ^mkdir -p $created_clusters_dirpath } (^cp -pr $"($cluster_path)/*" $cluster_env_path) - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" + rm -rf $"($cluster_env_path)/*.ncl" $"($cluster_env_path)/nickel" let wk_vars = $"($created_clusters_dirpath)/($defs.server.hostname).yaml" # if $defs.cluster.name == "kubernetes" and ("/tmp/k8s_join.sh" | path exists) { cp -pr "/tmp/k8s_join.sh" $cluster_env_path } @@ -212,7 +212,7 @@ export def run_cluster [ if not (is-debug-enabled) { rm -f $wk_vars rm -f $err_out - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" + rm -rf $"($cluster_env_path)/*.ncl" $"($cluster_env_path)/nickel" } return true } @@ -278,7 +278,7 @@ export def run_cluster [ if not (is-debug-enabled) { rm -f $wk_vars rm -f $err_out - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" + rm -rf $"($cluster_env_path)/*.ncl" $"($cluster_env_path)/nickel" } true -} \ No newline at end of file +} diff --git a/nulib/clusters/run.nu-e b/nulib/clusters/run.nu-e deleted file mode 100644 index f5b62bc..0000000 --- a/nulib/clusters/run.nu-e +++ /dev/null @@ -1,284 +0,0 @@ -#use utils.nu cluster_get_file -#use utils/templates.nu on_template_path - -use std -use ../lib_provisioning/config/accessor.nu [is-debug-enabled, is-debug-check-enabled] - -def make_cmd_env_temp [ - defs: record - cluster_env_path: string - wk_vars: string -]: nothing -> string { - let cmd_env_temp = $"($cluster_env_path)/cmd_env_(mktemp --tmpdir-path $cluster_env_path --suffix ".sh" | path basename)" - # export all 'PROVISIONING_' $env vars to SHELL - ($"export NU_LOG_LEVEL=($env.NU_LOG_LEVEL)\n" + - ($env | items {|key, value| if ($key | str starts-with "PROVISIONING_") {echo $'export ($key)="($value)"\n'} } | compact --empty | to text) - ) | save --force $cmd_env_temp - $cmd_env_temp -} -def run_cmd [ - cmd_name: string - title: string - where: string - defs: record - cluster_env_path: string - wk_vars: string -]: nothing -> nothing { - _print $"($title) for ($defs.cluster.name) on ($defs.server.hostname) ($defs.pos.server) ..." - if $defs.check { return } - let runner = (grep "^#!" $"($cluster_env_path)/($cmd_name)" | str trim) - let run_ops = if (is-debug-enabled) { if ($runner | str contains "bash" ) { "-x" } else { "" } } else { "" } - let cmd_env_temp = make_cmd_env_temp $defs $cluster_env_path $wk_vars - if ($wk_vars | path exists) { - let run_res = if ($runner | str ends-with "bash" ) { - (^bash -c $"'source ($cmd_env_temp) ; bash ($run_ops) ($cluster_env_path)/($cmd_name) ($wk_vars) ($defs.pos.server) ($defs.pos.cluster) (^pwd)'" | complete) - } else if ($runner | str ends-with "nu" ) { - (^bash -c $"'source ($cmd_env_temp); ($env.NU) ($env.NU_ARGS) ($cluster_env_path)/($cmd_name)'" | complete) - } else { - (^bash -c $"'source ($cmd_env_temp); ($cluster_env_path)/($cmd_name) ($wk_vars)'" | complete) - } - rm -f $cmd_env_temp - if $run_res.exit_code != 0 { - (throw-error $"πŸ›‘ Error server ($defs.server.hostname) cluster ($defs.cluster.name) - ($cluster_env_path)/($cmd_name) with ($wk_vars) ($defs.pos.server) ($defs.pos.cluster) (^pwd)" - $run_res.stdout - $where --span (metadata $run_res).span) - exit 1 - } - if not (is-debug-enabled) { rm -f $"($cluster_env_path)/prepare" } - } -} -export def run_cluster_library [ - defs: record - cluster_path: string - cluster_env_path: string - wk_vars: string -]: nothing -> bool { - if not ($cluster_path | path exists) { return false } - let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace "~" $env.HOME) - let cluster_server_name = $defs.server.hostname - rm -rf ($cluster_env_path | path join "*.k") ($cluster_env_path | path join "kcl") - mkdir ($cluster_env_path | path join "kcl") - - let err_out = ($cluster_env_path | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".err") | path basename) - let kcl_temp = ($cluster_env_path | path join "kcl" | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".k" ) | path basename) - - let wk_format = if $env.PROVISIONING_WK_FORMAT == "json" { "json" } else { "yaml" } - let wk_data = { defs: $defs.settings.data, pos: $defs.pos, server: $defs.server } - if $wk_format == "json" { - $wk_data | to json | save --force $wk_vars - } else { - $wk_data | to yaml | save --force $wk_vars - } - if $env.PROVISIONING_USE_KCL { - cd ($defs.settings.infra_path | path join $defs.settings.infra) - let kcl_cluster_path = if ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) { - ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k") - } else if (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) { - (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k") - } else { "" } - if ($kcl_temp | path exists) { rm -f $kcl_temp } - let res = (^kcl import -m $wk_format $wk_vars -o $kcl_temp | complete) - if $res.exit_code != 0 { - print $"❗KCL import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found " - print $res.stdout - rm -f $kcl_temp - cd $env.PWD - return false - } - # Very important! Remove external block for import and re-format it - # ^sed -i "s/^{//;s/^}//" $kcl_temp - open $kcl_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $kcl_temp - ^kcl fmt $kcl_temp - if $kcl_cluster_path != "" and ($kcl_cluster_path | path exists) { cat $kcl_cluster_path | save --append $kcl_temp } - # } else { print $"❗ No cluster kcl ($defs.cluster.k) path found " ; return false } - if $env.PROVISIONING_KEYS_PATH != "" { - #use sops on_sops - let keys_path = ($defs.settings.src_path | path join $env.PROVISIONING_KEYS_PATH) - if not ($keys_path | path exists) { - if (is-debug-enabled) { - print $"❗Error KEYS_PATH (_ansi red_bold)($keys_path)(_ansi reset) found " - } else { - print $"❗Error (_ansi red_bold)KEYS_PATH(_ansi reset) not found " - } - return false - } - (on_sops d $keys_path) | save --append $kcl_temp - if ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $defs.server.hostname | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $defs.server.hostname| path join $"($defs.cluster.name).k" ) | save --append $kcl_temp - } else if ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp - } else if ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).k" | path exists ) { - cat ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp - } - let res = (^kcl $kcl_temp -o $wk_vars | complete) - if $res.exit_code != 0 { - print $"❗KCL errors (_ansi red_bold)($kcl_temp)(_ansi reset) found " - print $res.stdout - rm -f $wk_vars - cd $env.PWD - return false - } - rm -f $kcl_temp $err_out - } else if ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).yaml" | path exists) { - cat ($defs.settings.src_path | path join "extensions" | path join "extensions" | path join "clusters" | path join $"($defs.cluster.name).yaml" ) | tee { save -a $wk_vars } | ignore - } - cd $env.PWD - } - (^sed -i $"s/NOW/($env.NOW)/g" $wk_vars) - if $defs.cluster_install_mode == "library" { - let cluster_data = (open $wk_vars) - let verbose = if (is-debug-enabled) { true } else { false } - if $cluster_data.cluster.copy_paths? != null { - #use utils/files.nu * - for it in $cluster_data.cluster.copy_paths { - let it_list = ($it | split row "|" | default []) - let cp_source = ($it_list | get -o 0 | default "") - let cp_target = ($it_list | get -o 1 | default "") - if ($cp_source | path exists) { - copy_prov_files $cp_source ($defs.settings.infra_path | path join $defs.settings.infra) $"($cluster_env_path)/($cp_target)" false $verbose - } else if ($"($prov_resources_path)/($cp_source)" | path exists) { - copy_prov_files $prov_resources_path $cp_source $"($cluster_env_path)/($cp_target)" false $verbose - } else if ($cp_source | file exists) { - copy_prov_file $cp_source $"($cluster_env_path)/($cp_target)" $verbose - } else if ($"($prov_resources_path)/($cp_source)" | path exists) { - copy_prov_file $"($prov_resources_path)/($cp_source)" $"($cluster_env_path)/($cp_target)" $verbose - } - } - } - } - rm -f ($cluster_env_path | path join "kcl") ($cluster_env_path | path join "*.k") - on_template_path $cluster_env_path $wk_vars true true - if ($cluster_env_path | path join $"env-($defs.cluster.name)" | path exists) { - ^sed -i 's,\t,,g;s,^ ,,g;/^$/d' ($cluster_env_path | path join $"env-($defs.cluster.name)") - } - if ($cluster_env_path | path join "prepare" | path exists) { - run_cmd "prepare" "Prepare" "run_cluster_library" $defs $cluster_env_path $wk_vars - if ($cluster_env_path | path join "resources" | path exists) { - on_template_path ($cluster_env_path | path join "resources") $wk_vars false true - } - } - if not (is-debug-enabled) { - rm -f ($cluster_env_path | path join "*.j2") $err_out $kcl_temp - } - true -} -export def run_cluster [ - defs: record - cluster_path: string - env_path: string -]: nothing -> bool { - if not ($cluster_path | path exists) { return false } - if $defs.check { return } - let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace "~" $env.HOME) - let created_clusters_dirpath = ($defs.settings.data.created_clusters_dirpath | default "/tmp" | - str replace "~" $env.HOME | str replace "NOW" $env.NOW | str replace "./" $"($defs.settings.src_path)/") - let cluster_server_name = $defs.server.hostname - - let cluster_env_path = if $defs.cluster_install_mode == "server" { $"($env_path)_($defs.cluster_install_mode)" } else { $env_path } - - if not ( $cluster_env_path | path exists) { ^mkdir -p $cluster_env_path } - if not ( $created_clusters_dirpath | path exists) { ^mkdir -p $created_clusters_dirpath } - - (^cp -pr $"($cluster_path)/*" $cluster_env_path) - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" - - let wk_vars = $"($created_clusters_dirpath)/($defs.server.hostname).yaml" - # if $defs.cluster.name == "kubernetes" and ("/tmp/k8s_join.sh" | path exists) { cp -pr "/tmp/k8s_join.sh" $cluster_env_path } - let require_j2 = (^ls ($cluster_env_path | path join "*.j2") err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })) - - - let res = if $defs.cluster_install_mode == "library" or $require_j2 != "" { - (run_cluster_library $defs $cluster_path $cluster_env_path $wk_vars) - } - if not $res { - if not (is-debug-enabled) { rm -f $wk_vars } - return $res - } - let err_out = ($env_path | path join (mktemp --tmpdir-path $env_path --suffix ".err") | path basename) - let tar_ops = if (is-debug-enabled) { "v" } else { "" } - let bash_ops = if (is-debug-enabled) { "bash -x" } else { "" } - - let res_tar = (^tar -C $cluster_env_path $"-c($tar_ops)zf" $"/tmp/($defs.cluster.name).tar.gz" . | complete) - if $res_tar.exit_code != 0 { - _print ( - $"πŸ›‘ Error (_ansi red_bold)tar cluster(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset)" + - $" cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) ($cluster_env_path) -> /tmp/($defs.cluster.name).tar.gz" - ) - _print $res_tar.stdout - return false - } - if $defs.check { - if not (is-debug-enabled) { - rm -f $wk_vars - rm -f $err_out - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" - } - return true - } - let is_local = (^ip addr | grep "inet " | grep "$defs.ip") - if $is_local != "" and not (is-debug-check-enabled) { - if $defs.cluster_install_mode == "getfile" { - if (cluster_get_file $defs.settings $defs.cluster $defs.server $defs.ip true true) { return false } - return true - } - rm -rf $"/tmp/($defs.cluster.name)" - mkdir $"/tmp/($defs.cluster.name)" - cd $"/tmp/($defs.cluster.name)" - tar x($tar_ops)zf $"/tmp/($defs.cluster.name).tar.gz" - let res_run = (^sudo $bash_ops $"./install-($defs.cluster.name).sh" err> $err_out | complete) - if $res_run.exit_code != 0 { - (throw-error $"πŸ›‘ Error server ($defs.server.hostname) cluster ($defs.cluster.name) - ./install-($defs.cluster.name).sh ($defs.server_pos) ($defs.cluster_pos) (^pwd)" - $"($res_run.stdout)\n(cat $err_out)" - "run_cluster_library" --span (metadata $res_run).span) - exit 1 - } - fi - rm -fr $"/tmp/($defs.cluster.name).tar.gz" $"/tmp/($defs.cluster.name)" - } else { - if $defs.cluster_install_mode == "getfile" { - if (cluster_get_file $defs.settings $defs.cluster $defs.server $defs.ip true false) { return false } - return true - } - if not (is-debug-check-enabled) { - #use ssh.nu * - let scp_list: list = ([] | append $"/tmp/($defs.cluster.name).tar.gz") - if not (scp_to $defs.settings $defs.server $scp_list "/tmp" $defs.ip) { - _print ( - $"πŸ›‘ Error (_ansi red_bold)ssh_cp(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " + - $" cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) /tmp/($defs.cluster.name).tar.gz" - ) - return false - } - let cmd = ( - $"rm -rf /tmp/($defs.cluster.name) ; mkdir /tmp/($defs.cluster.name) ; cd /tmp/($defs.cluster.name) ;" + - $" sudo tar x($tar_ops)zf /tmp/($defs.cluster.name).tar.gz;" + - $" sudo ($bash_ops) ./install-($defs.cluster.name).sh " # ($env.PROVISIONING_MATCH_CMD) " - ) - if not (ssh_cmd $defs.settings $defs.server true $cmd $defs.ip) { - _print ( - $"πŸ›‘ Error (_ansi red_bold)ssh_cmd(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " + - $" cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) install_($defs.cluster.name).sh" - ) - return false - } - # if $defs.cluster.name == "kubernetes" { let _res_k8s = (scp_from $defs.settings $defs.server "/tmp/k8s_join.sh" "/tmp" $defs.ip) } - if not (is-debug-enabled) { - let rm_cmd = $"sudo rm -f /tmp/($defs.cluster.name).tar.gz; sudo rm -rf /tmp/($defs.cluster.name)" - let _res = (ssh_cmd $defs.settings $defs.server true $rm_cmd $defs.ip) - rm -f $"/tmp/($defs.cluster.name).tar.gz" - } - } - } - if ($"($cluster_path)/postrun" | path exists ) { - cp $"($cluster_path)/postrun" $"($cluster_env_path)/postrun" - run_cmd "postrun" "PostRune" "run_cluster_library" $defs $cluster_env_path $wk_vars - } - if not (is-debug-enabled) { - rm -f $wk_vars - rm -f $err_out - rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl" - } - true -} diff --git a/nulib/clusters/utils.nu b/nulib/clusters/utils.nu index e3b2ab4..1520a48 100644 --- a/nulib/clusters/utils.nu +++ b/nulib/clusters/utils.nu @@ -1,61 +1,61 @@ -#use ssh.nu * +#use ssh.nu * export def cluster_get_file [ settings: record cluster: record server: record - live_ip: string + live_ip: string req_sudo: bool local_mode: bool ]: nothing -> bool { let target_path = ($cluster.target_path | default "") - if $target_path == "" { + if $target_path == "" { _print $"πŸ›‘ No (_ansi red_bold)target_path(_ansi reset) found in ($server.hostname) cluster ($cluster.name)" return false } let source_path = ($cluster.soruce_path | default "") - if $source_path == "" { + if $source_path == "" { _print $"πŸ›‘ No (_ansi red_bold)source_path(_ansi reset) found in ($server.hostname) cluster ($cluster.name)" return false } - if $local_mode { - let res = (^cp $source_path $target_path | combine) - if $res.exit_code != 0 { + if $local_mode { + let res = (^cp $source_path $target_path | combine) + if $res.exit_code != 0 { _print $"πŸ›‘ Error get_file [ local-mode ] (_ansi red_bold)($source_path) to ($target_path)(_ansi reset) in ($server.hostname) cluster ($cluster.name)" _print $res.stdout return false - } + } return true } let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } let ssh_key_path = ($server.ssh_key_path | default "") - if $ssh_key_path == "" { + if $ssh_key_path == "" { _print $"πŸ›‘ No (_ansi red_bold)ssh_key_path(_ansi reset) found in ($server.hostname) cluster ($cluster.name)" return false } - if not ($ssh_key_path | path exists) { + if not ($ssh_key_path | path exists) { _print $"πŸ›‘ Error (_ansi red_bold)($ssh_key_path)(_ansi reset) not found for ($server.hostname) cluster ($cluster.name)" return false } mut cmd = if $req_sudo { "sudo" } else { "" } let wk_path = $"/home/($env.SSH_USER)/($source_path| path basename)" - $cmd = $"($cmd) cp ($source_path) ($wk_path); sudo chown ($env.SSH_USER) ($wk_path)" + $cmd = $"($cmd) cp ($source_path) ($wk_path); sudo chown ($env.SSH_USER) ($wk_path)" let wk_path = $"/home/($env.SSH_USER)/($source_path | path basename)" let res = (ssh_cmd $settings $server false $cmd $ip ) if not $res { return false } if not (scp_from $settings $server $wk_path $target_path $ip ) { return false } - let rm_cmd = if $req_sudo { - $"sudo rm -f ($wk_path)" - } else { - $"rm -f ($wk_path)" + let rm_cmd = if $req_sudo { + $"sudo rm -f ($wk_path)" + } else { + $"rm -f ($wk_path)" } return (ssh_cmd $settings $server false $rm_cmd $ip ) } diff --git a/nulib/dashboard/marimo_integration.nu b/nulib/dashboard/marimo_integration.nu index cbef47e..0774b77 100644 --- a/nulib/dashboard/marimo_integration.nu +++ b/nulib/dashboard/marimo_integration.nu @@ -498,4 +498,4 @@ export def main [ print " ai-insights - AI-powered insights dashboard" } } -} \ No newline at end of file +} diff --git a/nulib/dataframes/log_processor.nu b/nulib/dataframes/log_processor.nu index c7d42ce..7490c34 100644 --- a/nulib/dataframes/log_processor.nu +++ b/nulib/dataframes/log_processor.nu @@ -544,4 +544,4 @@ export def monitor_logs [ sleep 60sec # Check every minute } } -} \ No newline at end of file +} diff --git a/nulib/dataframes/polars_integration.nu b/nulib/dataframes/polars_integration.nu index 906c492..8d9e7fc 100644 --- a/nulib/dataframes/polars_integration.nu +++ b/nulib/dataframes/polars_integration.nu @@ -503,4 +503,4 @@ def benchmark_polars_operations [data: list, ops: list]: nothing -> any } $df -} \ No newline at end of file +} diff --git a/nulib/demo_ai.nu b/nulib/demo_ai.nu index 8645ea9..89b8efb 100644 --- a/nulib/demo_ai.nu +++ b/nulib/demo_ai.nu @@ -4,20 +4,20 @@ print "πŸ€– AI Integration FIXED & READY!" print "===============================" print "" print "βœ… Status: All syntax errors resolved" -print "βœ… Core functionality: AI library working" +print "βœ… Core functionality: AI library working" print "βœ… Implementation: All features completed" print "" print "πŸ“‹ What was implemented:" print " 1. Template Generation: AI-powered configs" print " 2. Natural Language Queries: --ai_query flag" -print " 3. Plugin Architecture: OpenAI/Claude/Generic" +print " 3. Plugin Architecture: OpenAI/Claude/Generic" print " 4. Webhook Integration: Chat platforms" print "" print "πŸ”§ To enable, set environment variable:" print " export OPENAI_API_KEY='your-key'" print " export ANTHROPIC_API_KEY='your-key'" -print " export LLM_API_KEY='your-key'" +print " export LLM_API_KEY='your-key'" print "" -print " And enable in KCL: ai.enabled = true" +print " And enable in Nickel: ai.enabled = true" print "" print "🎯 AI integration COMPLETE!" diff --git a/nulib/env.nu b/nulib/env.nu index f132c4b..fad68ee 100644 --- a/nulib/env.nu +++ b/nulib/env.nu @@ -29,7 +29,9 @@ export-env { ($env.PROVISIONING_KLOUD_PATH? | default "") } - let config = (get-config) + # Don't load config during export-env to avoid hanging on module parsing + # Config will be loaded on-demand when accessed later + let config = {} # Try to get PROVISIONING path from config, environment, or detect from project structure let provisioning_from_config = (config-get "provisioning.path" "" --config $config) @@ -100,7 +102,7 @@ export-env { $env.PROVISIONING_INFRA_PATH = ($env.PROVISIONING_KLOUD_PATH? | default (config-get "paths.infra" | default $env.PWD ) | into string) - $env.PROVISIONING_DFLT_SET = (config-get "paths.files.settings" | default "settings.k" | into string) + $env.PROVISIONING_DFLT_SET = (config-get "paths.files.settings" | default "settings.ncl" | into string) $env.NOW = (date now | format date "%Y_%m_%d_%H_%M_%S") $env.PROVISIONING_MATCH_DATE = ($env.PROVISIONING_MATCH_DATE? | default "%Y_%m") @@ -120,10 +122,10 @@ export-env { $env.PROVISIONING_GENERATE_DIRPATH = "generate" $env.PROVISIONING_GENERATE_DEFSFILE = "defs.toml" - $env.PROVISIONING_KEYS_PATH = (config-get "paths.files.keys" ".keys.k" --config $config) + $env.PROVISIONING_KEYS_PATH = (config-get "paths.files.keys" ".keys.ncl" --config $config) - $env.PROVISIONING_USE_KCL = if (^bash -c "type -P kcl" | is-not-empty) { true } else { false } - $env.PROVISIONING_USE_KCL_PLUGIN = if ( (version).installed_plugins | str contains "kcl" ) { true } else { false } + $env.PROVISIONING_USE_nickel = if (^bash -c "type -P nickel" | is-not-empty) { true } else { false } + $env.PROVISIONING_USE_NICKEL_PLUGIN = if ( (version).installed_plugins | str contains "nickel" ) { true } else { false } #$env.PROVISIONING_J2_PARSER = ($env.PROVISIONING_$TOOLS_PATH | path join "parsetemplate.py") #$env.PROVISIONING_J2_PARSER = (^bash -c "type -P tera") $env.PROVISIONING_USE_TERA_PLUGIN = if ( (version).installed_plugins | str contains "tera" ) { true } else { false } @@ -151,12 +153,15 @@ export-env { $env.PROVISIONING_USE_SOPS = (config-get "sops.use_sops" | default "age" | into string) $env.PROVISIONING_USE_KMS = (config-get "sops.use_kms" | default "" | into string) $env.PROVISIONING_SECRET_PROVIDER = (config-get "sops.secret_provider" | default "sops" | into string) - + # AI Configuration $env.PROVISIONING_AI_ENABLED = (config-get "ai.enabled" | default false | into bool | into string) $env.PROVISIONING_AI_PROVIDER = (config-get "ai.provider" | default "openai" | into string) $env.PROVISIONING_LAST_ERROR = "" + # CLI Daemon Configuration + $env.PROVISIONING_DAEMON_URL = ($env.PROVISIONING_DAEMON_URL? | default "http://localhost:9091" | into string) + # For SOPS if settings below fails -> look at: sops_env.nu loaded when is need to set env context let curr_infra = (config-get "paths.infra" "" --config $config) @@ -196,10 +201,10 @@ export-env { # $env.PROVISIONING_NO_TERMINAL = true # } } - # KCL Module Path Configuration - # Set up KCL_MOD_PATH to help KCL resolve modules when running from different directories - $env.KCL_MOD_PATH = ($env.KCL_MOD_PATH? | default [] | append [ - ($env.PROVISIONING | path join "kcl") + # Nickel Module Path Configuration + # Set up NICKEL_IMPORT_PATH to help Nickel resolve modules when running from different directories + $env.NICKEL_IMPORT_PATH = ($env.NICKEL_IMPORT_PATH? | default [] | append [ + ($env.PROVISIONING | path join "nickel") ($env.PROVISIONING_PROVIDERS_PATH) $env.PWD ] | uniq | str join ":") @@ -242,6 +247,12 @@ export-env { # Load providers environment settings... # use ../../providers/prov_lib/env_middleware.nu + + # Auto-load tera plugin if available for template rendering at env initialization + # Call this in a block that runs AFTER the export-env completes + if ( (version).installed_plugins | str contains "tera" ) { + (plugin use tera) + } } export def "show_env" [ @@ -293,7 +304,7 @@ export def "show_env" [ PROVISIONING_KEYS_PATH: $env.PROVISIONING_KEYS_PATH, - PROVISIONING_USE_KCL: $"($env.PROVISIONING_USE_KCL)", + PROVISIONING_USE_nickel: $"($env.PROVISIONING_USE_nickel)", PROVISIONING_J2_PARSER: ($env.PROVISIONING_J2_PARSER? | default ""), PROVISIONING_URL: $env.PROVISIONING_URL, @@ -318,4 +329,10 @@ export def "show_env" [ } else { $env_vars } -} \ No newline at end of file +} + +# Get CLI daemon URL for template rendering and other daemon operations +# Returns the daemon endpoint, checking environment variable first, then default +export def get-cli-daemon-url [] { + $env.PROVISIONING_DAEMON_URL? | default "http://localhost:9091" +} diff --git a/nulib/help_minimal.nu b/nulib/help_minimal.nu index 97843a2..7a12262 100644 --- a/nulib/help_minimal.nu +++ b/nulib/help_minimal.nu @@ -313,13 +313,13 @@ def help-utilities []: nothing -> string { " provisioning ssh - Connect to server\n\n" + (ansi cyan) + "Cache Features:" + (ansi rst) + "\n" + - " β€’ Intelligent TTL management (KCL: 30m, SOPS: 15m, Final: 5m)\n" + + " β€’ Intelligent TTL management (Nickel: 30m, SOPS: 15m, Final: 5m)\n" + " β€’ 95-98% faster config loading\n" + " β€’ SOPS cache with 0600 permissions\n" + " β€’ Works without active workspace\n\n" + (ansi cyan) + "Cache Configuration:" + (ansi rst) + "\n" + - " provisioning cache config set ttl_kcl 3000 # Set KCL TTL\n" + + " provisioning cache config set ttl_nickel 3000 # Set Nickel TTL\n" + " provisioning cache config set enabled false # Disable cache\n" ) } diff --git a/nulib/infras/utils.nu b/nulib/infras/utils.nu index 04deb53..26e3d99 100644 --- a/nulib/infras/utils.nu +++ b/nulib/infras/utils.nu @@ -37,9 +37,9 @@ export def "main list" [ # List directory contents, filter for directories that: # 1. Do not start with underscore (not hidden/system) # 2. Are directories - # 3. Contain a settings.k file (marks it as a real infra) + # 3. Contain a settings.ncl file (marks it as a real infra) let infras = (ls -s $infra_dir | where {|it| - ((($it.name | str starts-with "_") == false) and ($it.type == "dir") and (($infra_dir | path join $it.name "settings.k") | path exists)) + ((($it.name | str starts-with "_") == false) and ($it.type == "dir") and (($infra_dir | path join $it.name "settings.ncl") | path exists)) } | each {|it| $it.name} | sort) if ($infras | length) > 0 { @@ -109,7 +109,7 @@ export def "main validate" [ # List available infras if ($infra_dir | path exists) { let infras = (ls -s $infra_dir | where {|it| - ((($it.name | str starts-with "_") == false) and ($it.type == "dir") and (($infra_dir | path join $it.name "settings.k") | path exists)) + ((($it.name | str starts-with "_") == false) and ($it.type == "dir") and (($infra_dir | path join $it.name "settings.ncl") | path exists)) } | each {|it| $it.name} | sort) for infra in $infras { @@ -127,8 +127,8 @@ export def "main validate" [ } # Load infrastructure configuration files - let settings_file = ($target_path | path join "settings.k") - let servers_file = ($target_path | path join "defs" "servers.k") + let settings_file = ($target_path | path join "settings.ncl") + let servers_file = ($target_path | path join "defs" "servers.ncl") if not ($settings_file | path exists) { _print $"❌ Settings file not found: ($settings_file)" diff --git a/nulib/lib_minimal.nu b/nulib/lib_minimal.nu new file mode 100644 index 0000000..ee07761 --- /dev/null +++ b/nulib/lib_minimal.nu @@ -0,0 +1,167 @@ +#!/usr/bin/env nu +# Minimal Library - Fast path for interactive commands +# NO config loading, NO platform bootstrap +# Follows: @.claude/guidelines/nushell/NUSHELL_GUIDELINES.md + +# Get user config path (centralized location) +# Rule 2: Single purpose function +# Cross-platform support (macOS, Linux, Windows) +def get-user-config-path []: nothing -> string { + let home = $env.HOME + let os_name = (uname | get operating-system | str downcase) + + let config_path = match $os_name { + "darwin" => $"($home)/Library/Application Support/provisioning/user_config.yaml", + _ => $"($home)/.config/provisioning/user_config.yaml" + } + + $config_path | path expand +} + +# List all registered workspaces +# Rule 1: Explicit types, Rule 4: Early returns +# Rule 2: Single purpose - only list workspaces +export def workspace-list []: nothing -> list { + let user_config = (get-user-config-path) + + # Rule 4: Early return if config doesn't exist + if not ($user_config | path exists) { + print "No workspaces configured yet." + return [] + } + + # Rule 15: Atomic read operation + # Rule 13: Try-catch for I/O operations + let config = (try { + open $user_config + } catch {|err| + print "Error reading user config: $err.msg" + return [] + }) + + let active = ($config | get --optional active_workspace | default "") + let workspaces = ($config | get --optional workspaces | default []) + + # Rule 8: Pure transformation (no side effects) + if ($workspaces | length) == 0 { + print "No workspaces registered." + return [] + } + + $workspaces | each {|ws| + { + name: $ws.name + path: $ws.path + active: ($ws.name == $active) + last_used: ($ws | get --optional last_used | default "Never") + } + } +} + +# Get active workspace name +# Rule 1: Explicit types, Rule 4: Early returns +export def workspace-active []: nothing -> string { + let user_config = (get-user-config-path) + + # Rule 4: Early return + if not ($user_config | path exists) { + return "" + } + + # Rule 15: Atomic read, Rule 8: Pure function + try { + open $user_config | get --optional active_workspace | default "" + } catch { + "" + } +} + +# Get workspace info by name +# Rule 1: Explicit types, Rule 4: Early returns +export def workspace-info [name: string]: nothing -> record { + let user_config = (get-user-config-path) + + # Rule 4: Early return if config doesn't exist + if not ($user_config | path exists) { + return { name: $name, path: "", exists: false } + } + + # Rule 15: Atomic read operation + let config = (try { + open $user_config + } catch { + return { name: $name, path: "", exists: false } + }) + + let workspaces = ($config | get --optional workspaces | default []) + let ws = ($workspaces | where { $in.name == $name } | first) + + if ($ws | is-empty) { + return { name: $name, path: "", exists: false } + } + + # Rule 8: Pure transformation + { + name: $ws.name + path: $ws.path + exists: true + last_used: ($ws | get --optional last_used | default "Never") + } +} + +# Quick status check (orchestrator health + active workspace) +# Rule 1: Explicit types, Rule 13: Appropriate error handling +export def status-quick []: nothing -> record { + # Direct HTTP check (no bootstrap overhead) + # Rule 13: Use try-catch for network operations + let orch_health = (try { + http get --max-time 2sec "http://localhost:9090/health" + } catch {|err| + null + }) + + let orch_status = if ($orch_health != null) { + "running" + } else { + "stopped" + } + + let active_ws = (workspace-active) + + # Rule 8: Pure transformation + { + orchestrator: $orch_status + workspace: $active_ws + timestamp: (date now | format date "%Y-%m-%d %H:%M:%S") + } +} + +# Display essential environment variables +# Rule 1: Explicit types, Rule 8: Pure function (read-only) +export def env-quick []: nothing -> record { + # Rule 8: No side effects, just reading env vars + { + PROVISIONING_ROOT: ($env.PROVISIONING_ROOT? | default "not set") + PROVISIONING_ENV: ($env.PROVISIONING_ENV? | default "not set") + PROVISIONING_DEBUG: ($env.PROVISIONING_DEBUG? | default "false") + HOME: $env.HOME + PWD: $env.PWD + } +} + +# Show quick help for fast-path commands +# Rule 1: Explicit types, Rule 8: Pure function +export def quick-help []: nothing -> string { + "Provisioning CLI - Fast Path Commands + +Quick Commands (< 100ms): + workspace list List all registered workspaces + workspace active Show currently active workspace + status Quick health check + env Show essential environment variables + help [command] Show help for a command + +For full help: + provisioning help Show all available commands + provisioning help Show help for specific command" +} diff --git a/nulib/lib_provisioning/ai/README.md b/nulib/lib_provisioning/ai/README.md index 5490637..fb448f6 100644 --- a/nulib/lib_provisioning/ai/README.md +++ b/nulib/lib_provisioning/ai/README.md @@ -5,20 +5,23 @@ This module provides comprehensive AI capabilities for the provisioning system, ## Features ### πŸ€– **Core AI Capabilities** -- Natural language KCL file generation + +- Natural language Nickel file generation - Intelligent template creation - Infrastructure query processing - Configuration validation and improvement - Chat/webhook integration -### πŸ“ **KCL Generation Types** -- **Server Configurations** (`servers.k`) - Generate server definitions with storage, networking, and services -- **Provider Defaults** (`*_defaults.k`) - Create provider-specific default settings -- **Settings Configuration** (`settings.k`) - Generate main infrastructure settings +### πŸ“ **Nickel Generation Types** + +- **Server Configurations** (`servers.ncl`) - Generate server definitions with storage, networking, and services +- **Provider Defaults** (`*_defaults.ncl`) - Create provider-specific default settings +- **Settings Configuration** (`settings.ncl`) - Generate main infrastructure settings - **Cluster Configuration** - Kubernetes and container orchestration setups - **Task Services** - Individual service configurations ### πŸ”§ **AI Providers Supported** + - **OpenAI** (GPT-4, GPT-3.5) - **Anthropic Claude** (Claude-3.5 Sonnet, Claude-3) - **Generic/Local** (Ollama, local LLM APIs) @@ -26,6 +29,7 @@ This module provides comprehensive AI capabilities for the provisioning system, ## Configuration ### Environment Variables + ```bash # Enable AI functionality export PROVISIONING_AI_ENABLED=true @@ -42,10 +46,11 @@ export LLM_API_KEY="your-generic-api-key" export PROVISIONING_AI_MODEL="gpt-4" export PROVISIONING_AI_TEMPERATURE="0.3" export PROVISIONING_AI_MAX_TOKENS="2048" -``` +```plaintext -### KCL Configuration -```kcl +### Nickel Configuration + +```nickel import settings settings.Settings { @@ -60,9 +65,10 @@ settings.Settings { enable_webhook_ai = False } } -``` +```plaintext ### YAML Configuration (`ai.yaml`) + ```yaml enabled: true provider: "openai" @@ -73,33 +79,35 @@ timeout: 30 enable_template_ai: true enable_query_ai: true enable_webhook_ai: false -``` +```plaintext ## Usage ### 🎯 **Command Line Interface** #### Generate Infrastructure with AI + ```bash # Interactive generation ./provisioning ai generate --interactive # Generate specific configurations -./provisioning ai gen -t server -p upcloud -i "3 Kubernetes nodes with Ceph storage" -o servers.k -./provisioning ai gen -t defaults -p aws -i "Production environment in us-west-2" -o aws_defaults.k -./provisioning ai gen -t settings -i "E-commerce platform with secrets management" -o settings.k +./provisioning ai gen -t server -p upcloud -i "3 Kubernetes nodes with Ceph storage" -o servers.ncl +./provisioning ai gen -t defaults -p aws -i "Production environment in us-west-2" -o aws_defaults.ncl +./provisioning ai gen -t settings -i "E-commerce platform with secrets management" -o settings.ncl # Enhanced generation with validation ./provisioning generate-ai servers "High-availability Kubernetes cluster with 3 control planes and 5 workers" --validate --provider upcloud # Improve existing configurations -./provisioning ai improve -i existing_servers.k -o improved_servers.k +./provisioning ai improve -i existing_servers.ncl -o improved_servers.ncl -# Validate and fix KCL files -./provisioning ai validate -i servers.k -``` +# Validate and fix Nickel files +./provisioning ai validate -i servers.ncl +```plaintext #### Interactive AI Chat + ```bash # Start chat session ./provisioning ai chat @@ -112,25 +120,27 @@ enable_webhook_ai: false # Show configuration ./provisioning ai config -``` +```plaintext ### 🧠 **Programmatic API** -#### Generate KCL Files +#### Generate Nickel Files + ```nushell use lib_provisioning/ai/templates.nu * # Generate server configuration -let servers = (generate_server_kcl "3 Kubernetes nodes for production workloads" "upcloud" "servers.k") +let servers = (generate_server_nickel "3 Kubernetes nodes for production workloads" "upcloud" "servers.ncl") # Generate provider defaults -let defaults = (generate_defaults_kcl "High-availability setup in EU region" "aws" "aws_defaults.k") +let defaults = (generate_defaults_nickel "High-availability setup in EU region" "aws" "aws_defaults.ncl") # Generate complete infrastructure let result = (generate_full_infra_ai "E-commerce platform with database and caching" "upcloud" "" false) -``` +```plaintext #### Process Natural Language Queries + ```nushell use lib_provisioning/ai/lib.nu * @@ -141,12 +151,13 @@ let response = (ai_process_query "Show me all servers with high CPU usage") let template = (ai_generate_template "Docker Swarm cluster with monitoring" "cluster") # Validate configurations -let validation = (validate_and_fix_kcl "servers.k") -``` +let validation = (validate_and_fix_nickel "servers.ncl") +```plaintext ### 🌐 **Webhook Integration** #### HTTP Webhook + ```bash curl -X POST http://your-server/webhook \ -H "Content-Type: application/json" \ @@ -155,9 +166,10 @@ curl -X POST http://your-server/webhook \ "user_id": "user123", "channel": "infrastructure" }' -``` +```plaintext #### Slack Integration + ```nushell # Process Slack webhook payload let slack_payload = { @@ -167,9 +179,10 @@ let slack_payload = { } let response = (process_slack_webhook $slack_payload) -``` +```plaintext #### Discord Integration + ```nushell # Process Discord webhook let discord_payload = { @@ -179,13 +192,14 @@ let discord_payload = { } let response = (process_discord_webhook $discord_payload) -``` +```plaintext ## Examples ### πŸ—οΈ **Infrastructure Generation Examples** #### 1. Kubernetes Cluster Setup + ```bash ./provisioning generate-ai servers " High-availability Kubernetes cluster with: @@ -194,10 +208,11 @@ High-availability Kubernetes cluster with: - Dedicated storage nodes with Ceph - Private networking with load balancer - Monitoring and logging stack -" --provider upcloud --output k8s_cluster_servers.k --validate -``` +" --provider upcloud --output k8s_cluster_servers.ncl --validate +```plaintext #### 2. AWS Production Environment + ```bash ./provisioning generate-ai defaults " AWS production environment configuration: @@ -209,10 +224,11 @@ AWS production environment configuration: - ElastiCache for caching - CloudFront CDN - Route53 DNS management -" --provider aws --output aws_prod_defaults.k -``` +" --provider aws --output aws_prod_defaults.ncl +```plaintext #### 3. Development Environment + ```bash ./provisioning generate-ai infra " Development environment for a microservices application: @@ -224,7 +240,7 @@ Development environment for a microservices application: - Development tools (Git, CI/CD agents) - Monitoring (Prometheus, Grafana) " --provider local --interactive -``` +```plaintext ### πŸ’¬ **Chat Examples** @@ -244,7 +260,7 @@ Development environment for a microservices application: **AI:** *"Perfect! I'll generate an UpCloud configuration with monitoring. Here's your infrastructure setup:* -```kcl +```nickel import upcloud_prov servers = [ // Load balancer @@ -257,16 +273,17 @@ servers = [ // Database servers with replication // Monitoring stack with Prometheus/Grafana ] -``` +```plaintext *This configuration includes 7 servers optimized for high availability and performance. Would you like me to explain any specific part or generate additional configurations?"* ### πŸš€ **Advanced Features** #### Interactive Configuration Builder + ```bash ./provisioning ai generate --interactive -``` +```plaintext This launches an interactive session that asks specific questions to build optimal configurations: @@ -278,30 +295,31 @@ This launches an interactive session that asks specific questions to build optim 6. **Budget Constraints** - Cost optimization preferences #### Configuration Optimization + ```bash # Analyze and improve existing configurations -./provisioning ai improve existing_config.k --output optimized_config.k +./provisioning ai improve existing_config.ncl --output optimized_config.ncl # Get AI suggestions for performance improvements -./provisioning ai query --prompt "How can I optimize this configuration for better performance?" --context file:servers.k -``` +./provisioning ai query --prompt "How can I optimize this configuration for better performance?" --context file:servers.ncl +```plaintext ## Integration with Existing Workflows ### πŸ”„ **Workflow Integration** 1. **Generate** configurations with AI -2. **Validate** using KCL compiler +2. **Validate** using Nickel compiler 3. **Review** and customize as needed 4. **Apply** using provisioning commands 5. **Monitor** and iterate ```bash # Complete workflow example -./provisioning generate-ai servers "Production Kubernetes cluster" --validate --output servers.k +./provisioning generate-ai servers "Production Kubernetes cluster" --validate --output servers.ncl ./provisioning server create --check # Review before creation ./provisioning server create # Actually create infrastructure -``` +```plaintext ### πŸ›‘οΈ **Security & Best Practices** @@ -322,33 +340,36 @@ This launches an interactive session that asks specific questions to build optim # Debug mode for troubleshooting ./provisioning generate-ai servers "test setup" --debug -``` +```plaintext ## Architecture ### πŸ—οΈ **Module Structure** -``` + +```plaintext ai/ β”œβ”€β”€ lib.nu # Core AI functionality and API integration -β”œβ”€β”€ templates.nu # KCL template generation functions +β”œβ”€β”€ templates.nu # Nickel template generation functions β”œβ”€β”€ webhook.nu # Chat/webhook processing β”œβ”€β”€ mod.nu # Module exports └── README.md # This documentation -``` +```plaintext ### πŸ”Œ **Integration Points** + - **Settings System** - AI configuration management - **Secrets Management** - Integration with SOPS/KMS for secure API keys - **Template Engine** - Enhanced with AI-generated content -- **Validation System** - Automated KCL syntax checking +- **Validation System** - Automated Nickel syntax checking - **CLI Commands** - Natural language command processing ### 🌊 **Data Flow** + 1. **Input** - Natural language description or chat message 2. **Intent Detection** - Parse and understand user requirements 3. **Context Building** - Gather relevant infrastructure context -4. **AI Processing** - Generate appropriate KCL configurations +4. **AI Processing** - Generate appropriate Nickel configurations 5. **Validation** - Syntax and semantic validation -6. **Output** - Formatted KCL files and user feedback +6. **Output** - Formatted Nickel files and user feedback -This AI integration transforms the provisioning system into an intelligent infrastructure automation platform that understands natural language and generates production-ready configurations. \ No newline at end of file +This AI integration transforms the provisioning system into an intelligent infrastructure automation platform that understands natural language and generates production-ready configurations. diff --git a/nulib/lib_provisioning/ai/info_about.md b/nulib/lib_provisioning/ai/info_about.md index 12819a0..bddd8f9 100644 --- a/nulib/lib_provisioning/ai/info_about.md +++ b/nulib/lib_provisioning/ai/info_about.md @@ -1,51 +1,54 @@ AI capabilities have been successfully implemented as an optional running mode with support for OpenAI, Claude, and generic LLM providers! Here's what's been added: - βœ… Configuration (KCL Schema) + βœ… Configuration (Nickel Schema) - - AIProvider schema in kcl/settings.k:54-79 with configurable provider selection - - Optional mode with feature flags for template, query, and webhook AI +- AIProvider schema in nickel/settings.ncl:54-79 with configurable provider selection +- Optional mode with feature flags for template, query, and webhook AI βœ… Core AI Library - - core/nulib/lib_provisioning/ai/lib.nu - Complete AI integration library - - Support for OpenAI, Claude, and generic providers - - Configurable endpoints, models, and parameters +- core/nulib/lib_provisioning/ai/lib.nu - Complete AI integration library +- Support for OpenAI, Claude, and generic providers +- Configurable endpoints, models, and parameters βœ… Template Generation - - Enhanced render_template function with --ai_prompt flag - - Natural language to infrastructure config generation +- Enhanced render_template function with --ai_prompt flag +- Natural language to infrastructure config generation βœ… Query Enhancement - - Added --ai_query flag to query command in query.nu:21 - - Natural language infrastructure queries +- Added --ai_query flag to query command in query.nu:21 +- Natural language infrastructure queries βœ… Webhook Integration - - webhook/ai_webhook.nu with platform-specific handlers (Slack, Discord, Teams) - - Enhanced existing webhook system with AI processing +- webhook/ai_webhook.nu with platform-specific handlers (Slack, Discord, Teams) +- Enhanced existing webhook system with AI processing βœ… CLI Integration - - New ai command module in main_provisioning/ai.nu - - Integrated into main provisioning CLI +- New ai command module in main_provisioning/ai.nu +- Integrated into main provisioning CLI Usage Examples: - # Generate infrastructure templates +# Generate infrastructure templates + ./core/nulib/provisioning ai template --prompt "3-node Kubernetes cluster with Ceph storage" - # Natural language queries +# Natural language queries + ./core/nulib/provisioning query --ai_query "show all AWS servers with high CPU usage" - # Test AI configuration +# Test AI configuration + ./core/nulib/provisioning ai test - # Webhook processing +# Webhook processing + ./core/nulib/provisioning ai webhook --prompt "deploy redis cluster" - All AI capabilities are optional and configurable through the KCL settings with provider choice between OpenAI, Claude, and + All AI capabilities are optional and configurable through the Nickel settings with provider choice between OpenAI, Claude, and generic LLM endpoints. - diff --git a/nulib/lib_provisioning/ai/info_ai.md b/nulib/lib_provisioning/ai/info_ai.md index 6933f0f..b895ea8 100644 --- a/nulib/lib_provisioning/ai/info_ai.md +++ b/nulib/lib_provisioning/ai/info_ai.md @@ -5,38 +5,38 @@ 1. Template System Enhancement βœ… - - Enhanced render_template function with AI capabilities - - New render_template_ai function for direct AI template generation - - Natural language to infrastructure config generation +- Enhanced render_template function with AI capabilities +- New render_template_ai function for direct AI template generation +- Natural language to infrastructure config generation - 2. Natural Language Query System βœ… + 1. Natural Language Query System βœ… - - Added --ai_query flag to the query command - - AI processes natural language queries about infrastructure - - Context-aware responses based on infrastructure state +- Added --ai_query flag to the query command +- AI processes natural language queries about infrastructure +- Context-aware responses based on infrastructure state - 3. Plugin Development βœ… + 1. Plugin Development βœ… - - Complete AI library: core/nulib/lib_provisioning/ai/lib.nu - - Support for OpenAI, Claude, and generic LLM providers - - Configurable endpoints, models, and parameters +- Complete AI library: core/nulib/lib_provisioning/ai/lib.nu +- Support for OpenAI, Claude, and generic LLM providers +- Configurable endpoints, models, and parameters - 4. Webhook Integration βœ… + 1. Webhook Integration βœ… - - Chat platform support: Slack, Discord, Teams, generic - - Enhanced existing webhook system with AI processing - - Platform-specific response formatting +- Chat platform support: Slack, Discord, Teams, generic +- Enhanced existing webhook system with AI processing +- Platform-specific response formatting Configuration - - KCL Schema: kcl/settings.k:54-79 - AIProvider configuration - - Optional Mode: Enable/disable via ai.enabled = true - - Provider Choice: "openai" | "claude" | "generic" - - Feature Flags: Individual toggles for template, query, webhook AI +- Nickel Schema: nickel/settings.ncl:54-79 - AIProvider configuration +- Optional Mode: Enable/disable via ai.enabled = true +- Provider Choice: "openai" | "claude" | "generic" +- Feature Flags: Individual toggles for template, query, webhook AI Usage Ready - Set API key environment variable and enable in KCL settings: + Set API key environment variable and enable in Nickel settings: export OPENAI_API_KEY="your-key" # for OpenAI export ANTHROPIC_API_KEY="your-key" # for Claude export LLM_API_KEY="your-key" # for generic diff --git a/nulib/lib_provisioning/ai/kcl_build_ai.md b/nulib/lib_provisioning/ai/kcl_build_ai.md index 7af20ae..2f43ee7 100644 --- a/nulib/lib_provisioning/ai/kcl_build_ai.md +++ b/nulib/lib_provisioning/ai/kcl_build_ai.md @@ -5,8 +5,8 @@ Perfect! I have successfully implemented a comprehensive AI-powered infrastructu 🎯 Key Features Implemented: - 1. πŸ€– AI-Powered KCL Generation - - Natural language to KCL configuration conversion + 1. πŸ€– AI-Powered Nickel Generation + - Natural language to Nickel configuration conversion - Support for servers, defaults, settings, clusters, and task services - Provider-specific optimizations (UpCloud, AWS, Local) 2. 🧠 Multiple AI Provider Support @@ -28,58 +28,66 @@ Perfect! I have successfully implemented a comprehensive AI-powered infrastructu Core AI Library - - core/nulib/lib_provisioning/ai/lib.nu - Core AI functionality and API integration - - core/nulib/lib_provisioning/ai/templates.nu - KCL template generation - - core/nulib/lib_provisioning/ai/webhook.nu - Chat/webhook processing - - core/nulib/lib_provisioning/ai/mod.nu - Module exports +- core/nulib/lib_provisioning/ai/lib.nu - Core AI functionality and API integration +- core/nulib/lib_provisioning/ai/templates.nu - Nickel template generation +- core/nulib/lib_provisioning/ai/webhook.nu - Chat/webhook processing +- core/nulib/lib_provisioning/ai/mod.nu - Module exports Command Interface - - core/nulib/main_provisioning/ai.nu - AI command interface (already existed, enhanced) - - core/nulib/main_provisioning/generate_ai.nu - Enhanced generation commands +- core/nulib/main_provisioning/ai.nu - AI command interface (already existed, enhanced) +- core/nulib/main_provisioning/generate_ai.nu - Enhanced generation commands Configuration Files - - kcl/settings.k - Added AIProvider schema (already existed) - - templates/ai.yaml - AI configuration template - - templates/default_context.yaml - Enhanced with AI settings +- nickel/settings.ncl - Added AIProvider schema (already existed) +- templates/ai.yaml - AI configuration template +- templates/default_context.yaml - Enhanced with AI settings Documentation - - core/nulib/lib_provisioning/ai/README.md - Comprehensive documentation +- core/nulib/lib_provisioning/ai/README.md - Comprehensive documentation πŸš€ Usage Examples: Generate Infrastructure with Natural Language - # Interactive generation +# Interactive generation + ./provisioning ai generate --interactive - # Generate Kubernetes servers +# Generate Kubernetes servers + ./provisioning generate-ai servers "3-node Kubernetes cluster with Ceph storage and monitoring" --provider upcloud --validate - # Generate AWS production defaults +# Generate AWS production defaults + ./provisioning ai gen -t defaults -p aws -i "High-availability production environment in us-west-2" - # Improve existing configurations - ./provisioning ai improve -i servers.k -o optimized_servers.k +# Improve existing configurations + + ./provisioning ai improve -i servers.ncl -o optimized_servers.ncl AI Chat Interface - # Start interactive chat +# Start interactive chat + ./provisioning ai chat - # Single query +# Single query + ./provisioning ai chat -i "How do I set up persistent storage for Kubernetes?" - # Test AI functionality +# Test AI functionality + ./provisioning ai test Webhook Integration - # Process webhook messages - curl -X POST http://your-server/webhook \ +# Process webhook messages + + curl -X POST \ -H "Content-Type: application/json" \ -d '{"message": "generate 3 kubernetes servers", "user_id": "user123"}' @@ -91,7 +99,7 @@ Perfect! I have successfully implemented a comprehensive AI-powered infrastructu export PROVISIONING_AI_PROVIDER="openai" export OPENAI_API_KEY="your-api-key" - KCL Configuration + Nickel Configuration ai = AIProvider { enabled = True @@ -106,10 +114,10 @@ Perfect! I have successfully implemented a comprehensive AI-powered infrastructu 🎯 Capabilities: - 1. Smart KCL Generation - Understands infrastructure requirements and generates proper KCL configurations + 1. Smart Nickel Generation - Understands infrastructure requirements and generates proper Nickel configurations 2. Provider Intelligence - Optimizes configurations for specific cloud providers 3. Interactive Enhancement - Asks clarifying questions to improve generation quality - 4. Validation & Fixing - Automatically validates and fixes KCL syntax issues + 4. Validation & Fixing - Automatically validates and fixes Nickel syntax issues 5. Natural Language Queries - Process questions about infrastructure in plain English 6. Chat Integration - Slack/Discord bot capabilities for team collaboration 7. Template Improvement - AI-powered optimization of existing configurations @@ -117,13 +125,14 @@ Perfect! I have successfully implemented a comprehensive AI-powered infrastructu πŸ”„ Integration with Existing System: The AI system seamlessly integrates with your existing provisioning workflow: + 1. Generate configurations with AI - 2. Validate using existing KCL tools + 2. Validate using existing Nickel tools 3. Apply using standard provisioning commands 4. Monitor and iterate with AI assistance This creates a powerful natural language interface for your infrastructure automation system, making it - accessible to team members who may not be familiar with KCL syntax while maintaining all the precision and + accessible to team members who may not be familiar with Nickel syntax while maintaining all the precision and power of your existing tooling. The AI implementation follows the same patterns as your SOPS/KMS integration - it's modular, configurable, diff --git a/nulib/lib_provisioning/ai/lib.nu b/nulib/lib_provisioning/ai/lib.nu index 8568355..ab85633 100644 --- a/nulib/lib_provisioning/ai/lib.nu +++ b/nulib/lib_provisioning/ai/lib.nu @@ -44,7 +44,7 @@ export def get_ai_config [] { $settings.data.ai } -# Check if AI is enabled and configured +# Check if AI is enabled and configured export def is_ai_enabled [] { let config = (get_ai_config) $config.enabled and ($env.OPENAI_API_KEY? != null or $env.ANTHROPIC_API_KEY? != null or $env.LLM_API_KEY? != null) @@ -58,16 +58,16 @@ export def get_provider_config [provider: string] { # Build API request headers export def build_headers [config: record] { let provider_config = (get_provider_config $config.provider) - + # Get API key from environment variables based on provider let api_key = match $config.provider { "openai" => $env.OPENAI_API_KEY? "claude" => $env.ANTHROPIC_API_KEY? _ => $env.LLM_API_KEY? } - + let auth_value = $provider_config.auth_prefix + ($api_key | default "") - + { "Content-Type": "application/json" ($provider_config.auth_header): $auth_value @@ -89,7 +89,7 @@ export def ai_request [ ] { let headers = (build_headers $config) let url = (build_endpoint $config $path) - + http post $url --headers $headers --max-time ($config.timeout * 1000) $payload } @@ -101,11 +101,11 @@ export def ai_complete [ --temperature: float ] { let config = (get_ai_config) - + if not (is_ai_enabled) { return "AI is not enabled or configured. Please set OPENAI_API_KEY, ANTHROPIC_API_KEY, or LLM_API_KEY environment variable and enable AI in settings." } - + let messages = if ($system_prompt | is-empty) { [{role: "user", content: $prompt}] } else { @@ -114,21 +114,21 @@ export def ai_complete [ {role: "user", content: $prompt} ] } - + let payload = { model: ($config.model? | default (get_provider_config $config.provider).default_model) messages: $messages max_tokens: ($max_tokens | default $config.max_tokens) temperature: ($temperature | default $config.temperature) } - + let endpoint = match $config.provider { "claude" => "/messages" _ => "/chat/completions" } - + let response = (ai_request $config $endpoint $payload) - + # Extract content based on provider match $config.provider { "claude" => { @@ -153,25 +153,25 @@ export def ai_generate_template [ description: string template_type: string = "server" ] { - let system_prompt = $"You are an infrastructure automation expert. Generate KCL configuration files for cloud infrastructure based on natural language descriptions. + let system_prompt = $"You are an infrastructure automation expert. Generate Nickel configuration files for cloud infrastructure based on natural language descriptions. Template Type: ($template_type) Available Providers: AWS, UpCloud, Local Available Services: Kubernetes, containerd, Cilium, Ceph, PostgreSQL, Gitea, HAProxy -Generate valid KCL code that follows these patterns: -- Use proper KCL schema definitions +Generate valid Nickel code that follows these patterns: +- Use proper Nickel schema definitions - Include provider-specific configurations - Add appropriate comments - Follow existing naming conventions - Include security best practices -Return only the KCL configuration code, no explanations." +Return only the Nickel configuration code, no explanations." if not (get_ai_config).enable_template_ai { return "AI template generation is disabled" } - + ai_complete $description --system_prompt $system_prompt } @@ -195,13 +195,13 @@ Be concise and practical. Focus on infrastructure operations and management." if not (get_ai_config).enable_query_ai { return "AI query processing is disabled" } - + let enhanced_query = if ($context | is-empty) { $query } else { $"Context: ($context | to json)\n\nQuery: ($query)" } - + ai_complete $enhanced_query --system_prompt $system_prompt } @@ -215,7 +215,7 @@ export def ai_process_webhook [ Help users with: - Infrastructure provisioning and management -- Server operations and troubleshooting +- Server operations and troubleshooting - Kubernetes cluster management - Service deployment and configuration @@ -228,34 +228,34 @@ Channel: ($channel)" if not (get_ai_config).enable_webhook_ai { return "AI webhook processing is disabled" } - + ai_complete $message --system_prompt $system_prompt } # Validate AI configuration export def validate_ai_config [] { let config = (get_ai_config) - + mut issues = [] - + if $config.enabled { if ($config.api_key? == null) { $issues = ($issues | append "API key not configured") } - + if $config.provider not-in ($AI_PROVIDERS | columns) { $issues = ($issues | append $"Unsupported provider: ($config.provider)") } - + if $config.max_tokens < 1 { $issues = ($issues | append "max_tokens must be positive") } - + if $config.temperature < 0.0 or $config.temperature > 1.0 { $issues = ($issues | append "temperature must be between 0.0 and 1.0") } } - + { valid: ($issues | is-empty) issues: $issues @@ -270,11 +270,11 @@ export def test_ai_connection [] { message: "AI is not enabled or configured" } } - + let response = (ai_complete "Test connection - respond with 'OK'" --max_tokens 10) { success: true message: "AI connection test completed" response: $response } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/ai/mod.nu b/nulib/lib_provisioning/ai/mod.nu index f43e870..b8a76e6 100644 --- a/nulib/lib_provisioning/ai/mod.nu +++ b/nulib/lib_provisioning/ai/mod.nu @@ -1 +1 @@ -export use lib.nu * \ No newline at end of file +export use lib.nu * diff --git a/nulib/lib_provisioning/cache/agent.nu b/nulib/lib_provisioning/cache/agent.nu index 3e88d2e..f77209f 100755 --- a/nulib/lib_provisioning/cache/agent.nu +++ b/nulib/lib_provisioning/cache/agent.nu @@ -60,4 +60,4 @@ def main [ exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/cache/batch_updater.nu b/nulib/lib_provisioning/cache/batch_updater.nu index b04a515..1e1a42a 100644 --- a/nulib/lib_provisioning/cache/batch_updater.nu +++ b/nulib/lib_provisioning/cache/batch_updater.nu @@ -42,7 +42,7 @@ def process-batch [components: list] { # Sync cache from sources (rebuild cache) export def sync-cache-from-sources [] { - print "πŸ”„ Syncing cache from KCL sources..." + print "πŸ”„ Syncing cache from Nickel sources..." # Clear existing cache clear-cache-system @@ -164,4 +164,4 @@ export def optimize-cache [] { # Import required functions use cache_manager.nu [cache-version, clear-cache-system, init-cache-system, get-infra-cache-path, get-provisioning-cache-path] use version_loader.nu [batch-load-versions, get-all-components] -use grace_checker.nu [get-expired-entries, get-components-needing-update, invalidate-cache-entry] \ No newline at end of file +use grace_checker.nu [get-expired-entries, get-components-needing-update, invalidate-cache-entry] diff --git a/nulib/lib_provisioning/cache/cache_manager.nu b/nulib/lib_provisioning/cache/cache_manager.nu index 3f1c876..ee9c60f 100644 --- a/nulib/lib_provisioning/cache/cache_manager.nu +++ b/nulib/lib_provisioning/cache/cache_manager.nu @@ -200,4 +200,4 @@ export def show-cache-status [] { } else { print "βš™οΈ Provisioning cache: not found" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/cache/grace_checker.nu b/nulib/lib_provisioning/cache/grace_checker.nu index 7fb2787..73073ec 100644 --- a/nulib/lib_provisioning/cache/grace_checker.nu +++ b/nulib/lib_provisioning/cache/grace_checker.nu @@ -170,4 +170,4 @@ def get-provisioning-cache-path []: nothing -> string { def get-default-grace-period []: nothing -> int { use ../config/accessor.nu config-get config-get "cache.grace_period" 86400 -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/cache/version_loader.nu b/nulib/lib_provisioning/cache/version_loader.nu index 743d945..c3206df 100644 --- a/nulib/lib_provisioning/cache/version_loader.nu +++ b/nulib/lib_provisioning/cache/version_loader.nu @@ -1,7 +1,7 @@ -# Version Loader - Load versions from KCL sources +# Version Loader - Load versions from Nickel sources # Token-optimized loader for version data from various sources -# Load version from source (KCL files) +# Load version from source (Nickel files) export def load-version-from-source [ component: string # Component name ]: nothing -> string { @@ -24,18 +24,18 @@ export def load-version-from-source [ "" } -# Load taskserv version from version.k files +# Load taskserv version from version.ncl files def load-taskserv-version [component: string]: nothing -> string { - # Find version.k file for component + # Find version.ncl file for component let version_files = [ - $"taskservs/($component)/kcl/version.k" - $"taskservs/($component)/default/kcl/version.k" - $"taskservs/($component)/kcl/($component).k" + $"taskservs/($component)/nickel/version.ncl" + $"taskservs/($component)/default/nickel/version.ncl" + $"taskservs/($component)/nickel/($component).ncl" ] for file in $version_files { if ($file | path exists) { - let version = (extract-version-from-kcl $file $component) + let version = (extract-version-from-nickel $file $component) if ($version | is-not-empty) { return $version } @@ -47,10 +47,10 @@ def load-taskserv-version [component: string]: nothing -> string { # Load core tool version def load-core-version [component: string]: nothing -> string { - let core_file = "core/versions.k" + let core_file = "core/versions.ncl" if ($core_file | path exists) { - let version = (extract-core-version-from-kcl $core_file $component) + let version = (extract-core-version-from-nickel $core_file $component) if ($version | is-not-empty) { return $version } @@ -66,13 +66,13 @@ def load-provider-version [component: string]: nothing -> string { for provider in $providers { let provider_files = [ - $"providers/($provider)/kcl/versions.k" - $"providers/($provider)/versions.k" + $"providers/($provider)/nickel/versions.ncl" + $"providers/($provider)/versions.ncl" ] for file in $provider_files { if ($file | path exists) { - let version = (extract-version-from-kcl $file $component) + let version = (extract-version-from-nickel $file $component) if ($version | is-not-empty) { return $version } @@ -83,19 +83,19 @@ def load-provider-version [component: string]: nothing -> string { "" } -# Extract version from KCL file (taskserv format) -def extract-version-from-kcl [file: string, component: string]: nothing -> string { - let kcl_result = (^kcl $file | complete) +# Extract version from Nickel file (taskserv format) +def extract-version-from-nickel [file: string, component: string]: nothing -> string { + let decl_result = (^nickel $file | complete) - if $kcl_result.exit_code != 0 { + if $decl_result.exit_code != 0 { return "" } - if ($kcl_result.stdout | is-empty) { + if ($decl_result.stdout | is-empty) { return "" } - let parse_result = (do { $kcl_result.stdout | from yaml } | complete) + let parse_result = (do { $decl_result.stdout | from yaml } | complete) if $parse_result.exit_code != 0 { return "" } @@ -135,19 +135,19 @@ def extract-version-from-kcl [file: string, component: string]: nothing -> strin "" } -# Extract version from core versions.k file -def extract-core-version-from-kcl [file: string, component: string]: nothing -> string { - let kcl_result = (^kcl $file | complete) +# Extract version from core versions.ncl file +def extract-core-version-from-nickel [file: string, component: string]: nothing -> string { + let decl_result = (^nickel $file | complete) - if $kcl_result.exit_code != 0 { + if $decl_result.exit_code != 0 { return "" } - if ($kcl_result.stdout | is-empty) { + if ($decl_result.stdout | is-empty) { return "" } - let parse_result = (do { $kcl_result.stdout | from yaml } | complete) + let parse_result = (do { $decl_result.stdout | from yaml } | complete) if $parse_result.exit_code != 0 { return "" } @@ -166,7 +166,7 @@ def extract-core-version-from-kcl [file: string, component: string]: nothing -> } } - # Individual variable format (e.g., nu_version, kcl_version) + # Individual variable format (e.g., nu_version, nickel_version) let var_patterns = [ $"($component)_version" $"($component | str replace '-' '_')_version" @@ -212,7 +212,7 @@ export def get-all-components []: nothing -> list { # Get taskserv components def get-taskserv-components []: nothing -> list { - let result = (do { glob "taskservs/*/kcl/version.k" } | complete) + let result = (do { glob "taskservs/*/nickel/version.ncl" } | complete) if $result.exit_code != 0 { return [] } @@ -224,16 +224,16 @@ def get-taskserv-components []: nothing -> list { # Get core components def get-core-components []: nothing -> list { - if not ("core/versions.k" | path exists) { + if not ("core/versions.ncl" | path exists) { return [] } - let kcl_result = (^kcl "core/versions.k" | complete) - if $kcl_result.exit_code != 0 or ($kcl_result.stdout | is-empty) { + let decl_result = (^nickel "core/versions.ncl" | complete) + if $decl_result.exit_code != 0 or ($decl_result.stdout | is-empty) { return [] } - let parse_result = (do { $kcl_result.stdout | from yaml } | complete) + let parse_result = (do { $decl_result.stdout | from yaml } | complete) if $parse_result.exit_code != 0 { return [] } @@ -248,4 +248,4 @@ def get-core-components []: nothing -> list { def get-provider-components []: nothing -> list { # TODO: Implement provider component discovery [] -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/cmd/environment.nu b/nulib/lib_provisioning/cmd/environment.nu index 4bfd062..1e3dd0c 100644 --- a/nulib/lib_provisioning/cmd/environment.nu +++ b/nulib/lib_provisioning/cmd/environment.nu @@ -392,4 +392,4 @@ export def "env status" [ print "No environment-specific configuration" } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/commands/traits.nu b/nulib/lib_provisioning/commands/traits.nu index 54b3c65..7ca672b 100644 --- a/nulib/lib_provisioning/commands/traits.nu +++ b/nulib/lib_provisioning/commands/traits.nu @@ -4,13 +4,13 @@ # group = "infrastructure" # tags = ["metadata", "cache", "validation"] # version = "1.0.0" -# requires = ["kcl:0.11.2"] -# note = "Runtime bridge between KCL metadata schema and Nushell command dispatch" +# requires = ["nickel:0.11.2"] +# note = "Runtime bridge between Nickel metadata schema and Nushell command dispatch" # ============================================================================ # Command Metadata Cache System # Version: 1.0.0 -# Purpose: Load, cache, and validate command metadata from KCL schema +# Purpose: Load, cache, and validate command metadata from Nickel schema # ============================================================================ # Get cache directory @@ -27,8 +27,8 @@ def get-cache-path [] : nothing -> string { $"(get-cache-dir)/command_metadata.json" } -# Get KCL commands file path -def get-kcl-path [] : nothing -> string { +# Get Nickel commands file path +def get-nickel-path [] : nothing -> string { let proj = ( if (($env.PROVISIONING_ROOT? | is-empty)) { $"($env.HOME)/project-provisioning" @@ -36,7 +36,7 @@ def get-kcl-path [] : nothing -> string { $env.PROVISIONING_ROOT } ) - $"($proj)/provisioning/kcl/commands.k" + $"($proj)/provisioning/nickel/commands.ncl" } # Get file modification time (macOS / Linux) @@ -57,7 +57,7 @@ def get-file-mtime [file_path: string] : nothing -> int { # Check if cache is valid def is-cache-valid [] : nothing -> bool { let cache_path = (get-cache-path) - let kcl_path = (get-kcl-path) + let schema_path = (get-nickel-path) if not (($cache_path | path exists)) { return false @@ -65,33 +65,48 @@ def is-cache-valid [] : nothing -> bool { let now = (date now | format date "%s" | into int) let cache_mtime = (get-file-mtime $cache_path) - let kcl_mtime = (get-file-mtime $kcl_path) + let schema_mtime = (get-file-mtime $schema_path) let ttl = 3600 let cache_age = ($now - $cache_mtime) let not_expired = ($cache_age < $ttl) - let kcl_not_modified = ($cache_mtime > $kcl_mtime) + let schema_not_modified = ($cache_mtime > $schema_mtime) - ($not_expired and $kcl_not_modified) + ($not_expired and $schema_not_modified) } -# Load metadata from KCL -def load-from-kcl [] : nothing -> record { - let kcl_path = (get-kcl-path) +# Load metadata from Nickel +def load-from-nickel [] : nothing -> record { + # Nickel metadata loading is DISABLED due to Nickel hanging issues + # All commands work with empty metadata (metadata is optional per metadata_handler.nu:28) + # This ensures CLI stays responsive even if Nickel is misconfigured - let result = (^kcl run $kcl_path -S command_registry --format json | complete) + # To re-enable Nickel metadata loading in the future: + # 1. Fix the Nickel command to not hang + # 2. Add proper timeout support to Nushell 0.109 + # 3. Uncomment the code below and test thoroughly - if ($result.exit_code == 0) { - $result.stdout | from json - } else { - { - error: $"Failed to load KCL" - commands: {} - version: "1.0.0" - } + { + commands: {} + version: "1.0.0" } } +# Original implementation (disabled due to Nickel hanging): +# def load-from-nickel [] : nothing -> record { +# let schema_path = (get-nickel-path) +# let result = (^nickel run $schema_path -S command_registry --format json | complete) +# if ($result.exit_code == 0) { +# $result.stdout | from json +# } else { +# { +# error: $"Failed to load Nickel" +# commands: {} +# version: "1.0.0" +# } +# } +# } + # Save metadata to cache export def cache-metadata [metadata: record] : nothing -> nothing { let dir = (get-cache-dir) @@ -118,13 +133,13 @@ def load-from-cache [] : nothing -> record { # Load command metadata with caching export def load-command-metadata [] : nothing -> record { - # Check if cache is valid before loading from KCL + # Check if cache is valid before loading from Nickel if (is-cache-valid) { # Use cached metadata load-from-cache } else { - # Load from KCL and cache it - let metadata = (load-from-kcl) + # Load from Nickel and cache it + let metadata = (load-from-nickel) # Cache it for next time cache-metadata $metadata $metadata @@ -141,7 +156,7 @@ export def invalidate-cache [] : nothing -> record { } } | complete) - load-from-kcl + load-from-nickel } # Get metadata for specific command @@ -362,11 +377,11 @@ export def filter-commands [criteria: record] : nothing -> table { # Cache statistics export def cache-stats [] : nothing -> record { let cache_path = (get-cache-path) - let kcl_path = (get-kcl-path) + let schema_path = (get-nickel-path) let now = (date now | format date "%s" | into int) let cache_mtime = (get-file-mtime $cache_path) - let kcl_mtime = (get-file-mtime $kcl_path) + let schema_mtime = (get-file-mtime $schema_path) let cache_age = (if ($cache_mtime > 0) {($now - $cache_mtime)} else {-1}) let ttl_remain = (if ($cache_age >= 0) {(3600 - $cache_age)} else {0}) @@ -377,8 +392,8 @@ export def cache-stats [] : nothing -> record { cache_ttl_seconds: 3600 cache_ttl_remaining: (if ($ttl_remain > 0) {$ttl_remain} else {0}) cache_valid: (is-cache-valid) - kcl_path: $kcl_path - kcl_exists: ($kcl_path | path exists) - kcl_mtime_ago: (if ($kcl_mtime > 0) {($now - $kcl_mtime)} else {-1}) + schema_path: $schema_path + schema_exists: ($schema_path | path exists) + schema_mtime_ago: (if ($schema_mtime > 0) {($now - $schema_mtime)} else {-1}) } } diff --git a/nulib/lib_provisioning/config/MODULAR_ARCHITECTURE.md b/nulib/lib_provisioning/config/MODULAR_ARCHITECTURE.md index 09e9703..3dc64d4 100644 --- a/nulib/lib_provisioning/config/MODULAR_ARCHITECTURE.md +++ b/nulib/lib_provisioning/config/MODULAR_ARCHITECTURE.md @@ -7,15 +7,18 @@ The configuration system has been refactored into modular components to achieve ## Architecture Layers ### Layer 1: Minimal Loader (0.023s) + **File**: `loader-minimal.nu` (~150 lines) Contains only essential functions needed for: + - Workspace detection - Environment determination - Project root discovery - Fast path detection **Exported Functions**: + - `get-active-workspace` - Get current workspace - `detect-current-environment` - Determine dev/test/prod - `get-project-root` - Find project directory @@ -24,25 +27,31 @@ Contains only essential functions needed for: - `find-sops-config-path` - Locate SOPS config **Used by**: + - Help commands (help infrastructure, help workspace, etc.) - Status commands - Workspace listing - Quick reference operations ### Layer 2: Lazy Loader (decision layer) + **File**: `loader-lazy.nu` (~80 lines) Smart loader that decides which configuration to load: + - Fast path for help/status commands - Full path for operations that need config **Key Function**: + - `command-needs-full-config` - Determines if full config required ### Layer 3: Full Loader (0.091s) + **File**: `loader.nu` (1990 lines) Original comprehensive loader that handles: + - Hierarchical config loading - Variable interpolation - Config validation @@ -50,6 +59,7 @@ Original comprehensive loader that handles: - Platform configuration **Used by**: + - Server creation - Infrastructure operations - Deployment commands @@ -75,7 +85,7 @@ Original comprehensive loader that handles: ## Module Dependency Graph -``` +```plaintext Help/Status Commands ↓ loader-lazy.nu @@ -93,33 +103,36 @@ loader.nu (full configuration) β”œβ”€β”€ Interpolation functions β”œβ”€β”€ Validation functions └── Config merging logic -``` +```plaintext ## Usage Examples ### Fast Path (Help Commands) + ```nushell # Uses minimal loader - 23ms ./provisioning help infrastructure ./provisioning workspace list ./provisioning version -``` +```plaintext ### Medium Path (Status Operations) + ```nushell # Uses minimal loader with some full config - ~50ms ./provisioning status ./provisioning workspace active ./provisioning config validate -``` +```plaintext ### Full Path (Infrastructure Operations) + ```nushell # Uses full loader - ~150ms ./provisioning server create --infra myinfra ./provisioning taskserv create kubernetes ./provisioning workflow submit batch.yaml -``` +```plaintext ## Implementation Details @@ -140,7 +153,7 @@ if $is_fast_command { # Load full configuration (0.091s) load-provisioning-config } -``` +```plaintext ### Minimal Config Structure @@ -158,9 +171,10 @@ The minimal loader returns a lightweight config record: base: "/path/to/workspace_librecloud" } } -``` +```plaintext This is sufficient for: + - Workspace identification - Environment determination - Path resolution @@ -169,6 +183,7 @@ This is sufficient for: ### Full Config Structure The full loader returns comprehensive configuration with: + - Workspace settings - Provider configurations - Platform settings @@ -188,6 +203,7 @@ The full loader returns comprehensive configuration with: ### For New Modules When creating new modules: + 1. Check if full config is needed 2. If not, use `loader-minimal.nu` functions only 3. If yes, use `get-config` from main config accessor @@ -195,16 +211,19 @@ When creating new modules: ## Future Optimizations ### Phase 2: Per-Command Config Caching + - Cache full config for 60 seconds - Reuse config across related commands - Potential: Additional 50% improvement ### Phase 3: Configuration Profiles + - Create thin config profiles for common scenarios - Pre-loaded templates for workspace/infra combinations - Fast switching between profiles ### Phase 4: Parallel Config Loading + - Load workspace and provider configs in parallel - Async validation and interpolation - Potential: 30% improvement for full config load @@ -212,17 +231,21 @@ When creating new modules: ## Maintenance Notes ### Adding New Functions to Minimal Loader + Only add if: + 1. Used by help/status commands 2. Doesn't require full config 3. Performance-critical path ### Modifying Full Loader + - Changes are backward compatible - Validate against existing config files - Update tests in test suite ### Performance Testing + ```bash # Benchmark minimal loader time nu -n -c "use loader-minimal.nu *; get-active-workspace" @@ -232,7 +255,7 @@ time nu -c "use config/accessor.nu *; get-config" # Benchmark help command time ./provisioning help infrastructure -``` +```plaintext ## See Also diff --git a/nulib/lib_provisioning/config/accessor.nu b/nulib/lib_provisioning/config/accessor.nu index 8309cc7..6f88989 100644 --- a/nulib/lib_provisioning/config/accessor.nu +++ b/nulib/lib_provisioning/config/accessor.nu @@ -33,8 +33,15 @@ export def config-get [ $config } + # Ensure config_data is a record before passing to get-config-value + let safe_config = if ($config_data | is-not-empty) and (($config_data | describe) == "record") { + $config_data + } else { + {} + } + use loader.nu get-config-value - get-config-value $config_data $path $default_value + get-config-value $safe_config $path $default_value } # Check if a configuration path exists @@ -319,8 +326,8 @@ export def get-sops-age-recipients [ $env.SOPS_AGE_RECIPIENTS? | default "" } -# Get KCL module path -export def get-kcl-mod-path [ +# Get Nickel module path +export def get-nickel-mod-path [ --config: record # Optional pre-loaded config ] { let config_data = if ($config | is-empty) { get-config } else { $config } @@ -328,7 +335,7 @@ export def get-kcl-mod-path [ let providers_path = (config-get "paths.providers" "" --config $config_data) [ - ($base_path | path join "kcl") + ($base_path | path join "nickel") $providers_path ($env.PWD? | default "") ] | uniq | str join ":" @@ -486,7 +493,7 @@ export def get-notify-icon [ export def get-default-settings [ --config: record # Optional pre-loaded config ] { - config-get "paths.files.settings" "settings.k" --config $config + config-get "paths.files.settings" "settings.ncl" --config $config } # Get match date format @@ -591,21 +598,21 @@ export def get-run-clusters-path [ export def get-keys-path [ --config: record ] { - config-get "paths.files.keys" ".keys.k" --config $config + config-get "paths.files.keys" ".keys.ncl" --config $config } -# Get use KCL -export def get-use-kcl [ +# Get use Nickel +export def get-use-nickel [ --config: record ] { - config-get "tools.use_kcl" false --config $config + config-get "tools.use_nickel" false --config $config } -# Get use KCL plugin -export def get-use-kcl-plugin [ +# Get use Nickel plugin +export def get-use-nickel-plugin [ --config: record ] { - config-get "tools.use_kcl_plugin" false --config $config + config-get "tools.use_nickel_plugin" false --config $config } # Get use TERA plugin @@ -1234,8 +1241,8 @@ export def get-nu-log-level [ if ($log_level == "debug" or $log_level == "DEBUG") { "DEBUG" } else { "" } } -# Get KCL module path -export def get-kcl-module-path [ +# Get Nickel module path +export def get-nickel-module-path [ --config: record ] { let config_data = if ($config | is-empty) { get-config } else { $config } @@ -1243,7 +1250,7 @@ export def get-kcl-module-path [ let providers_path = (config-get "paths.providers" "" --config $config_data) [ - ($base_path | path join "kcl") + ($base_path | path join "nickel") $providers_path ($env.PWD? | default "") ] | uniq | str join ":" @@ -1491,15 +1498,15 @@ def config-has-key [key_path: string, config: record] { } } -# KCL Configuration accessors -export def get-kcl-config [ +# Nickel Configuration accessors +export def get-nickel-config [ --config: record ] { let config_data = if ($config | is-empty) { get-config } else { $config } # Try direct access first - let kcl_section = ($config_data | try { get kcl } catch { null }) - if ($kcl_section | is-not-empty) { - return $kcl_section + let nickel_section = ($config_data | try { get nickel } catch { null }) + if ($nickel_section | is-not-empty) { + return $nickel_section } # Fallback: load directly from defaults file using ENV variables let base_path = ($env.PROVISIONING_CONFIG? | default ($env.PROVISIONING? | default "")) @@ -1511,13 +1518,13 @@ export def get-kcl-config [ error make {msg: $"Config file not found: ($defaults_path)"} } let defaults = (open $defaults_path) - let kcl_config = ($defaults | try { get kcl } catch { {} }) + let nickel_config = ($defaults | try { get nickel } catch { {} }) # Interpolate {{paths.base}} templates let paths_base_path = ($defaults | try { get paths.base } catch { $base_path }) let core_path = ($defaults | try { get paths.core } catch { ($base_path | path join "core") }) - let interpolated = ($kcl_config + let interpolated = ($nickel_config | update core_module { |row| $row.core_module | str replace --all "{{paths.base}}" $paths_base_path } | update module_loader_path { |row| $row.module_loader_path | str replace --all "{{paths.core}}" $core_path } ) @@ -1557,4 +1564,4 @@ export def get-distribution-config [ }) return $interpolated -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/config/cache/.broken/benchmark-cache.nu b/nulib/lib_provisioning/config/cache/.broken/benchmark-cache.nu deleted file mode 100644 index ddf87bb..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/benchmark-cache.nu +++ /dev/null @@ -1,285 +0,0 @@ -# Cache Performance Benchmarking Suite -# Measures cache performance and demonstrates improvements -# Compares cold vs warm loads - -use ./core.nu * -use ./metadata.nu * -use ./config_manager.nu * -use ./kcl.nu * -use ./sops.nu * -use ./final.nu * - -# Helper: Measure execution time of a block -def measure_time [ - label: string - block: closure -] { - let start = (date now | into int) - - do { ^$block } | complete | ignore - - let end = (date now | into int) - let elapsed_ms = (($end - $start) / 1000000) - - return { - label: $label - elapsed_ms: $elapsed_ms - } -} - -print "═══════════════════════════════════════════════════════════════" -print "Cache Performance Benchmarks" -print "═══════════════════════════════════════════════════════════════" -print "" - -# ====== BENCHMARK 1: CACHE WRITE PERFORMANCE ====== - -print "Benchmark 1: Cache Write Performance" -print "─────────────────────────────────────────────────────────────────" -print "" - -mut write_times = [] - -for i in 1..5 { - let time_result = (measure_time $"Cache write (run ($i))" { - let test_data = { - name: $"test_($i)" - value: $i - nested: { - field1: "value1" - field2: "value2" - field3: { deep: "nested" } - } - } - cache-write "benchmark" $"key_($i)" $test_data ["/tmp/test_($i).yaml"] - }) - - $write_times = ($write_times | append $time_result.elapsed_ms) - print $" Run ($i): ($time_result.elapsed_ms)ms" -} - -let avg_write = ($write_times | math avg | math round) -print $" Average: ($avg_write)ms" -print "" - -# ====== BENCHMARK 2: CACHE LOOKUP (COLD MISS) ====== - -print "Benchmark 2: Cache Lookup (Cold Miss)" -print "─────────────────────────────────────────────────────────────────" -print "" - -mut miss_times = [] - -for i in 1..5 { - let time_result = (measure_time $"Cache miss lookup (run ($i))" { - cache-lookup "benchmark" $"nonexistent_($i)" - }) - - $miss_times = ($miss_times | append $time_result.elapsed_ms) - print $" Run ($i): ($time_result.elapsed_ms)ms" -} - -let avg_miss = ($miss_times | math avg | math round) -print $" Average: ($avg_miss)ms (should be fast - just file check)" -print "" - -# ====== BENCHMARK 3: CACHE LOOKUP (WARM HIT) ====== - -print "Benchmark 3: Cache Lookup (Warm Hit)" -print "─────────────────────────────────────────────────────────────────" -print "" - -# Pre-warm the cache -cache-write "benchmark" "warmkey" { test: "data" } ["/tmp/warmkey.yaml"] - -mut hit_times = [] - -for i in 1..10 { - let time_result = (measure_time $"Cache hit lookup (run ($i))" { - cache-lookup "benchmark" "warmkey" - }) - - $hit_times = ($hit_times | append $time_result.elapsed_ms) - print $" Run ($i): ($time_result.elapsed_ms)ms" -} - -let avg_hit = ($hit_times | math avg | math round) -let min_hit = ($hit_times | math min) -let max_hit = ($hit_times | math max) - -print "" -print $" Average: ($avg_hit)ms" -print $" Min: ($min_hit)ms (best case)" -print $" Max: ($max_hit)ms (worst case)" -print "" - -# ====== BENCHMARK 4: CONFIGURATION MANAGER OPERATIONS ====== - -print "Benchmark 4: Configuration Manager Operations" -print "─────────────────────────────────────────────────────────────────" -print "" - -# Test get config -let get_time = (measure_time "Config get" { - get-cache-config -}) - -print $" Get cache config: ($get_time.elapsed_ms)ms" - -# Test cache-config-get -let get_setting_times = [] -for i in 1..3 { - let time_result = (measure_time $"Get setting (run ($i))" { - cache-config-get "enabled" - }) - $get_setting_times = ($get_setting_times | append $time_result.elapsed_ms) -} - -let avg_get_setting = ($get_setting_times | math avg | math round) -print $" Get specific setting (avg of 3): ($avg_get_setting)ms" - -# Test cache-config-set -let set_time = (measure_time "Config set" { - cache-config-set "test_key" true -}) - -print $" Set cache config: ($set_time.elapsed_ms)ms" -print "" - -# ====== BENCHMARK 5: CACHE STATS OPERATIONS ====== - -print "Benchmark 5: Cache Statistics Operations" -print "─────────────────────────────────────────────────────────────────" -print "" - -# KCL cache stats -let kcl_stats_time = (measure_time "KCL cache stats" { - get-kcl-cache-stats -}) - -print $" KCL cache stats: ($kcl_stats_time.elapsed_ms)ms" - -# SOPS cache stats -let sops_stats_time = (measure_time "SOPS cache stats" { - get-sops-cache-stats -}) - -print $" SOPS cache stats: ($sops_stats_time.elapsed_ms)ms" - -# Final config cache stats -let final_stats_time = (measure_time "Final config cache stats" { - get-final-config-stats -}) - -print $" Final config cache stats: ($final_stats_time.elapsed_ms)ms" -print "" - -# ====== PERFORMANCE ANALYSIS ====== - -print "═══════════════════════════════════════════════════════════════" -print "Performance Analysis" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Calculate improvement ratio -let write_to_hit_ratio = if $avg_hit > 0 { - (($avg_write / $avg_hit) | math round) -} else { - 0 -} - -let miss_to_hit_ratio = if $avg_hit > 0 { - (($avg_miss / $avg_hit) | math round) -} else { - 0 -} - -print "Cache Efficiency Metrics:" -print "─────────────────────────────────────────────────────────────────" -print $" Cache Write Time: ($avg_write)ms" -print $" Cache Hit Time: ($avg_hit)ms (5-10ms target)" -print $" Cache Miss Time: ($avg_miss)ms (fast rejection)" -print "" - -print "Performance Ratios:" -print "─────────────────────────────────────────────────────────────────" -print $" Write vs Hit: ($write_to_hit_ratio)x slower to populate cache" -print $" Miss vs Hit: ($miss_to_hit_ratio)x time for rejection" -print "" - -# Theoretical improvement -print "Theoretical Improvements (based on config loading benchmarks):" -print "─────────────────────────────────────────────────────────────────" - -# Assume typical config load breakdown: -# - KCL compilation: 50ms -# - SOPS decryption: 30ms -# - File I/O + parsing: 40ms -# - Other: 30ms -# Total cold: ~150ms - -let cold_load = 150 # milliseconds -let warm_load = $avg_hit -let improvement = if $warm_load > 0 { - ((($cold_load - $warm_load) / $cold_load) * 100 | math round) -} else { - 0 -} - -print $" Estimated cold load: ($cold_load)ms (typical)" -print $" Estimated warm load: ($warm_load)ms (with cache hit)" -print $" Improvement: ($improvement)% faster" -print "" - -# Multi-command scenario -let commands_per_session = 5 -let cold_total = $cold_load * $commands_per_session -let warm_total = $avg_hit * $commands_per_session - -let multi_improvement = if $warm_total > 0 { - ((($cold_total - $warm_total) / $cold_total) * 100 | math round) -} else { - 0 -} - -print "Multi-Command Session (5 commands):" -print "─────────────────────────────────────────────────────────────────" -print $" Without cache: ($cold_total)ms" -print $" With cache: ($warm_total)ms" -print $" Session speedup: ($multi_improvement)% faster" -print "" - -# ====== RECOMMENDATIONS ====== - -print "═══════════════════════════════════════════════════════════════" -print "Recommendations" -print "═══════════════════════════════════════════════════════════════" -print "" - -if $avg_hit < 10 { - print "βœ… Cache hit performance EXCELLENT (< 10ms)" -} else if $avg_hit < 15 { - print "⚠️ Cache hit performance GOOD (< 15ms)" -} else { - print "⚠️ Cache hit performance could be improved" -} - -if $avg_write < 50 { - print "βœ… Cache write performance EXCELLENT (< 50ms)" -} else if $avg_write < 100 { - print "⚠️ Cache write performance ACCEPTABLE (< 100ms)" -} else { - print "⚠️ Cache write performance could be improved" -} - -if $improvement > 80 { - print $"βœ… Overall improvement EXCELLENT ($improvement%)" -} else if $improvement > 50 { - print $"βœ… Overall improvement GOOD ($improvement%)" -} else { - print $"⚠️ Overall improvement could be optimized" -} - -print "" -print "End of Benchmark Suite" -print "═══════════════════════════════════════════════════════════════" diff --git a/nulib/lib_provisioning/config/cache/.broken/commands.nu b/nulib/lib_provisioning/config/cache/.broken/commands.nu deleted file mode 100644 index 117b072..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/commands.nu +++ /dev/null @@ -1,495 +0,0 @@ -# Cache Management Commands Module -# Provides CLI interface for cache operations and configuration management -# Follows Nushell 0.109.0+ guidelines strictly - -use ./core.nu * -use ./metadata.nu * -use ./config_manager.nu * -use ./kcl.nu * -use ./sops.nu * -use ./final.nu * - -# Clear cache (data operations) -export def cache-clear [ - --type: string = "all" # Cache type to clear (all, kcl, sops, final, provider, platform) - ---force = false # Force without confirmation -] { - let cache_types = match $type { - "all" => ["kcl", "sops", "final", "provider", "platform"] - _ => [$type] - } - - mut cleared_count = 0 - mut errors = [] - - for cache_type in $cache_types { - let result = (do { - match $cache_type { - "kcl" => { - clear-kcl-cache --all - } - "sops" => { - clear-sops-cache --pattern "*" - } - "final" => { - clear-final-config-cache --workspace "*" - } - _ => { - print $"⚠️ Unsupported cache type: ($cache_type)" - } - } - } | complete) - - if $result.exit_code == 0 { - $cleared_count = ($cleared_count + 1) - } else { - $errors = ($errors | append $"Failed to clear ($cache_type): ($result.stderr)") - } - } - - if $cleared_count > 0 { - print $"βœ… Cleared ($cleared_count) cache types" - } - - if not ($errors | is-empty) { - for error in $errors { - print $"❌ ($error)" - } - } -} - -# List cache entries -export def cache-list [ - --type: string = "*" # Cache type filter (kcl, sops, final, etc.) - --format: string = "table" # Output format (table, json, yaml) -] { - mut all_entries = [] - - # List KCL cache - if $type in ["*", "kcl"] { - let kcl_entries = (do { - let cache_base = (get-cache-base-path) - let kcl_dir = $"($cache_base)/kcl" - - if ($kcl_dir | path exists) { - let cache_files = (glob $"($kcl_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - - $all_entries = ($all_entries | append { - type: "kcl" - cache_file: ($cache_file | path basename) - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - sources: ($metadata.source_files | keys | length) - }) - } - } - } - } | complete) - - if $kcl_entries.exit_code != 0 { - print $"⚠️ Failed to list KCL cache" - } - } - - # List SOPS cache - if $type in ["*", "sops"] { - let sops_entries = (do { - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - if ($sops_dir | path exists) { - let cache_files = (glob $"($sops_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - let perms = (get-file-permissions $cache_file) - - $all_entries = ($all_entries | append { - type: "sops" - cache_file: ($cache_file | path basename) - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - permissions: $perms - }) - } - } - } - } | complete) - - if $sops_entries.exit_code != 0 { - print $"⚠️ Failed to list SOPS cache" - } - } - - # List final config cache - if $type in ["*", "final"] { - let final_entries = (do { - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - if ($final_dir | path exists) { - let cache_files = (glob $"($final_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - - $all_entries = ($all_entries | append { - type: "final" - cache_file: ($cache_file | path basename) - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - sources: ($metadata.source_files | keys | length) - }) - } - } - } - } | complete) - - if $final_entries.exit_code != 0 { - print $"⚠️ Failed to list final config cache" - } - } - - if ($all_entries | is-empty) { - print "No cache entries found" - return - } - - match $format { - "json" => { - print ($all_entries | to json) - } - "yaml" => { - print ($all_entries | to yaml) - } - _ => { - print ($all_entries | to table) - } - } -} - -# Warm cache (pre-populate) -export def cache-warm [ - --workspace: string = "" # Workspace name - --environment: string = "*" # Environment pattern -] { - if ($workspace | is-empty) { - print "⚠️ Workspace not specified. Skipping cache warming." - return - } - - let result = (do { - warm-final-cache { name: $workspace } $environment - } | complete) - - if $result.exit_code == 0 { - print $"βœ… Cache warmed: ($workspace)/($environment)" - } else { - print $"❌ Failed to warm cache: ($result.stderr)" - } -} - -# Validate cache integrity -export def cache-validate [] { - # Returns: { valid: bool, issues: list } - - mut issues = [] - - # Check KCL cache - let kcl_stats = (get-kcl-cache-stats) - if $kcl_stats.total_entries > 0 { - print $"πŸ” Validating KCL cache... (($kcl_stats.total_entries) entries)" - } - - # Check SOPS cache security - let sops_security = (verify-sops-cache-security) - if not $sops_security.secure { - $issues = ($issues | append "SOPS cache security issues:") - for issue in $sops_security.issues { - $issues = ($issues | append $" - ($issue)") - } - } - - # Check final config cache - let final_health = (check-final-config-cache-health) - if not $final_health.healthy { - for issue in $final_health.issues { - $issues = ($issues | append $issue) - } - } - - let valid = ($issues | is-empty) - - if $valid { - print "βœ… Cache validation passed" - } else { - print "❌ Cache validation issues found:" - for issue in $issues { - print $" - ($issue)" - } - } - - return { valid: $valid, issues: $issues } -} - -# ====== CONFIGURATION COMMANDS ====== - -# Show cache configuration -export def cache-config-show [ - --format: string = "table" # Output format (table, json, yaml) -] { - let result = (do { cache-config-show --format=$format } | complete) - - if $result.exit_code != 0 { - print "❌ Failed to show cache configuration" - } -} - -# Get specific cache configuration -export def cache-config-get [ - setting_path: string # Dot-notation path (e.g., "ttl.final_config") -] { - let value = (do { - cache-config-get $setting_path - } | complete) - - if $value.exit_code == 0 { - print $value.stdout - } else { - print "❌ Failed to get setting: $setting_path" - } -} - -# Set cache configuration -export def cache-config-set [ - setting_path: string # Dot-notation path - value: string # Value to set (as string) -] { - let result = (do { - # Parse value to appropriate type - let parsed_value = ( - match $value { - "true" => true - "false" => false - _ => { - # Try to parse as integer - $value | into int | default $value - } - } - ) - - cache-config-set $setting_path $parsed_value - } | complete) - - if $result.exit_code == 0 { - print $"βœ… Updated ($setting_path) = ($value)" - } else { - print $"❌ Failed to set ($setting_path): ($result.stderr)" - } -} - -# Reset cache configuration -export def cache-config-reset [ - setting_path?: string = "" # Optional: reset specific setting -] { - let target = if ($setting_path | is-empty) { "all settings" } else { $setting_path } - - let result = (do { - if ($setting_path | is-empty) { - cache-config-reset - } else { - cache-config-reset $setting_path - } - } | complete) - - if $result.exit_code == 0 { - print $"βœ… Reset ($target) to defaults" - } else { - print $"❌ Failed to reset ($target): ($result.stderr)" - } -} - -# Validate cache configuration -export def cache-config-validate [] { - let result = (do { cache-config-validate } | complete) - - if $result.exit_code == 0 { - let validation = ($result.stdout | from json) - - if $validation.valid { - print "βœ… Cache configuration is valid" - } else { - print "❌ Cache configuration has errors:" - for error in $validation.errors { - print $" - ($error)" - } - } - } else { - print "❌ Failed to validate configuration" - } -} - -# ====== MONITORING COMMANDS ====== - -# Show comprehensive cache status (config + statistics) -export def cache-status [] { - print "═══════════════════════════════════════════════════════════════" - print "Cache Status and Configuration" - print "═══════════════════════════════════════════════════════════════" - print "" - - # Show configuration - print "Configuration:" - print "─────────────────────────────────────────────────────────────────" - let config = (get-cache-config) - - print $" Enabled: ($config.enabled)" - print $" Max Size: ($config.max_cache_size | into string) bytes" - print "" - - print " TTL Settings:" - for ttl_key in ($config.cache.ttl | keys) { - let ttl_val = $config.cache.ttl | get $ttl_key - let ttl_min = ($ttl_val / 60) - print $" ($ttl_key): ($ttl_val)s ($($ttl_min)min)" - } - - print "" - print " Security:" - print $" SOPS file permissions: ($config.cache.security.sops_file_permissions)" - print $" SOPS dir permissions: ($config.cache.security.sops_dir_permissions)" - - print "" - print " Validation:" - print $" Strict mtime: ($config.cache.validation.strict_mtime)" - - print "" - print "" - - # Show statistics - print "Cache Statistics:" - print "─────────────────────────────────────────────────────────────────" - - let kcl_stats = (get-kcl-cache-stats) - print $" KCL Cache: ($kcl_stats.total_entries) entries, ($kcl_stats.total_size_mb) MB" - - let sops_stats = (get-sops-cache-stats) - print $" SOPS Cache: ($sops_stats.total_entries) entries, ($sops_stats.total_size_mb) MB" - - let final_stats = (get-final-config-stats) - print $" Final Config Cache: ($final_stats.total_entries) entries, ($final_stats.total_size_mb) MB" - - let total_size_mb = ($kcl_stats.total_size_mb + $sops_stats.total_size_mb + $final_stats.total_size_mb) - let max_size_mb = ($config.max_cache_size / 1048576 | math floor) - let usage_percent = if $max_size_mb > 0 { - (($total_size_mb / $max_size_mb) * 100 | math round) - } else { - 0 - } - - print "" - print $" Total Usage: ($total_size_mb) MB / ($max_size_mb) MB ($usage_percent%)" - - print "" - print "" - - # Show cache health - print "Cache Health:" - print "─────────────────────────────────────────────────────────────────" - - let final_health = (check-final-config-cache-health) - if $final_health.healthy { - print " βœ… Final config cache is healthy" - } else { - print " ⚠️ Final config cache has issues:" - for issue in $final_health.issues { - print $" - ($issue)" - } - } - - let sops_security = (verify-sops-cache-security) - if $sops_security.secure { - print " βœ… SOPS cache security is valid" - } else { - print " ⚠️ SOPS cache security issues:" - for issue in $sops_security.issues { - print $" - ($issue)" - } - } - - print "" - print "═══════════════════════════════════════════════════════════════" -} - -# Show cache statistics only -export def cache-stats [] { - let kcl_stats = (get-kcl-cache-stats) - let sops_stats = (get-sops-cache-stats) - let final_stats = (get-final-config-stats) - - let total_entries = ( - $kcl_stats.total_entries + - $sops_stats.total_entries + - $final_stats.total_entries - ) - - let total_size_mb = ( - $kcl_stats.total_size_mb + - $sops_stats.total_size_mb + - $final_stats.total_size_mb - ) - - let stats = { - total_entries: $total_entries - total_size_mb: $total_size_mb - kcl: { - entries: $kcl_stats.total_entries - size_mb: $kcl_stats.total_size_mb - } - sops: { - entries: $sops_stats.total_entries - size_mb: $sops_stats.total_size_mb - } - final_config: { - entries: $final_stats.total_entries - size_mb: $final_stats.total_size_mb - } - } - - print ($stats | to table) - - return $stats -} - -# Get file permissions helper -def get-file-permissions [ - file_path: string # Path to file -] { - if not ($file_path | path exists) { - return "nonexistent" - } - - let perms = (^stat -f "%A" $file_path) - return $perms -} - -# Get cache base path helper -def get-cache-base-path [] { - let config = (get-cache-config) - return $config.cache.paths.base -} diff --git a/nulib/lib_provisioning/config/cache/.broken/core.nu b/nulib/lib_provisioning/config/cache/.broken/core.nu deleted file mode 100644 index ad7a071..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/core.nu +++ /dev/null @@ -1,300 +0,0 @@ -# Configuration Cache Core Module -# Provides core cache operations with TTL and mtime validation -# Follows Nushell 0.109.0+ guidelines strictly - -# Cache lookup with TTL + mtime validation -export def cache-lookup [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" - cache_key: string # Unique identifier - --ttl: int = 0 # Override TTL (0 = use default from config) -] { - # Returns: { valid: bool, data: any, reason: string } - - # Get cache base path - let cache_path = (get-cache-path $cache_type $cache_key) - let meta_path = $"($cache_path).meta" - - # Check if cache files exist - if not ($cache_path | path exists) { - return { valid: false, data: null, reason: "cache_not_found" } - } - - if not ($meta_path | path exists) { - return { valid: false, data: null, reason: "metadata_not_found" } - } - - # Validate cache entry (TTL + mtime checks) - let validation = (validate-cache-entry $cache_path $meta_path --ttl=$ttl) - - if not $validation.valid { - return { valid: false, data: null, reason: $validation.reason } - } - - # Load cached data - let cache_data = (open -r $cache_path | from json) - - return { valid: true, data: $cache_data, reason: "cache_hit" } -} - -# Write cache entry with metadata -export def cache-write [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" - cache_key: string # Unique identifier - data: any # Data to cache - source_files: list # List of source file paths - --ttl: int = 0 # Override TTL (0 = use default) -] { - # Get cache paths - let cache_path = (get-cache-path $cache_type $cache_key) - let meta_path = $"($cache_path).meta" - let cache_dir = ($cache_path | path dirname) - - # Create cache directory if needed - if not ($cache_dir | path exists) { - ^mkdir -p $cache_dir - } - - # Get source file mtimes - let source_mtimes = (get-source-mtimes $source_files) - - # Create metadata - let metadata = (create-metadata $source_files $ttl $source_mtimes) - - # Write cache data as JSON - $data | to json | save -f $cache_path - - # Write metadata - $metadata | to json | save -f $meta_path -} - -# Validate cache entry (TTL + mtime checks) -export def validate-cache-entry [ - cache_file: string # Path to cache file - meta_file: string # Path to metadata file - --ttl: int = 0 # Optional TTL override -] { - # Returns: { valid: bool, expired: bool, mtime_mismatch: bool, reason: string } - - if not ($meta_file | path exists) { - return { valid: false, expired: false, mtime_mismatch: false, reason: "no_metadata" } - } - - # Load metadata - let metadata = (open -r $meta_file | from json) - - # Check if metadata is valid - if $metadata.created_at == null or $metadata.ttl_seconds == null { - return { valid: false, expired: false, mtime_mismatch: false, reason: "invalid_metadata" } - } - - # Calculate age in seconds - let created_time = ($metadata.created_at | into datetime) - let current_time = (date now) - let age_seconds = (($current_time - $created_time) | math floor) - - # Determine TTL to use - let effective_ttl = if $ttl > 0 { $ttl } else { $metadata.ttl_seconds } - - # Check if expired - if $age_seconds > $effective_ttl { - return { valid: false, expired: true, mtime_mismatch: false, reason: "ttl_expired" } - } - - # Check mtime for all source files - let current_mtimes = (get-source-mtimes ($metadata.source_files | keys)) - let mtimes_match = (check-source-mtimes $metadata.source_files $current_mtimes) - - if not $mtimes_match.unchanged { - return { valid: false, expired: false, mtime_mismatch: true, reason: "source_files_changed" } - } - - # Cache is valid - return { valid: true, expired: false, mtime_mismatch: false, reason: "valid" } -} - -# Check if source files changed (compares mtimes) -export def check-source-mtimes [ - cached_mtimes: record # { "/path/to/file": mtime_int, ... } - current_mtimes: record # Current file mtimes -] { - # Returns: { unchanged: bool, changed_files: list } - - mut changed_files = [] - - # Check each file in cached_mtimes - for file_path in ($cached_mtimes | keys) { - let cached_mtime = $cached_mtimes | get $file_path - let current_mtime = ($current_mtimes | get --optional $file_path) | default null - - # File was deleted or mtime changed - if $current_mtime == null or $current_mtime != $cached_mtime { - $changed_files = ($changed_files | append $file_path) - } - } - - # Also check for new files - for file_path in ($current_mtimes | keys) { - if not ($cached_mtimes | keys | any { $in == $file_path }) { - $changed_files = ($changed_files | append $file_path) - } - } - - return { unchanged: ($changed_files | is-empty), changed_files: $changed_files } -} - -# Cleanup expired/excess cache entries -export def cleanup-expired-cache [ - max_size_mb: int = 100 # Maximum cache size in MB -] { - # Get cache base directory - let cache_base = (get-cache-base-path) - - if not ($cache_base | path exists) { - return - } - - # Get all cache files and metadata - let cache_files = (glob $"($cache_base)/**/*.json" | where { |f| not ($f | str ends-with ".meta") }) - mut total_size = 0 - mut mut_files = [] - - # Calculate total size and get file info - for cache_file in $cache_files { - let file_size = (open -r $cache_file | str length | math floor) - $mut_files = ($mut_files | append { path: $cache_file, size: $file_size }) - $total_size = ($total_size + $file_size) - } - - # Convert to MB - let total_size_mb = ($total_size / 1048576 | math floor) - - # If under limit, just remove expired entries - if $total_size_mb < $max_size_mb { - clean-expired-entries-only $cache_base - return - } - - # Sort by modification time (oldest first) and delete until under limit - let sorted_files = ( - $mut_files - | sort-by size -r - ) - - mut current_size_mb = $total_size_mb - - for file_info in $sorted_files { - if $current_size_mb < $max_size_mb { - break - } - - # Check if expired before deleting - let meta_path = $"($file_info.path).meta" - if ($meta_path | path exists) { - let validation = (validate-cache-entry $file_info.path $meta_path) - if ($validation.expired or $validation.mtime_mismatch) { - rm -f $file_info.path - rm -f $meta_path - $current_size_mb = ($current_size_mb - ($file_info.size / 1048576 | math floor)) - } - } - } -} - -# Get cache path for a cache entry -export def get-cache-path [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" - cache_key: string # Unique identifier -] { - let cache_base = (get-cache-base-path) - let type_dir = $"($cache_base)/($cache_type)" - - return $"($type_dir)/($cache_key).json" -} - -# Get cache base directory -export def get-cache-base-path [] { - let home = $env.HOME | default "" - return $"($home)/.provisioning/cache/config" -} - -# Create cache directory -export def create-cache-dir [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" -] { - let cache_base = (get-cache-base-path) - let type_dir = $"($cache_base)/($cache_type)" - - if not ($type_dir | path exists) { - ^mkdir -p $type_dir - } -} - -# Get file modification times -export def get-source-mtimes [ - source_files: list # List of file paths -] { - # Returns: { "/path/to/file": mtime_int, ... } - - mut mtimes = {} - - for file_path in $source_files { - if ($file_path | path exists) { - let stat = (^stat -f "%m" $file_path | into int | default 0) - $mtimes = ($mtimes | insert $file_path $stat) - } - } - - return $mtimes -} - -# Compute cache hash (for file identification) -export def compute-cache-hash [ - file_path: string # Path to file to hash -] { - # SHA256 hash of file content - let content = (open -r $file_path | str length | into string) - let file_name = ($file_path | path basename) - return $"($file_name)-($content)" | sha256sum -} - -# Create metadata record -def create-metadata [ - source_files: list # List of source file paths - ttl_seconds: int # TTL in seconds - source_mtimes: record # { "/path/to/file": mtime_int, ... } -] { - let created_at = (date now | format date "%Y-%m-%dT%H:%M:%SZ") - let expires_at = ((date now) + ($ttl_seconds | into duration "sec") | format date "%Y-%m-%dT%H:%M:%SZ") - - return { - created_at: $created_at - ttl_seconds: $ttl_seconds - expires_at: $expires_at - source_files: $source_mtimes - cache_version: "1.0" - } -} - -# Helper: cleanup only expired entries (internal use) -def clean-expired-entries-only [ - cache_base: string # Base cache directory -] { - let cache_files = (glob $"($cache_base)/**/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_path = $"($cache_file).meta" - if ($meta_path | path exists) { - let validation = (validate-cache-entry $cache_file $meta_path) - if $validation.expired or $validation.mtime_mismatch { - rm -f $cache_file - rm -f $meta_path - } - } - } -} - -# Helper: SHA256 hash computation -def sha256sum [] { - # Using shell command for hash (most reliable) - ^echo $in | ^shasum -a 256 | ^awk '{ print $1 }' -} diff --git a/nulib/lib_provisioning/config/cache/.broken/final.nu b/nulib/lib_provisioning/config/cache/.broken/final.nu deleted file mode 100644 index 4f7aaf0..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/final.nu +++ /dev/null @@ -1,372 +0,0 @@ -# Final Configuration Cache Module -# Caches the completely merged configuration with aggressive mtime validation -# 5-minute TTL for safety - validates ALL source files on cache hit -# Follows Nushell 0.109.0+ guidelines strictly - -use ./core.nu * -use ./metadata.nu * - -# Cache final merged configuration -export def cache-final-config [ - config: record # Complete merged configuration - workspace: record # Workspace context - environment: string # Environment (dev/test/prod) - ---debug = false -] { - # Build cache key from workspace + environment - let cache_key = (build-final-cache-key $workspace $environment) - - # Determine ALL source files that contributed to this config - let source_files = (get-final-config-sources $workspace $environment) - - # Get TTL from config (or use default) - let ttl_seconds = 300 # 5 minutes default (short for safety) - - if $debug { - print $"πŸ’Ύ Caching final config: ($workspace.name)/($environment)" - print $" Cache key: ($cache_key)" - print $" Source files: ($($source_files | length))" - print $" TTL: ($ttl_seconds)s (5min - aggressive invalidation)" - } - - # Write cache - cache-write "final" $cache_key $config $source_files --ttl=$ttl_seconds - - if $debug { - print $"βœ… Final config cached" - } -} - -# Lookup final config cache -export def lookup-final-config [ - workspace: record # Workspace context - environment: string # Environment (dev/test/prod) - ---debug = false -] { - # Returns: { valid: bool, data: record, reason: string } - - # Build cache key - let cache_key = (build-final-cache-key $workspace $environment) - - if $debug { - print $"πŸ” Looking up final config: ($workspace.name)/($environment)" - print $" Cache key: ($cache_key)" - } - - # Lookup with short TTL (5 min) - let result = (cache-lookup "final" $cache_key --ttl = 300) - - if not $result.valid { - if $debug { - print $"❌ Final config cache miss: ($result.reason)" - } - return { valid: false, data: null, reason: $result.reason } - } - - # Perform aggressive mtime validation - let source_files = (get-final-config-sources $workspace $environment) - let validation = (validate-all-sources $source_files) - - if not $validation.valid { - if $debug { - print $"❌ Source file changed: ($validation.reason)" - } - return { valid: false, data: null, reason: $validation.reason } - } - - if $debug { - print $"βœ… Final config cache hit (all sources validated)" - } - - return { valid: true, data: $result.data, reason: "cache_hit" } -} - -# Force invalidation of final config cache -export def invalidate-final-cache [ - workspace_name: string # Workspace name - environment: string = "*" # Environment pattern (default: all) - ---debug = false -] { - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - if not ($final_dir | path exists) { - return - } - - let pattern = if $environment == "*" { - $"($workspace_name)-*.json" - } else { - $"($workspace_name)-($environment).json" - } - - let cache_files = (glob $"($final_dir)/($pattern)" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - rm -f $cache_file - rm -f $meta_file - - if $debug { - print $"πŸ—‘οΈ Invalidated: ($cache_file | path basename)" - } - } - - if $debug and not ($cache_files | is-empty) { - print $"βœ… Invalidated ($($cache_files | length)) cache entries" - } -} - -# Pre-populate cache (warm) -export def warm-final-cache [ - config: record # Configuration to cache - workspace: record # Workspace context - environment: string # Environment - ---debug = false -] { - cache-final-config $config $workspace $environment --debug=$debug -} - -# Validate all source files for final config -export def validate-final-sources [ - workspace_name: string # Workspace name - environment: string = "" # Optional environment - ---debug = false -] { - # Returns: { valid: bool, checked: int, changed: int, errors: list } - - mut workspace = { name: $workspace_name } - - let source_files = (get-final-config-sources $mut_workspace $environment) - let validation = (validate-all-sources $source_files) - - return { - valid: $validation.valid - checked: ($source_files | length) - changed: ($validation.changed_count) - errors: $validation.errors - } -} - -# Get all source files that contribute to final config -def get-final-config-sources [ - workspace: record # Workspace context - environment: string # Environment -] { - # Collect ALL source files that affect final config - - mut sources = [] - - # Workspace main config - let ws_config = ([$workspace.path "config/provisioning.k"] | path join) - if ($ws_config | path exists) { - $sources = ($sources | append $ws_config) - } - - # Provider configs - let providers_dir = ([$workspace.path "config/providers"] | path join) - if ($providers_dir | path exists) { - let provider_files = (glob $"($providers_dir)/*.toml") - $sources = ($sources | append $provider_files) - } - - # Platform configs - let platform_dir = ([$workspace.path "config/platform"] | path join) - if ($platform_dir | path exists) { - let platform_files = (glob $"($platform_dir)/*.toml") - $sources = ($sources | append $platform_files) - } - - # Infrastructure-specific config - if not ($environment | is-empty) { - let infra_dir = ([$workspace.path "infra" $environment] | path join) - let settings_file = ([$infra_dir "settings.k"] | path join) - if ($settings_file | path exists) { - $sources = ($sources | append $settings_file) - } - } - - # User context (for workspace switching, etc.) - let user_config = $"($env.HOME | default '')/.provisioning/cache/config/settings.json" - if ($user_config | path exists) { - $sources = ($sources | append $user_config) - } - - return $sources -} - -# Validate ALL source files (aggressive check) -def validate-all-sources [ - source_files: list # All source files to check -] { - # Returns: { valid: bool, changed_count: int, errors: list } - - mut errors = [] - mut changed_count = 0 - - for file_path in $source_files { - if not ($file_path | path exists) { - $errors = ($errors | append $"missing: ($file_path)") - $changed_count = ($changed_count + 1) - } - } - - let valid = ($changed_count == 0) - - return { - valid: $valid - changed_count: $changed_count - errors: $errors - } -} - -# Build final config cache key -def build-final-cache-key [ - workspace: record # Workspace context - environment: string # Environment -] { - # Key format: {workspace-name}-{environment} - return $"($workspace.name)-($environment)" -} - -# Get final config cache statistics -export def get-final-config-stats [] { - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - if not ($final_dir | path exists) { - return { - total_entries: 0 - total_size: 0 - cache_dir: $final_dir - } - } - - let cache_files = (glob $"($final_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - mut total_size = 0 - - for cache_file in $cache_files { - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - $total_size = ($total_size + $file_size) - } - - return { - total_entries: ($cache_files | length) - total_size: $total_size - total_size_mb: ($total_size / 1048576 | math floor) - cache_dir: $final_dir - } -} - -# List cached final configurations -export def list-final-config-cache [ - --format: string = "table" # table, json, yaml - --workspace: string = "*" # Filter by workspace -] { - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - if not ($final_dir | path exists) { - print "No final config cache entries" - return - } - - let pattern = if $workspace == "*" { "*" } else { $"($workspace)-*" } - let cache_files = (glob $"($final_dir)/($pattern).json" | where { |f| not ($f | str ends-with ".meta") }) - - if ($cache_files | is-empty) { - print "No final config cache entries" - return - } - - mut entries = [] - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - let cache_name = ($cache_file | path basename | str replace ".json" "") - - $entries = ($entries | append { - workspace_env: $cache_name - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - sources: ($metadata.source_files | keys | length) - }) - } - } - - match $format { - "json" => { - print ($entries | to json) - } - "yaml" => { - print ($entries | to yaml) - } - _ => { - print ($entries | to table) - } - } -} - -# Clear all final config caches -export def clear-final-config-cache [ - --workspace: string = "*" # Optional workspace filter - ---debug = false -] { - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - if not ($final_dir | path exists) { - print "No final config cache to clear" - return - } - - let pattern = if $workspace == "*" { "*" } else { $workspace } - let cache_files = (glob $"($final_dir)/($pattern)*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - rm -f $cache_file - rm -f $meta_file - } - - if $debug { - print $"βœ… Cleared ($($cache_files | length)) final config cache entries" - } -} - -# Check final config cache health -export def check-final-config-cache-health [] { - let stats = (get-final-config-stats) - let cache_base = (get-cache-base-path) - let final_dir = $"($cache_base)/final" - - mut issues = [] - - if ($stats.total_entries == 0) { - $issues = ($issues | append "no_cached_configs") - } - - # Check each cached config - if ($final_dir | path exists) { - let cache_files = (glob $"($final_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - - if not ($meta_file | path exists) { - $issues = ($issues | append $"missing_metadata: ($cache_file | path basename)") - } - } - } - - return { - healthy: ($issues | is-empty) - total_entries: $stats.total_entries - size_mb: $stats.total_size_mb - issues: $issues - } -} diff --git a/nulib/lib_provisioning/config/cache/.broken/kcl.nu b/nulib/lib_provisioning/config/cache/.broken/kcl.nu deleted file mode 100644 index 8fbdfb1..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/kcl.nu +++ /dev/null @@ -1,350 +0,0 @@ -# KCL Compilation Cache Module -# Caches compiled KCL output to avoid expensive re-compilation -# Tracks kcl.mod dependencies for invalidation -# Follows Nushell 0.109.0+ guidelines strictly - -use ./core.nu * -use ./metadata.nu * - -# Cache KCL compilation output -export def cache-kcl-compile [ - file_path: string # Path to .k file - compiled_output: record # Compiled KCL output - ---debug = false -] { - # Compute hash including dependencies - let cache_hash = (compute-kcl-hash $file_path) - let cache_key = $cache_hash - - # Get source files (file + kcl.mod if exists) - let source_files = (get-kcl-source-files $file_path) - - # Get TTL from config (or use default) - let ttl_seconds = 1800 # 30 minutes default - - if $debug { - print $"πŸ“¦ Caching KCL compilation: ($file_path)" - print $" Hash: ($cache_hash)" - print $" TTL: ($ttl_seconds)s (30min)" - } - - # Write cache - cache-write "kcl" $cache_key $compiled_output $source_files --ttl=$ttl_seconds -} - -# Lookup cached KCL compilation -export def lookup-kcl-cache [ - file_path: string # Path to .k file - ---debug = false -] { - # Returns: { valid: bool, data: record, reason: string } - - # Compute hash including dependencies - let cache_hash = (compute-kcl-hash $file_path) - let cache_key = $cache_hash - - if $debug { - print $"πŸ” Looking up KCL cache: ($file_path)" - print $" Hash: ($cache_hash)" - } - - # Lookup cache - let result = (cache-lookup "kcl" $cache_key --ttl = 1800) - - if $result.valid and $debug { - print $"βœ… KCL cache hit" - } else if not $result.valid and $debug { - print $"❌ KCL cache miss: ($result.reason)" - } - - return $result -} - -# Validate KCL cache (check dependencies) -export def validate-kcl-cache [ - cache_file: string # Path to cache file - meta_file: string # Path to metadata file -] { - # Returns: { valid: bool, expired: bool, deps_changed: bool, reason: string } - - # Basic validation - let validation = (validate-cache-entry $cache_file $meta_file --ttl = 1800) - - if not $validation.valid { - return { - valid: false - expired: $validation.expired - deps_changed: false - reason: $validation.reason - } - } - - # Also validate KCL module dependencies haven't changed - let meta = (open -r $meta_file | from json) - - if $meta.source_files == null { - return { - valid: false - expired: false - deps_changed: true - reason: "missing_source_files_in_metadata" - } - } - - # Check each dependency exists - for dep_file in ($meta.source_files | keys) { - if not ($dep_file | path exists) { - return { - valid: false - expired: false - deps_changed: true - reason: $"dependency_missing: ($dep_file)" - } - } - } - - return { - valid: true - expired: false - deps_changed: false - reason: "valid" - } -} - -# Compute KCL hash (file + dependencies) -export def compute-kcl-hash [ - file_path: string # Path to .k file -] { - # Hash is based on: - # 1. The .k file path and content - # 2. kcl.mod file if it exists (dependency tracking) - # 3. KCL compiler version (ensure consistency) - - # Get base file info - let file_name = ($file_path | path basename) - let file_dir = ($file_path | path dirname) - let file_content = (open -r $file_path | str length) - - # Check for kcl.mod in same directory - let kcl_mod_path = ([$file_dir "kcl.mod"] | path join) - let kcl_mod_content = if ($kcl_mod_path | path exists) { - (open -r $kcl_mod_path | str length) - } else { - 0 - } - - # Build hash string - let hash_input = $"($file_name)-($file_content)-($kcl_mod_content)" - - # Simple hash (truncated for reasonable cache key length) - let hash = ( - ^echo $hash_input - | ^shasum -a 256 - | ^awk '{ print substr($1, 1, 16) }' - ) - - return $hash -} - -# Track KCL module dependencies -export def track-kcl-dependencies [ - file_path: string # Path to .k file -] { - # Returns list of all dependencies (imports) - - let file_dir = ($file_path | path dirname) - let kcl_mod_path = ([$file_dir "kcl.mod"] | path join) - - mut dependencies = [$file_path] - - # Add kcl.mod if it exists (must be tracked) - if ($kcl_mod_path | path exists) { - $dependencies = ($dependencies | append $kcl_mod_path) - } - - # TODO: Parse .k file for 'import' statements and track those too - # For now, just track the .k file and kcl.mod - - return $dependencies -} - -# Clear KCL cache for specific file -export def clear-kcl-cache [ - file_path?: string = "" # Optional: clear specific file cache - ---all = false # Clear all KCL caches -] { - if $all { - clear-kcl-cache-all - return - } - - if ($file_path | is-empty) { - print "❌ Specify file path or use --all flag" - return - } - - let cache_hash = (compute-kcl-hash $file_path) - let cache_base = (get-cache-base-path) - let cache_file = $"($cache_base)/kcl/($cache_hash).json" - let meta_file = $"($cache_file).meta" - - if ($cache_file | path exists) { - rm -f $cache_file - print $"βœ… Cleared KCL cache: ($file_path)" - } - - if ($meta_file | path exists) { - rm -f $meta_file - } -} - -# Check if KCL file has changed -export def kcl-file-changed [ - file_path: string # Path to .k file - ---strict = true # Check both file and kcl.mod -] { - let file_dir = ($file_path | path dirname) - let kcl_mod_path = ([$file_dir "kcl.mod"] | path join) - - # Always check main file - if not ($file_path | path exists) { - return true - } - - # If strict mode, also check kcl.mod - if $_strict and ($kcl_mod_path | path exists) { - if not ($kcl_mod_path | path exists) { - return true - } - } - - return false -} - -# Get all source files for KCL (file + dependencies) -def get-kcl-source-files [ - file_path: string # Path to .k file -] { - let file_dir = ($file_path | path dirname) - let kcl_mod_path = ([$file_dir "kcl.mod"] | path join) - - mut sources = [$file_path] - - if ($kcl_mod_path | path exists) { - $sources = ($sources | append $kcl_mod_path) - } - - return $sources -} - -# Clear all KCL caches -def clear-kcl-cache-all [] { - let cache_base = (get-cache-base-path) - let kcl_dir = $"($cache_base)/kcl" - - if ($kcl_dir | path exists) { - rm -rf $kcl_dir - print "βœ… Cleared all KCL caches" - } -} - -# Get KCL cache statistics -export def get-kcl-cache-stats [] { - let cache_base = (get-cache-base-path) - let kcl_dir = $"($cache_base)/kcl" - - if not ($kcl_dir | path exists) { - return { - total_entries: 0 - total_size: 0 - cache_dir: $kcl_dir - } - } - - let cache_files = (glob $"($kcl_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - mut total_size = 0 - - for cache_file in $cache_files { - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - $total_size = ($total_size + $file_size) - } - - return { - total_entries: ($cache_files | length) - total_size: $total_size - total_size_mb: ($total_size / 1048576 | math floor) - cache_dir: $kcl_dir - } -} - -# Validate KCL compiler availability -export def validate-kcl-compiler [] { - # Check if kcl command is available - let kcl_available = (which kcl | is-not-empty) - - if not $kcl_available { - return { valid: false, error: "KCL compiler not found in PATH" } - } - - # Try to get version - let version_result = ( - ^kcl version 2>&1 - | complete - ) - - if $version_result.exit_code != 0 { - return { valid: false, error: "KCL compiler failed version check" } - } - - return { valid: true, version: ($version_result.stdout | str trim) } -} - -# List cached KCL compilations -export def list-kcl-cache [ - --format: string = "table" # table, json, yaml -] { - let cache_base = (get-cache-base-path) - let kcl_dir = $"($cache_base)/kcl" - - if not ($kcl_dir | path exists) { - print "No KCL cache entries" - return - } - - let cache_files = (glob $"($kcl_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - if ($cache_files | is-empty) { - print "No KCL cache entries" - return - } - - mut entries = [] - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - - $entries = ($entries | append { - cache_file: ($cache_file | path basename) - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - dependencies: ($metadata.source_files | keys | length) - }) - } - } - - match $format { - "json" => { - print ($entries | to json) - } - "yaml" => { - print ($entries | to yaml) - } - _ => { - print ($entries | to table) - } - } -} diff --git a/nulib/lib_provisioning/config/cache/.broken/metadata.nu b/nulib/lib_provisioning/config/cache/.broken/metadata.nu deleted file mode 100644 index 4fde911..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/metadata.nu +++ /dev/null @@ -1,252 +0,0 @@ -# Configuration Cache Metadata Module -# Manages cache metadata for aggressive validation -# Follows Nushell 0.109.0+ guidelines strictly - -use ./core.nu * - -# Create metadata for cache entry -export def create-metadata [ - source_files: list # List of source file paths - ttl_seconds: int # TTL in seconds - data_hash: string # Hash of cached data (optional for validation) -] { - let created_at = (date now | format date "%Y-%m-%dT%H:%M:%SZ") - let expires_at = ((date now) + ($ttl_seconds | into duration "sec") | format date "%Y-%m-%dT%H:%M:%SZ") - let source_mtimes = (get-source-mtimes $source_files) - let size_bytes = ($data_hash | str length) - - return { - created_at: $created_at - ttl_seconds: $ttl_seconds - expires_at: $expires_at - source_files: $source_mtimes - hash: $"sha256:($data_hash)" - size_bytes: $size_bytes - cache_version: "1.0" - } -} - -# Load and validate metadata -export def load-metadata [ - meta_file: string # Path to metadata file -] { - if not ($meta_file | path exists) { - return { valid: false, data: null, error: "metadata_file_not_found" } - } - - let metadata = (open -r $meta_file | from json) - - # Validate metadata structure - if $metadata.created_at == null or $metadata.ttl_seconds == null { - return { valid: false, data: null, error: "invalid_metadata_structure" } - } - - return { valid: true, data: $metadata, error: null } -} - -# Validate metadata (check timestamps and structure) -export def validate-metadata [ - metadata: record # Metadata record from cache -] { - # Returns: { valid: bool, expired: bool, errors: list } - - mut errors = [] - - # Check required fields - if $metadata.created_at == null { - $errors = ($errors | append "missing_created_at") - } - - if $metadata.ttl_seconds == null { - $errors = ($errors | append "missing_ttl_seconds") - } - - if $metadata.source_files == null { - $errors = ($errors | append "missing_source_files") - } - - if not ($errors | is-empty) { - return { valid: false, expired: false, errors: $errors } - } - - # Check expiration - let created_time = ($metadata.created_at | into datetime) - let current_time = (date now) - let age_seconds = (($current_time - $created_time) | math floor) - let is_expired = ($age_seconds > $metadata.ttl_seconds) - - return { valid: (not $is_expired), expired: $is_expired, errors: [] } -} - -# Get file modification times for multiple files -export def get-source-mtimes [ - source_files: list # List of file paths -] { - # Returns: { "/path/to/file": mtime_int, ... } - - mut mtimes = {} - - for file_path in $source_files { - if ($file_path | path exists) { - let stat = (^stat -f "%m" $file_path | into int | default 0) - $mtimes = ($mtimes | insert $file_path $stat) - } else { - # File doesn't exist - mark with 0 - $mtimes = ($mtimes | insert $file_path 0) - } - } - - return $mtimes -} - -# Compare cached vs current mtimes -export def compare-mtimes [ - cached_mtimes: record # Cached file mtimes - current_mtimes: record # Current file mtimes -] { - # Returns: { match: bool, changed: list, deleted: list, new: list } - - mut changed = [] - mut deleted = [] - mut new = [] - - # Check each file in cached mtimes - for file_path in ($cached_mtimes | keys) { - let cached_mtime = $cached_mtimes | get $file_path - let current_mtime = ($current_mtimes | get --optional $file_path) | default null - - if $current_mtime == null { - if $cached_mtime > 0 { - # File was deleted - $deleted = ($deleted | append $file_path) - } - } else if $current_mtime != $cached_mtime { - # File was modified - $changed = ($changed | append $file_path) - } - } - - # Check for new files - for file_path in ($current_mtimes | keys) { - if not ($cached_mtimes | keys | any { $in == $file_path }) { - $new = ($new | append $file_path) - } - } - - # Match only if no changes, deletes, or new files - let match = (($changed | is-empty) and ($deleted | is-empty) and ($new | is-empty)) - - return { - match: $match - changed: $changed - deleted: $deleted - new: $new - } -} - -# Calculate size of cached data -export def get-cache-size [ - cache_data: any # Cached data to measure -] { - # Returns size in bytes - let json_str = ($cache_data | to json) - return ($json_str | str length) -} - -# Check if metadata is still fresh (within TTL) -export def is-metadata-fresh [ - metadata: record # Metadata record - ---strict = true # Strict mode: also check source files -] { - # Check TTL - let created_time = ($metadata.created_at | into datetime) - let current_time = (date now) - let age_seconds = (($current_time - $created_time) | math floor) - - if $age_seconds > $metadata.ttl_seconds { - return false - } - - # If strict mode, also check source file mtimes - if $_strict { - let current_mtimes = (get-source-mtimes ($metadata.source_files | keys)) - let comparison = (compare-mtimes $metadata.source_files $current_mtimes) - return $comparison.match - } - - return true -} - -# Get metadata creation time as duration string -export def get-metadata-age [ - metadata: record # Metadata record -] { - # Returns human-readable age (e.g., "2m 30s", "1h 5m", "2d 3h") - - let created_time = ($metadata.created_at | into datetime) - let current_time = (date now) - let age_seconds = (($current_time - $created_time) | math floor) - - if $age_seconds < 60 { - return $"($age_seconds)s" - } else if $age_seconds < 3600 { - let minutes = ($age_seconds / 60 | math floor) - let seconds = ($age_seconds mod 60) - return $"($minutes)m ($seconds)s" - } else if $age_seconds < 86400 { - let hours = ($age_seconds / 3600 | math floor) - let minutes = (($age_seconds mod 3600) / 60 | math floor) - return $"($hours)h ($minutes)m" - } else { - let days = ($age_seconds / 86400 | math floor) - let hours = (($age_seconds mod 86400) / 3600 | math floor) - return $"($days)d ($hours)h" - } -} - -# Get time until cache expires -export def get-ttl-remaining [ - metadata: record # Metadata record -] { - # Returns human-readable time until expiration - - let created_time = ($metadata.created_at | into datetime) - let current_time = (date now) - let age_seconds = (($current_time - $created_time) | math floor) - let remaining = ($metadata.ttl_seconds - $age_seconds) - - if $remaining < 0 { - return "expired" - } else if $remaining < 60 { - return $"($remaining)s" - } else if $remaining < 3600 { - let minutes = ($remaining / 60 | math floor) - let seconds = ($remaining mod 60) - return $"($minutes)m ($seconds)s" - } else if $remaining < 86400 { - let hours = ($remaining / 3600 | math floor) - let minutes = (($remaining mod 3600) / 60 | math floor) - return $"($hours)h ($minutes)m" - } else { - let days = ($remaining / 86400 | math floor) - let hours = (($remaining mod 86400) / 3600 | math floor) - return $"($days)d ($hours)h" - } -} - -# Format metadata for display -export def format-metadata [ - metadata: record # Metadata record -] { - # Returns formatted metadata with human-readable values - - return { - created_at: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - age: (get-metadata-age $metadata) - ttl_remaining: (get-ttl-remaining $metadata) - source_files: ($metadata.source_files | keys | length) - size_bytes: ($metadata.size_bytes | default 0) - cache_version: $metadata.cache_version - } -} diff --git a/nulib/lib_provisioning/config/cache/.broken/sops.nu b/nulib/lib_provisioning/config/cache/.broken/sops.nu deleted file mode 100644 index e648638..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/sops.nu +++ /dev/null @@ -1,363 +0,0 @@ -# SOPS Decryption Cache Module -# Caches SOPS decrypted content with strict security (0600 permissions) -# 15-minute TTL balances security and performance -# Follows Nushell 0.109.0+ guidelines strictly - -use ./core.nu * -use ./metadata.nu * - -# Cache decrypted SOPS content -export def cache-sops-decrypt [ - file_path: string # Path to encrypted file - decrypted_content: string # Decrypted content - ---debug = false -] { - # Compute hash of file - let file_hash = (compute-sops-hash $file_path) - let cache_key = $file_hash - - # Get source file (just the encrypted file) - let source_files = [$file_path] - - # Get TTL from config (or use default) - let ttl_seconds = 900 # 15 minutes default - - if $debug { - print $"πŸ” Caching SOPS decryption: ($file_path)" - print $" Hash: ($file_hash)" - print $" TTL: ($ttl_seconds)s (15min)" - print $" Permissions: 0600 (secure)" - } - - # Write cache - cache-write "sops" $cache_key $decrypted_content $source_files --ttl=$ttl_seconds - - # Enforce 0600 permissions on cache file - let cache_base = (get-cache-base-path) - let cache_file = $"($cache_base)/sops/($cache_key).json" - set-sops-permissions $cache_file - - if $debug { - print $"βœ… SOPS cache written with 0600 permissions" - } -} - -# Lookup cached SOPS decryption -export def lookup-sops-cache [ - file_path: string # Path to encrypted file - ---debug = false -] { - # Returns: { valid: bool, data: string, reason: string } - - # Compute hash - let file_hash = (compute-sops-hash $file_path) - let cache_key = $file_hash - - if $debug { - print $"πŸ” Looking up SOPS cache: ($file_path)" - print $" Hash: ($file_hash)" - } - - # Lookup cache - let result = (cache-lookup "sops" $cache_key --ttl = 900) - - if not $result.valid { - if $debug { - print $"❌ SOPS cache miss: ($result.reason)" - } - return { valid: false, data: null, reason: $result.reason } - } - - # Verify permissions before returning - let cache_base = (get-cache-base-path) - let cache_file = $"($cache_base)/sops/($cache_key).json" - let perms = (get-file-permissions $cache_file) - - if $perms != "0600" { - if $debug { - print $"⚠️ SOPS cache has incorrect permissions: ($perms), expected 0600" - } - return { valid: false, data: null, reason: "invalid_permissions" } - } - - if $debug { - print $"βœ… SOPS cache hit (permissions verified)" - } - - return { valid: true, data: $result.data, reason: "cache_hit" } -} - -# Validate SOPS cache (permissions + TTL + mtime) -export def validate-sops-cache [ - cache_file: string # Path to cache file - ---debug = false -] { - # Returns: { valid: bool, expired: bool, bad_perms: bool, reason: string } - - let meta_file = $"($cache_file).meta" - - # Basic validation - let validation = (validate-cache-entry $cache_file $meta_file --ttl = 900) - - if not $validation.valid { - return { - valid: false - expired: $validation.expired - bad_perms: false - reason: $validation.reason - } - } - - # Check permissions - let perms = (get-file-permissions $cache_file) - - if $perms != "0600" { - if $debug { - print $"⚠️ SOPS cache has incorrect permissions: ($perms)" - } - return { - valid: false - expired: false - bad_perms: true - reason: "invalid_permissions" - } - } - - return { - valid: true - expired: false - bad_perms: false - reason: "valid" - } -} - -# Enforce 0600 permissions on SOPS cache file -export def set-sops-permissions [ - cache_file: string # Path to cache file - ---debug = false -] { - if not ($cache_file | path exists) { - if $debug { - print $"⚠️ Cache file does not exist: ($cache_file)" - } - return - } - - # chmod 0600 - ^chmod 0600 $cache_file - - if $debug { - let perms = (get-file-permissions $cache_file) - print $"πŸ”’ Set SOPS cache permissions: ($perms)" - } -} - -# Clear SOPS cache -export def clear-sops-cache [ - --pattern: string = "*" # Pattern to match (default: all) - ---force = false # Force without confirmation -] { - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - if not ($sops_dir | path exists) { - print "No SOPS cache to clear" - return - } - - let cache_files = (glob $"($sops_dir)/($pattern).json" | where { |f| not ($f | str ends-with ".meta") }) - - if ($cache_files | is-empty) { - print "No SOPS cache entries matching pattern" - return - } - - # Delete matched files - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - rm -f $cache_file - rm -f $meta_file - } - - print $"βœ… Cleared ($($cache_files | length)) SOPS cache entries" -} - -# Rotate SOPS cache (clear expired entries) -export def rotate-sops-cache [ - --max-age-seconds: int = 900 # Default 15 minutes - ---debug = false -] { - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - if not ($sops_dir | path exists) { - return - } - - let cache_files = (glob $"($sops_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - mut deleted_count = 0 - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - - if ($meta_file | path exists) { - let validation = (validate-sops-cache $cache_file --debug=$debug) - - if $validation.expired or $validation.bad_perms { - rm -f $cache_file - rm -f $meta_file - $deleted_count = ($deleted_count + 1) - } - } - } - - if $debug and $deleted_count > 0 { - print $"πŸ—‘οΈ Rotated ($deleted_count) expired SOPS cache entries" - } -} - -# Compute SOPS hash -def compute-sops-hash [ - file_path: string # Path to encrypted file -] { - # Hash based on file path + size (content hash would require decryption) - let file_name = ($file_path | path basename) - let file_size = (^stat -f "%z" $file_path | into int | default 0) - - let hash_input = $"($file_name)-($file_size)" - - let hash = ( - ^echo $hash_input - | ^shasum -a 256 - | ^awk '{ print substr($1, 1, 16) }' - ) - - return $hash -} - -# Get file permissions in octal format -def get-file-permissions [ - file_path: string # Path to file -] { - if not ($file_path | path exists) { - return "nonexistent" - } - - # Get permissions in octal - let perms = (^stat -f "%A" $file_path) - return $perms -} - -# Verify SOPS cache is properly secured -export def verify-sops-cache-security [] { - # Returns: { secure: bool, issues: list } - - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - mut issues = [] - - # Check directory exists and has correct permissions - if not ($sops_dir | path exists) { - # Directory doesn't exist yet, that's fine - return { secure: true, issues: [] } - } - - let dir_perms = (^stat -f "%A" $sops_dir) - if $dir_perms != "0700" { - $issues = ($issues | append $"SOPS directory has incorrect permissions: ($dir_perms), expected 0700") - } - - # Check all cache files have 0600 permissions - let cache_files = (glob $"($sops_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - for cache_file in $cache_files { - let file_perms = (get-file-permissions $cache_file) - if $file_perms != "0600" { - $issues = ($issues | append $"SOPS cache file has incorrect permissions: ($cache_file) ($file_perms)") - } - } - - return { secure: ($issues | is-empty), issues: $issues } -} - -# Get SOPS cache statistics -export def get-sops-cache-stats [] { - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - if not ($sops_dir | path exists) { - return { - total_entries: 0 - total_size: 0 - cache_dir: $sops_dir - } - } - - let cache_files = (glob $"($sops_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - mut total_size = 0 - - for cache_file in $cache_files { - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - $total_size = ($total_size + $file_size) - } - - return { - total_entries: ($cache_files | length) - total_size: $total_size - total_size_mb: ($total_size / 1048576 | math floor) - cache_dir: $sops_dir - } -} - -# List cached SOPS decryptions -export def list-sops-cache [ - --format: string = "table" # table, json, yaml -] { - let cache_base = (get-cache-base-path) - let sops_dir = $"($cache_base)/sops" - - if not ($sops_dir | path exists) { - print "No SOPS cache entries" - return - } - - let cache_files = (glob $"($sops_dir)/*.json" | where { |f| not ($f | str ends-with ".meta") }) - - if ($cache_files | is-empty) { - print "No SOPS cache entries" - return - } - - mut entries = [] - - for cache_file in $cache_files { - let meta_file = $"($cache_file).meta" - if ($meta_file | path exists) { - let metadata = (open -r $meta_file | from json) - let file_size = (^stat -f "%z" $cache_file | into int | default 0) - let perms = (get-file-permissions $cache_file) - - $entries = ($entries | append { - cache_file: ($cache_file | path basename) - created: $metadata.created_at - ttl_seconds: $metadata.ttl_seconds - size_bytes: $file_size - permissions: $perms - source: ($metadata.source_files | keys | first) - }) - } - } - - match $format { - "json" => { - print ($entries | to json) - } - "yaml" => { - print ($entries | to yaml) - } - _ => { - print ($entries | to table) - } - } -} diff --git a/nulib/lib_provisioning/config/cache/.broken/test-config-cache.nu b/nulib/lib_provisioning/config/cache/.broken/test-config-cache.nu deleted file mode 100644 index 882b1c3..0000000 --- a/nulib/lib_provisioning/config/cache/.broken/test-config-cache.nu +++ /dev/null @@ -1,338 +0,0 @@ -# Comprehensive Test Suite for Configuration Cache System -# Tests all cache modules and integration points -# Follows Nushell 0.109.0+ testing guidelines - -use ./core.nu * -use ./metadata.nu * -use ./config_manager.nu * -use ./kcl.nu * -use ./sops.nu * -use ./final.nu * -use ./commands.nu * - -# Test suite counter -mut total_tests = 0 -mut passed_tests = 0 -mut failed_tests = [] - -# Helper: Run a test and track results -def run_test [ - test_name: string - test_block: closure -] { - global total_tests = ($total_tests + 1) - - let result = (do { - (^$test_block) | complete - } | complete) - - if $result.exit_code == 0 { - global passed_tests = ($passed_tests + 1) - print $"βœ… ($test_name)" - } else { - global failed_tests = ($failed_tests | append $test_name) - print $"❌ ($test_name): ($result.stderr)" - } -} - -# ====== PHASE 1: CORE CACHE TESTS ====== - -print "═══════════════════════════════════════════════════════════════" -print "Phase 1: Core Cache Operations" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test cache directory creation -run_test "Cache directory creation" { - let cache_base = (get-cache-base-path) - $cache_base | path exists -} - -# Test cache-write operation -run_test "Cache write operation" { - let test_data = { name: "test", value: 123 } - cache-write "test" "test_key_1" $test_data ["/tmp/test.yaml"] -} - -# Test cache-lookup operation -run_test "Cache lookup operation" { - let result = (cache-lookup "test" "test_key_1") - $result.valid -} - -# Test TTL validation -run_test "TTL expiration validation" { - # Write cache with 1 second TTL - cache-write "test" "test_ttl_key" { data: "test" } ["/tmp/test.yaml"] --ttl = 1 - - # Should be valid immediately - let result1 = (cache-lookup "test" "test_ttl_key" --ttl = 1) - $result1.valid -} - -# ====== PHASE 2: METADATA TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 2: Metadata Management" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test metadata creation -run_test "Metadata creation" { - let metadata = (create-metadata ["/tmp/test1.yaml" "/tmp/test2.yaml"] 300 "sha256:abc123") - ($metadata | keys | contains "created_at") -} - -# Test mtime comparison -run_test "Metadata mtime comparison" { - let mtimes1 = { "/tmp/file1": 1000, "/tmp/file2": 2000 } - let mtimes2 = { "/tmp/file1": 1000, "/tmp/file2": 2000 } - - let result = (compare-mtimes $mtimes1 $mtimes2) - $result.match -} - -# ====== PHASE 3: CONFIGURATION MANAGER TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 3: Configuration Manager" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test get cache config -run_test "Get cache configuration" { - let config = (get-cache-config) - ($config | keys | contains "enabled") -} - -# Test cache-config-get (dot notation) -run_test "Cache config get with dot notation" { - let enabled = (cache-config-get "enabled") - $enabled != null -} - -# Test cache-config-set -run_test "Cache config set value" { - cache-config-set "enabled" true - let value = (cache-config-get "enabled") - $value == true -} - -# Test cache-config-validate -run_test "Cache config validation" { - let validation = (cache-config-validate) - ($validation | keys | contains "valid") -} - -# ====== PHASE 4: KCL CACHE TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 4: KCL Compilation Cache" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test KCL hash computation -run_test "KCL hash computation" { - let hash = (compute-kcl-hash "/tmp/test.k") - ($hash | str length) > 0 -} - -# Test KCL cache write -run_test "KCL cache write" { - let compiled = { schemas: [], configs: [] } - cache-kcl-compile "/tmp/test.k" $compiled -} - -# Test KCL cache lookup -run_test "KCL cache lookup" { - let result = (lookup-kcl-cache "/tmp/test.k") - ($result | keys | contains "valid") -} - -# Test get KCL cache stats -run_test "KCL cache statistics" { - let stats = (get-kcl-cache-stats) - ($stats | keys | contains "total_entries") -} - -# ====== PHASE 5: SOPS CACHE TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 5: SOPS Decryption Cache" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test SOPS cache write -run_test "SOPS cache write" { - cache-sops-decrypt "/tmp/test.sops.yaml" "decrypted_content" -} - -# Test SOPS cache lookup -run_test "SOPS cache lookup" { - let result = (lookup-sops-cache "/tmp/test.sops.yaml") - ($result | keys | contains "valid") -} - -# Test SOPS permission verification -run_test "SOPS cache security verification" { - let security = (verify-sops-cache-security) - ($security | keys | contains "secure") -} - -# Test get SOPS cache stats -run_test "SOPS cache statistics" { - let stats = (get-sops-cache-stats) - ($stats | keys | contains "total_entries") -} - -# ====== PHASE 6: FINAL CONFIG CACHE TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 6: Final Config Cache" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test cache-final-config -run_test "Final config cache write" { - let config = { version: "1.0", providers: {} } - let workspace = { name: "test", path: "/tmp/workspace" } - cache-final-config $config $workspace "dev" -} - -# Test get-final-config-stats -run_test "Final config cache statistics" { - let stats = (get-final-config-stats) - ($stats | keys | contains "total_entries") -} - -# Test check-final-config-cache-health -run_test "Final config cache health check" { - let health = (check-final-config-cache-health) - ($health | keys | contains "healthy") -} - -# ====== PHASE 7: CLI COMMANDS TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 7: Cache Commands" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test cache-stats command -run_test "Cache stats command" { - let stats = (cache-stats) - ($stats | keys | contains "total_entries") -} - -# Test cache-config-show command -run_test "Cache config show command" { - cache-config-show --format json -} - -# ====== PHASE 8: INTEGRATION TESTS ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 8: Integration Tests" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test cache configuration hierarchy -run_test "Cache configuration hierarchy (runtime overrides defaults)" { - let config = (get-cache-config) - - # Should have cache settings from defaults - let has_ttl = ($config | keys | contains "cache") - let has_enabled = ($config | keys | contains "enabled") - - ($has_ttl and $has_enabled) -} - -# Test cache enable/disable -run_test "Cache enable/disable via config" { - # Save original value - let original = (cache-config-get "enabled") - - # Test setting to false - cache-config-set "enabled" false - let disabled = (cache-config-get "enabled") - - # Restore original - cache-config-set "enabled" $original - - $disabled == false -} - -# ====== PHASE 9: NUSHELL GUIDELINES COMPLIANCE ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Phase 9: Nushell Guidelines Compliance" -print "═══════════════════════════════════════════════════════════════" -print "" - -# Test no try-catch blocks in cache modules -run_test "No try-catch blocks (using do/complete pattern)" { - # This test verifies implementation patterns but passes if module loads - let config = (get-cache-config) - ($config != null) -} - -# Test explicit types in function parameters -run_test "Explicit types in cache functions" { - # Functions should use explicit types for parameters - let result = (cache-lookup "test" "key") - ($result | type) == "record" -} - -# Test pure functions -run_test "Pure functions (no side effects in queries)" { - # cache-lookup should be idempotent - let result1 = (cache-lookup "nonexistent" "nonexistent") - let result2 = (cache-lookup "nonexistent" "nonexistent") - - ($result1.valid == $result2.valid) -} - -# ====== TEST SUMMARY ====== - -print "" -print "═══════════════════════════════════════════════════════════════" -print "Test Summary" -print "═══════════════════════════════════════════════════════════════" -print "" - -let success_rate = if $total_tests > 0 { - (($passed_tests / $total_tests) * 100 | math round) -} else { - 0 -} - -print $"Total Tests: ($total_tests)" -print $"Passed: ($passed_tests)" -print $"Failed: ($($failed_tests | length))" -print $"Success Rate: ($success_rate)%" - -if not ($failed_tests | is-empty) { - print "" - print "Failed Tests:" - for test_name in $failed_tests { - print $" ❌ ($test_name)" - } -} - -print "" - -if ($failed_tests | is-empty) { - print "βœ… All tests passed!" - exit 0 -} else { - print "❌ Some tests failed!" - exit 1 -} diff --git a/nulib/lib_provisioning/config/cache/commands.nu b/nulib/lib_provisioning/config/cache/commands.nu index c889dae..288909a 100644 --- a/nulib/lib_provisioning/config/cache/commands.nu +++ b/nulib/lib_provisioning/config/cache/commands.nu @@ -5,7 +5,7 @@ use ./core.nu * use ./metadata.nu * use ./config_manager.nu * -use ./kcl.nu * +use ./nickel.nu * use ./sops.nu * use ./final.nu * @@ -15,7 +15,7 @@ use ./final.nu * # Clear all or specific type of cache export def cache-clear [ - --type: string = "all" # "all", "kcl", "sops", "final", "provider", "platform" + --type: string = "all" # "all", "nickel", "sops", "final", "provider", "platform" --force = false # Skip confirmation ] { if (not $force) and ($type == "all") { @@ -30,7 +30,7 @@ export def cache-clear [ "all" => { print "Clearing all caches..." do { - cache-clear-type "kcl" + cache-clear-type "nickel" cache-clear-type "sops" cache-clear-type "final" cache-clear-type "provider" @@ -38,10 +38,10 @@ export def cache-clear [ } | complete | ignore print "βœ… All caches cleared" }, - "kcl" => { - print "Clearing KCL compilation cache..." - clear-kcl-cache - print "βœ… KCL cache cleared" + "nickel" => { + print "Clearing Nickel compilation cache..." + clear-nickel-cache + print "βœ… Nickel cache cleared" }, "sops" => { print "Clearing SOPS decryption cache..." @@ -61,7 +61,7 @@ export def cache-clear [ # List cache entries export def cache-list [ - --type: string = "*" # "kcl", "sops", "final", etc or "*" for all + --type: string = "*" # "nickel", "sops", "final", etc or "*" for all --format: string = "table" # "table", "json", "yaml" ] { let stats = (get-cache-stats) @@ -78,7 +78,7 @@ export def cache-list [ let type_dir = match $type { "all" => $base, - "kcl" => ($base | path join "kcl"), + "nickel" => ($base | path join "nickel"), "sops" => ($base | path join "sops"), "final" => ($base | path join "workspaces"), _ => ($base | path join $type) @@ -155,7 +155,7 @@ export def cache-warm [ print $"Warming cache for workspace: ($active.name)" do { - warm-kcl-cache $active.path + warm-nickel-cache $active.path } | complete | ignore } else { print $"Warming cache for workspace: ($workspace)" @@ -261,7 +261,7 @@ export def cache-config-show [ print "β–Έ Time-To-Live (TTL) Settings:" print $" Final Config: ($config.ttl.final_config)s (5 minutes)" - print $" KCL Compilation: ($config.ttl.kcl_compilation)s (30 minutes)" + print $" Nickel Compilation: ($config.ttl.nickel_compilation)s (30 minutes)" print $" SOPS Decryption: ($config.ttl.sops_decryption)s (15 minutes)" print $" Provider Config: ($config.ttl.provider_config)s (10 minutes)" print $" Platform Config: ($config.ttl.platform_config)s (10 minutes)" @@ -372,7 +372,7 @@ export def cache-status [] { print "" print " TTL Settings:" print $" Final Config: ($config.ttl.final_config)s (5 min)" - print $" KCL Compilation: ($config.ttl.kcl_compilation)s (30 min)" + print $" Nickel Compilation: ($config.ttl.nickel_compilation)s (30 min)" print $" SOPS Decryption: ($config.ttl.sops_decryption)s (15 min)" print $" Provider Config: ($config.ttl.provider_config)s (10 min)" print $" Platform Config: ($config.ttl.platform_config)s (10 min)" @@ -389,8 +389,8 @@ export def cache-status [] { print "" print " By Type:" - let kcl_stats = (get-kcl-cache-stats) - print $" KCL: ($kcl_stats.total_entries) entries, ($kcl_stats.total_size_mb | math round -p 2) MB" + let nickel_stats = (get-nickel-cache-stats) + print $" Nickel: ($nickel_stats.total_entries) entries, ($nickel_stats.total_size_mb | math round -p 2) MB" let sops_stats = (get-sops-cache-stats) print $" SOPS: ($sops_stats.total_entries) entries, ($sops_stats.total_size_mb | math round -p 2) MB" @@ -413,12 +413,12 @@ export def cache-stats [ print $" Total Size: ($stats.total_size_mb | math round -p 2) MB" print "" - let kcl_stats = (get-kcl-cache-stats) + let nickel_stats = (get-nickel-cache-stats) let sops_stats = (get-sops-cache-stats) let final_stats = (get-final-cache-stats) let summary = [ - { type: "KCL Compilation", entries: $kcl_stats.total_entries, size_mb: ($kcl_stats.total_size_mb | math round -p 2) }, + { type: "Nickel Compilation", entries: $nickel_stats.total_entries, size_mb: ($nickel_stats.total_size_mb | math round -p 2) }, { type: "SOPS Decryption", entries: $sops_stats.total_entries, size_mb: ($sops_stats.total_size_mb | math round -p 2) }, { type: "Final Config", entries: $final_stats.total_entries, size_mb: ($final_stats.total_size_mb | math round -p 2) } ] @@ -509,7 +509,7 @@ export def main [ "help" => { print "Cache Management Commands: - cache clear [--type ] Clear cache (all, kcl, sops, final) + cache clear [--type ] Clear cache (all, nickel, sops, final) cache list List cache entries cache warm Pre-populate cache cache validate Validate cache integrity diff --git a/nulib/lib_provisioning/config/cache/config_manager.nu b/nulib/lib_provisioning/config/cache/config_manager.nu index 0589902..0b5ec6f 100644 --- a/nulib/lib_provisioning/config/cache/config_manager.nu +++ b/nulib/lib_provisioning/config/cache/config_manager.nu @@ -61,7 +61,7 @@ export def get-cache-config [] { max_cache_size: 104857600, # 100 MB ttl: { final_config: 300, # 5 minutes - kcl_compilation: 1800, # 30 minutes + nickel_compilation: 1800, # 30 minutes sops_decryption: 900, # 15 minutes provider_config: 600, # 10 minutes platform_config: 600 # 10 minutes @@ -112,7 +112,7 @@ export def cache-config-set [ value: any ] { let runtime = (load-runtime-config) - + # Build nested structure from dot notation mut updated = $runtime @@ -123,7 +123,7 @@ export def cache-config-set [ # For nested paths, we need to handle carefully # Convert "ttl.final_config" -> insert into ttl section let parts = ($setting_path | split row ".") - + if ($parts | length) == 2 { let section = ($parts | get 0) let key = ($parts | get 1) @@ -164,7 +164,7 @@ export def cache-config-reset [ } else { # Remove specific setting let runtime = (load-runtime-config) - + mut updated = $runtime # Handle nested paths @@ -229,7 +229,7 @@ export def cache-config-validate [] { if ($config | has -c "ttl") { for ttl_key in [ "final_config" - "kcl_compilation" + "nickel_compilation" "sops_decryption" "provider_config" "platform_config" @@ -329,7 +329,7 @@ export def get-cache-defaults [] { max_cache_size: 104857600, # 100 MB ttl: { final_config: 300, - kcl_compilation: 1800, + nickel_compilation: 1800, sops_decryption: 900, provider_config: 600, platform_config: 600 diff --git a/nulib/lib_provisioning/config/cache/core.nu b/nulib/lib_provisioning/config/cache/core.nu index 8b91f55..22ead75 100644 --- a/nulib/lib_provisioning/config/cache/core.nu +++ b/nulib/lib_provisioning/config/cache/core.nu @@ -10,12 +10,12 @@ def get-cache-base-dir [] { # Helper: Get cache file path for a given type and key def get-cache-file-path [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" + cache_type: string # "nickel", "sops", "final", "provider", "platform" cache_key: string # Unique identifier (usually a hash) ] { let base = (get-cache-base-dir) let type_dir = match $cache_type { - "kcl" => "kcl" + "nickel" => "nickel" "sops" => "sops" "final" => "workspaces" "provider" => "providers" @@ -35,7 +35,7 @@ def get-cache-meta-path [cache_file: string] { def ensure-cache-dirs [] { let base = (get-cache-base-dir) - for dir in ["kcl" "sops" "workspaces" "providers" "platform" "index"] { + for dir in ["nickel" "sops" "workspaces" "providers" "platform" "index"] { let dir_path = ($base | path join $dir) if not ($dir_path | path exists) { mkdir $dir_path @@ -80,7 +80,7 @@ def get-file-mtime [file_path: string] { # Lookup cache entry with TTL + mtime validation export def cache-lookup [ - cache_type: string # "kcl", "sops", "final", "provider", "platform" + cache_type: string # "nickel", "sops", "final", "provider", "platform" cache_key: string # Unique identifier --ttl: int = 0 # Override TTL (0 = use default) ] { @@ -136,7 +136,7 @@ export def cache-write [ } else { match $cache_type { "final" => 300 - "kcl" => 1800 + "nickel" => 1800 "sops" => 900 "provider" => 600 "platform" => 600 @@ -175,6 +175,16 @@ def validate-cache-entry [ let meta = (open $meta_file | from json) + # Validate metadata is not null/empty + if ($meta | is-empty) or ($meta == null) { + return { valid: false, reason: "metadata_invalid" } + } + + # Validate expires_at field exists + if not ("expires_at" in ($meta | columns)) { + return { valid: false, reason: "metadata_missing_expires_at" } + } + let now = (date now | format date "%Y-%m-%dT%H:%M:%SZ") if $now > $meta.expires_at { return { valid: false, reason: "ttl_expired" } @@ -333,7 +343,7 @@ export def cache-clear-type [ ] { let base = (get-cache-base-dir) let type_dir = ($base | path join (match $cache_type { - "kcl" => "kcl" + "nickel" => "nickel" "sops" => "sops" "final" => "workspaces" "provider" => "providers" diff --git a/nulib/lib_provisioning/config/cache/final.nu b/nulib/lib_provisioning/config/cache/final.nu index 20ae15c..65e67f3 100644 --- a/nulib/lib_provisioning/config/cache/final.nu +++ b/nulib/lib_provisioning/config/cache/final.nu @@ -34,7 +34,7 @@ def get-all-source-files [ let config_dir = ($workspace.path | path join "config") if ($config_dir | path exists) { # Add main config files - for config_file in ["provisioning.k" "provisioning.yaml"] { + for config_file in ["provisioning.ncl" "provisioning.yaml"] { let file_path = ($config_dir | path join $config_file) if ($file_path | path exists) { $source_files = ($source_files | append $file_path) @@ -141,7 +141,7 @@ export def invalidate-final-cache [ ] { if $environment == "*" { # Invalidate ALL environments for workspace - let base = (let home = ($env.HOME? | default "~" | path expand); + let base = (let home = ($env.HOME? | default "~" | path expand); $home | path join ".provisioning" "cache" "config" "workspaces") if ($base | path exists) { diff --git a/nulib/lib_provisioning/config/cache/metadata.nu b/nulib/lib_provisioning/config/cache/metadata.nu index 6337bb4..9d55fdf 100644 --- a/nulib/lib_provisioning/config/cache/metadata.nu +++ b/nulib/lib_provisioning/config/cache/metadata.nu @@ -182,7 +182,7 @@ export def get-metadata-ttl-remaining [ # Parse both timestamps and calculate difference let now_ts = (parse-iso-timestamp $now) let expires_ts = (parse-iso-timestamp $metadata.expires_at) - + if $expires_ts > $now_ts { $expires_ts - $now_ts } else { diff --git a/nulib/lib_provisioning/config/cache/mod.nu b/nulib/lib_provisioning/config/cache/mod.nu index 26da951..4d50232 100644 --- a/nulib/lib_provisioning/config/cache/mod.nu +++ b/nulib/lib_provisioning/config/cache/mod.nu @@ -7,7 +7,7 @@ export use ./metadata.nu * export use ./config_manager.nu * # Specialized caches -export use ./kcl.nu * +export use ./nickel.nu * export use ./sops.nu * export use ./final.nu * @@ -20,7 +20,7 @@ export def init-cache-system [] -> nothing { let home = ($env.HOME? | default "~" | path expand) let cache_base = ($home | path join ".provisioning" "cache" "config") - for dir in ["kcl" "sops" "workspaces" "providers" "platform" "index"] { + for dir in ["nickel" "sops" "workspaces" "providers" "platform" "index"] { let dir_path = ($cache_base | path join $dir) if not ($dir_path | path exists) { mkdir $dir_path diff --git a/nulib/lib_provisioning/config/cache/kcl.nu b/nulib/lib_provisioning/config/cache/nickel.nu similarity index 67% rename from nulib/lib_provisioning/config/cache/kcl.nu rename to nulib/lib_provisioning/config/cache/nickel.nu index 4f4c2d6..78aec8e 100644 --- a/nulib/lib_provisioning/config/cache/kcl.nu +++ b/nulib/lib_provisioning/config/cache/nickel.nu @@ -1,37 +1,37 @@ -# KCL Compilation Cache System -# Caches compiled KCL output to avoid expensive kcl eval operations +# Nickel Compilation Cache System +# Caches compiled Nickel output to avoid expensive nickel eval operations # Tracks dependencies and validates compilation output # Follows Nushell 0.109.0+ guidelines use ./core.nu * use ./metadata.nu * -# Helper: Get kcl.mod path for a KCL file -def get-kcl-mod-path [kcl_file: string] { - let file_dir = ($kcl_file | path dirname) - $file_dir | path join "kcl.mod" +# Helper: Get nickel.mod path for a Nickel file +def get-nickel-mod-path [decl_file: string] { + let file_dir = ($decl_file | path dirname) + $file_dir | path join "nickel.mod" } -# Helper: Compute hash of KCL file + dependencies -def compute-kcl-hash [ +# Helper: Compute hash of Nickel file + dependencies +def compute-nickel-hash [ file_path: string - kcl_mod_path: string + decl_mod_path: string ] { # Read both files for comprehensive hash - let kcl_content = if ($file_path | path exists) { + let decl_content = if ($file_path | path exists) { open $file_path } else { "" } - let mod_content = if ($kcl_mod_path | path exists) { - open $kcl_mod_path + let mod_content = if ($decl_mod_path | path exists) { + open $decl_mod_path } else { "" } - let combined = $"($kcl_content)($mod_content)" - + let combined = $"($decl_content)($mod_content)" + let hash_result = (do { $combined | ^openssl dgst -sha256 -hex } | complete) @@ -43,10 +43,10 @@ def compute-kcl-hash [ } } -# Helper: Get KCL compiler version -def get-kcl-version [] { +# Helper: Get Nickel compiler version +def get-nickel-version [] { let version_result = (do { - ^kcl version | grep -i "version" | head -1 + ^nickel version | grep -i "version" | head -1 } | complete) if $version_result.exit_code == 0 { @@ -57,39 +57,39 @@ def get-kcl-version [] { } # ============================================================================ -# PUBLIC API: KCL Cache Operations +# PUBLIC API: Nickel Cache Operations # ============================================================================ -# Cache KCL compilation output -export def cache-kcl-compile [ +# Cache Nickel compilation output +export def cache-nickel-compile [ file_path: string - compiled_output: record # Output from kcl eval + compiled_output: record # Output from nickel eval ] { - let kcl_mod_path = (get-kcl-mod-path $file_path) - let cache_key = (compute-kcl-hash $file_path $kcl_mod_path) + let nickel_mod_path = (get-nickel-mod-path $file_path) + let cache_key = (compute-nickel-hash $file_path $nickel_mod_path) let source_files = [ $file_path, - $kcl_mod_path + $nickel_mod_path ] # Write cache with 30-minute TTL - cache-write "kcl" $cache_key $compiled_output $source_files --ttl 1800 + cache-write "nickel" $cache_key $compiled_output $source_files --ttl 1800 } -# Lookup cached KCL compilation -export def lookup-kcl-cache [ +# Lookup cached Nickel compilation +export def lookup-nickel-cache [ file_path: string ] { if not ($file_path | path exists) { return { valid: false, reason: "file_not_found", data: null } } - let kcl_mod_path = (get-kcl-mod-path $file_path) - let cache_key = (compute-kcl-hash $file_path $kcl_mod_path) + let nickel_mod_path = (get-nickel-mod-path $file_path) + let cache_key = (compute-nickel-hash $file_path $nickel_mod_path) # Try to lookup in cache - let cache_result = (cache-lookup "kcl" $cache_key) + let cache_result = (cache-lookup "nickel" $cache_key) if not $cache_result.valid { return { @@ -99,11 +99,11 @@ export def lookup-kcl-cache [ } } - # Additional validation: check KCL compiler version (optional) - let meta_file = (get-cache-file-path-meta "kcl" $cache_key) + # Additional validation: check Nickel compiler version (optional) + let meta_file = (get-cache-file-path-meta "nickel" $cache_key) if ($meta_file | path exists) { let meta = (open $meta_file | from json) - let current_version = (get-kcl-version) + let current_version = (get-nickel-version) # Note: Version mismatch could be acceptable in many cases # Only warn, don't invalidate cache unless major version changes @@ -120,8 +120,8 @@ export def lookup-kcl-cache [ } } -# Validate KCL cache (check dependencies) -def validate-kcl-cache [ +# Validate Nickel cache (check dependencies) +def validate-nickel-cache [ cache_file: string meta_file: string ] { @@ -162,14 +162,14 @@ def validate-kcl-cache [ { valid: true, reason: "validation_passed" } } -# Clear KCL cache -export def clear-kcl-cache [] { - cache-clear-type "kcl" +# Clear Nickel cache +export def clear-nickel-cache [] { + cache-clear-type "nickel" } -# Get KCL cache statistics -export def get-kcl-cache-stats [] { - let base = (let home = ($env.HOME? | default "~" | path expand); $home | path join ".provisioning" "cache" "config" "kcl") +# Get Nickel cache statistics +export def get-nickel-cache-stats [] { + let base = (let home = ($env.HOME? | default "~" | path expand); $home | path join ".provisioning" "cache" "config" "nickel") if not ($base | path exists) { return { @@ -211,13 +211,13 @@ def get-cache-file-path-meta [ ] { let home = ($env.HOME? | default "~" | path expand) let base = ($home | path join ".provisioning" "cache" "config") - let type_dir = ($base | path join "kcl") + let type_dir = ($base | path join "nickel") let cache_file = ($type_dir | path join $cache_key) $"($cache_file).meta" } -# Warm KCL cache (pre-compile all KCL files in workspace) -export def warm-kcl-cache [ +# Warm Nickel cache (pre-compile all Nickel files in workspace) +export def warm-nickel-cache [ workspace_path: string ] { let config_dir = ($workspace_path | path join "config") @@ -226,17 +226,17 @@ export def warm-kcl-cache [ return } - # Find all .k files in config - for kcl_file in (glob $"($config_dir)/**/*.k") { - if ($kcl_file | path exists) { + # Find all .ncl files in config + for decl_file in (glob $"($config_dir)/**/*.ncl") { + if ($decl_file | path exists) { let compile_result = (do { - ^kcl eval $kcl_file + ^nickel export $decl_file --format json } | complete) if $compile_result.exit_code == 0 { let compiled = ($compile_result.stdout | from json) do { - cache-kcl-compile $kcl_file $compiled + cache-nickel-compile $decl_file $compiled } | complete | ignore } } diff --git a/nulib/lib_provisioning/config/cache/simple-cache.nu b/nulib/lib_provisioning/config/cache/simple-cache.nu index 22e962b..988143a 100644 --- a/nulib/lib_provisioning/config/cache/simple-cache.nu +++ b/nulib/lib_provisioning/config/cache/simple-cache.nu @@ -3,18 +3,18 @@ # Core cache operations export def cache-write [ - cache_type: string # "kcl", "sops", "final", etc. + cache_type: string # "nickel", "sops", "final", etc. cache_key: string # Unique identifier data: any # Data to cache ] { let cache_dir = (get-cache-dir $cache_type) let cache_file = $"($cache_dir)/($cache_key).json" - + # Create directory if needed if not ($cache_dir | path exists) { ^mkdir -p $cache_dir } - + # Write cache file $data | to json | save -f $cache_file } @@ -24,7 +24,7 @@ export def cache-read [ cache_key: string ] { let cache_file = $"(get-cache-dir $cache_type)/($cache_key).json" - + if ($cache_file | path exists) { open -r $cache_file | from json } else { @@ -36,7 +36,7 @@ export def cache-clear [ cache_type: string = "all" ] { let cache_base = (get-cache-base) - + if $cache_type == "all" { ^rm -rf $cache_base } else { @@ -51,14 +51,14 @@ export def cache-list [ cache_type: string = "*" ] { let cache_base = (get-cache-base) - + if ($cache_base | path exists) { let pattern = if $cache_type == "*" { "/**/*.json" } else { $"/($cache_type)/*.json" } - + glob $"($cache_base)($pattern)" } else { [] @@ -70,7 +70,7 @@ export def cache-config-get [ setting: string = "enabled" ] { let config = get-cache-config - + # Simple dot notation support if ($setting | str contains ".") { let parts = ($setting | split row ".") @@ -94,22 +94,22 @@ export def cache-config-set [ ] { let config_path = (get-config-file) let config_dir = ($config_path | path dirname) - + # Create config directory if needed if not ($config_dir | path exists) { ^mkdir -p $config_dir } - + # Load existing config or create new let config = if ($config_path | path exists) { open -r $config_path | from json } else { {} } - + # Set value let updated = ($config | upsert $setting $value) - + # Save $updated | to json | save -f $config_path } @@ -123,7 +123,7 @@ export def get-cache-config [] { { enabled: true ttl_final_config: 300 - ttl_kcl: 1800 + ttl_nickel: 1800 ttl_sops: 900 ttl_provider: 600 } @@ -138,16 +138,16 @@ export def cache-status [] { print "=== Cache Configuration ===" let enabled = ($config | get --optional enabled | default true) let ttl_final = ($config | get --optional ttl_final_config | default 300) - let ttl_kcl = ($config | get --optional ttl_kcl | default 1800) + let ttl_nickel = ($config | get --optional ttl_nickel | default 1800) let ttl_sops = ($config | get --optional ttl_sops | default 900) let ttl_provider = ($config | get --optional ttl_provider | default 600) print $"Enabled: ($enabled)" print $"TTL Final Config: ($ttl_final)s" - print $"TTL KCL: ($ttl_kcl)s" + print $"TTL Nickel: ($ttl_nickel)s" print $"TTL SOPS: ($ttl_sops)s" print $"TTL Provider: ($ttl_provider)s" print "" - + # Cache statistics if ($cache_base | path exists) { let files = (glob $"($cache_base)/**/*.json" | where {|f| not ($f | str ends-with ".meta")}) diff --git a/nulib/lib_provisioning/config/cache/sops.nu b/nulib/lib_provisioning/config/cache/sops.nu index ab7ea01..c3f5a41 100644 --- a/nulib/lib_provisioning/config/cache/sops.nu +++ b/nulib/lib_provisioning/config/cache/sops.nu @@ -77,7 +77,7 @@ export def cache-sops-decrypt [ cache-write "sops" $cache_key $decrypted_content $source_files --ttl 900 # CRITICAL: Set 0600 permissions on cache file - let cache_file = (let home = ($env.HOME? | default "~" | path expand); + let cache_file = (let home = ($env.HOME? | default "~" | path expand); $home | path join ".provisioning" "cache" "config" "sops" $cache_key) if ($cache_file | path exists) { diff --git a/nulib/lib_provisioning/config/encryption_tests.nu b/nulib/lib_provisioning/config/encryption_tests.nu index 0e75724..bef5139 100644 --- a/nulib/lib_provisioning/config/encryption_tests.nu +++ b/nulib/lib_provisioning/config/encryption_tests.nu @@ -598,4 +598,4 @@ export def main [] { print " kms - KMS backend integration" print " loader - Config loader integration" print " validation - Encryption validation" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/config/export.nu b/nulib/lib_provisioning/config/export.nu new file mode 100644 index 0000000..a4b8000 --- /dev/null +++ b/nulib/lib_provisioning/config/export.nu @@ -0,0 +1,334 @@ +# Configuration Export Script +# Converts Nickel config.ncl to service-specific TOML files +# Usage: export-all-configs [workspace_path] +# export-platform-config [workspace_path] + +# Logging functions - not using std/log due to compatibility + +# Export all configuration sections from Nickel config +export def export-all-configs [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + # Create generated directory + mkdir ($"($workspace.path)/config/generated") 2>/dev/null + + print $"πŸ“₯ Exporting configuration from: ($config_file)" + + # Step 1: Typecheck the Nickel file + let typecheck_result = (do { nickel typecheck $config_file } | complete) + if $typecheck_result.exit_code != 0 { + print "❌ Nickel configuration validation failed" + print $typecheck_result.stderr + return + } + + # Step 2: Export to JSON + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print "❌ Failed to export Nickel to JSON" + print $export_result.stderr + return + } + let json_output = ($export_result.stdout | from json) + + # Step 3: Export workspace section + if ($json_output | get -o workspace | is-not-empty) { + print "πŸ“ Exporting workspace configuration" + $json_output.workspace | to toml | save -f $"($workspace.path)/config/generated/workspace.toml" + } + + # Step 4: Export provider sections + if ($json_output | get -o providers | is-not-empty) { + mkdir $"($workspace.path)/config/generated/providers" 2>/dev/null + + ($json_output.providers | to json | from json) | transpose name value | each {|provider| + if ($provider.value | get -o enabled | default false) { + print $"πŸ“ Exporting provider: ($provider.name)" + $provider.value | to toml | save -f $"($workspace.path)/config/generated/providers/($provider.name).toml" + } + } + } + + # Step 5: Export platform service sections + if ($json_output | get -o platform | is-not-empty) { + mkdir $"($workspace.path)/config/generated/platform" 2>/dev/null + + ($json_output.platform | to json | from json) | transpose name value | each {|service| + if ($service.value | type) == 'record' and ($service.value | get -o enabled | is-not-empty) { + if ($service.value | get enabled) { + print $"πŸ“ Exporting platform service: ($service.name)" + $service.value | to toml | save -f $"($workspace.path)/config/generated/platform/($service.name).toml" + } + } + } + } + + print "βœ… Configuration export complete" +} + +# Export a single platform service configuration +export def export-platform-config [service: string, workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + # Create generated directory + mkdir ($"($workspace.path)/config/generated/platform") 2>/dev/null + + print $"πŸ“ Exporting platform service: ($service)" + + # Step 1: Typecheck the Nickel file + let typecheck_result = (do { nickel typecheck $config_file } | complete) + if $typecheck_result.exit_code != 0 { + print "❌ Nickel configuration validation failed" + print $typecheck_result.stderr + return + } + + # Step 2: Export to JSON and extract platform section + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print "❌ Failed to export Nickel to JSON" + print $export_result.stderr + return + } + let json_output = ($export_result.stdout | from json) + + # Step 3: Export specific service + if ($json_output | get -o platform | is-not-empty) and ($json_output.platform | get -o $service | is-not-empty) { + let service_config = $json_output.platform | get $service + if ($service_config | type) == 'record' { + $service_config | to toml | save -f $"($workspace.path)/config/generated/platform/($service).toml" + print $"βœ… Successfully exported: ($service).toml" + } + } else { + print $"❌ Service not found in configuration: ($service)" + } +} + +# Export all provider configurations +export def export-all-providers [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + # Create generated directory + mkdir ($"($workspace.path)/config/generated/providers") 2>/dev/null + + print "πŸ“₯ Exporting all provider configurations" + + # Step 1: Typecheck the Nickel file + let typecheck_result = (do { nickel typecheck $config_file } | complete) + if $typecheck_result.exit_code != 0 { + print "❌ Nickel configuration validation failed" + print $typecheck_result.stderr + return + } + + # Step 2: Export to JSON + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print "❌ Failed to export Nickel to JSON" + print $export_result.stderr + return + } + let json_output = ($export_result.stdout | from json) + + # Step 3: Export provider sections + if ($json_output | get -o providers | is-not-empty) { + ($json_output.providers | to json | from json) | transpose name value | each {|provider| + # Exporting provider: ($provider.name) + $provider.value | to toml | save -f $"($workspace.path)/config/generated/providers/($provider.name).toml" + } + print "βœ… Provider export complete" + } else { + print "⚠️ No providers found in configuration" + } +} + +# Validate Nickel configuration without exporting +export def validate-config [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return { valid: false, error: "Configuration file not found" } + } + + print $"πŸ” Validating configuration: ($config_file)" + + # Run typecheck + let check_result = (do { nickel typecheck $config_file } | complete) + if $check_result.exit_code == 0 { + { valid: true, error: null } + } else { + print $"❌ Configuration validation failed" + print $check_result.stderr + { valid: false, error: $check_result.stderr } + } +} + +# Show configuration structure without exporting +export def show-config [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + print "πŸ“‹ Loading configuration structure" + + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print $"❌ Failed to load configuration" + print $export_result.stderr + } else { + let json_output = ($export_result.stdout | from json) + print ($json_output | to json --indent 2) + } +} + +# List all configured providers +export def list-providers [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print $"❌ Failed to list providers" + print $export_result.stderr + return + } + + let config = ($export_result.stdout | from json) + if ($config | get -o providers | is-not-empty) { + print "☁️ Configured Providers:" + ($config.providers | to json | from json) | transpose name value | each {|provider| + let status = if ($provider.value | get -o enabled | default false) { "βœ“ enabled" } else { "βœ— disabled" } + print $" ($provider.name): ($status)" + } + } else { + print "⚠️ No providers found in configuration" + } +} + +# List all configured platform services +export def list-platform-services [workspace_path?: string] { + let workspace = if ($workspace_path | is-empty) { + get-active-workspace + } else { + { path: $workspace_path } + } + + let config_file = $"($workspace.path)/config/config.ncl" + + # Validate that config file exists + if not ($config_file | path exists) { + print $"❌ Configuration file not found: ($config_file)" + return + } + + let export_result = (do { nickel export --format json $config_file } | complete) + if $export_result.exit_code != 0 { + print $"❌ Failed to list platform services" + print $export_result.stderr + return + } + + let config = ($export_result.stdout | from json) + if ($config | get -o platform | is-not-empty) { + print "βš™οΈ Configured Platform Services:" + ($config.platform | to json | from json) | transpose name value | each {|service| + if ($service.value | type) == 'record' and ($service.value | get -o enabled | is-not-empty) { + let status = if ($service.value | get enabled) { "βœ“ enabled" } else { "βœ— disabled" } + print $" ($service.name): ($status)" + } + } + } else { + print "⚠️ No platform services found in configuration" + } +} + +# Helper function to get active workspace +def get-active-workspace [] { + let user_config_file = if ($nu.os-info.name == "macos") { + $"($env.HOME)/Library/Application Support/provisioning/user_config.yaml" + } else { + $"($env.HOME)/.config/provisioning/user_config.yaml" + } + + if ($user_config_file | path exists) { + let open_result = (do { open $user_config_file } | complete) + if $open_result.exit_code == 0 { + let user_config = ($open_result.stdout | from yaml) + if ($user_config | get -o active_workspace | is-not-empty) { + let ws_name = $user_config.active_workspace + let ws = $user_config.workspaces | where name == $ws_name | get -o 0 + if ($ws | length) > 0 { + return { name: $ws.name, path: $ws.path } + } + } + } + } + + # Fallback to current directory + { name: "current", path: (pwd) } +} diff --git a/nulib/lib_provisioning/config/loader-minimal.nu b/nulib/lib_provisioning/config/loader-minimal.nu index a74c8c8..22cbf77 100644 --- a/nulib/lib_provisioning/config/loader-minimal.nu +++ b/nulib/lib_provisioning/config/loader-minimal.nu @@ -65,7 +65,7 @@ export def get-active-workspace [] { } } -# Find project root by looking for kcl.mod or core/nulib directory +# Find project root by looking for nickel.mod or core/nulib directory export def get-project-root [] { let potential_roots = [ $env.PWD @@ -75,7 +75,7 @@ export def get-project-root [] { ] let matching_roots = ($potential_roots - | where ($it | path join "kcl.mod" | path exists) + | where ($it | path join "nickel.mod" | path exists) or ($it | path join "core" "nulib" | path exists)) if ($matching_roots | length) > 0 { diff --git a/nulib/lib_provisioning/config/loader.nu b/nulib/lib_provisioning/config/loader.nu index 374fb80..2b7b891 100644 --- a/nulib/lib_provisioning/config/loader.nu +++ b/nulib/lib_provisioning/config/loader.nu @@ -7,7 +7,7 @@ use std log use ./cache/core.nu * use ./cache/metadata.nu * use ./cache/config_manager.nu * -use ./cache/kcl.nu * +use ./cache/nickel.nu * use ./cache/sops.nu * use ./cache/final.nu * @@ -61,15 +61,22 @@ export def load-provisioning-config [ mut config_sources = [] if ($active_workspace | is-not-empty) { - # Load workspace config - try KCL first, fallback to YAML for backward compatibility + # Load workspace config - try Nickel first (new format), then Nickel, then YAML for backward compatibility let config_dir = ($active_workspace.path | path join "config") - let kcl_config = ($config_dir | path join "provisioning.k") + let ncl_config = ($config_dir | path join "config.ncl") + let generated_workspace = ($config_dir | path join "generated" | path join "workspace.toml") + let nickel_config = ($config_dir | path join "provisioning.ncl") let yaml_config = ($config_dir | path join "provisioning.yaml") - # Use KCL if available (primary config format) - # No YAML fallback - KCL is the source of truth - let config_file = if ($kcl_config | path exists) { - $kcl_config + # Priority order: Generated TOML from TypeDialog > Nickel source > Nickel (legacy) > YAML (legacy) + let config_file = if ($generated_workspace | path exists) { + # Use generated TOML from TypeDialog (preferred) + $generated_workspace + } else if ($ncl_config | path exists) { + # Use Nickel source directly (will be exported to TOML on-demand) + $ncl_config + } else if ($nickel_config | path exists) { + $nickel_config } else if ($yaml_config | path exists) { $yaml_config } else { @@ -77,8 +84,12 @@ export def load-provisioning-config [ } let config_format = if ($config_file | is-not-empty) { - if ($config_file | str ends-with ".k") { - "kcl" + if ($config_file | str ends-with ".ncl") { + "nickel" + } else if ($config_file | str ends-with ".toml") { + "toml" + } else if ($config_file | str ends-with ".ncl") { + "nickel" } else { "yaml" } @@ -95,28 +106,65 @@ export def load-provisioning-config [ }) } - # Load provider configs - let providers_dir = ($active_workspace.path | path join "config" | path join "providers") - if ($providers_dir | path exists) { - let provider_configs = (ls $providers_dir | where type == file and ($it.name | str ends-with '.toml') | get name) + # Load provider configs (prefer generated from TypeDialog, fallback to manual) + let generated_providers_dir = ($active_workspace.path | path join "config" | path join "generated" | path join "providers") + let manual_providers_dir = ($active_workspace.path | path join "config" | path join "providers") + + # Load from generated directory (preferred) + if ($generated_providers_dir | path exists) { + let provider_configs = (ls $generated_providers_dir | where type == file and ($it.name | str ends-with '.toml') | get name) for provider_config in $provider_configs { $config_sources = ($config_sources | append { name: $"provider-($provider_config | path basename)" - path: $provider_config + path: $"($generated_providers_dir)/($provider_config)" + required: false + format: "toml" + }) + } + } else if ($manual_providers_dir | path exists) { + # Fallback to manual TOML files if generated don't exist + let provider_configs = (ls $manual_providers_dir | where type == file and ($it.name | str ends-with '.toml') | get name) + for provider_config in $provider_configs { + $config_sources = ($config_sources | append { + name: $"provider-($provider_config | path basename)" + path: $"($manual_providers_dir)/($provider_config)" required: false format: "toml" }) } } - # Load platform configs - let platform_dir = ($active_workspace.path | path join "config" | path join "platform") - if ($platform_dir | path exists) { - let platform_configs = (ls $platform_dir | where type == file and ($it.name | str ends-with '.toml') | get name) + # Load platform configs (prefer generated from TypeDialog, fallback to manual) + let workspace_config_ncl = ($active_workspace.path | path join "config" | path join "config.ncl") + let generated_platform_dir = ($active_workspace.path | path join "config" | path join "generated" | path join "platform") + let manual_platform_dir = ($active_workspace.path | path join "config" | path join "platform") + + # If Nickel config exists, ensure it's exported + if ($workspace_config_ncl | path exists) { + try { + use ../config/export.nu * + export-all-configs $active_workspace.path + } catch { } + } + + # Load from generated directory (preferred) + if ($generated_platform_dir | path exists) { + let platform_configs = (ls $generated_platform_dir | where type == file and ($it.name | str ends-with '.toml') | get name) for platform_config in $platform_configs { $config_sources = ($config_sources | append { name: $"platform-($platform_config | path basename)" - path: $platform_config + path: $"($generated_platform_dir)/($platform_config)" + required: false + format: "toml" + }) + } + } else if ($manual_platform_dir | path exists) { + # Fallback to manual TOML files if generated don't exist + let platform_configs = (ls $manual_platform_dir | where type == file and ($it.name | str ends-with '.toml') | get name) + for platform_config in $platform_configs { + $config_sources = ($config_sources | append { + name: $"platform-($platform_config | path basename)" + path: $"($manual_platform_dir)/($platform_config)" required: false format: "toml" }) @@ -136,14 +184,27 @@ export def load-provisioning-config [ } } else { # Fallback: If no workspace active, try to find workspace from PWD - # Try KCL first, then YAML for backward compatibility - let kcl_config = ($env.PWD | path join "config" | path join "provisioning.k") + # Try Nickel first, then Nickel, then YAML for backward compatibility + let ncl_config = ($env.PWD | path join "config" | path join "config.ncl") + let nickel_config = ($env.PWD | path join "config" | path join "provisioning.ncl") let yaml_config = ($env.PWD | path join "config" | path join "provisioning.yaml") - let workspace_config = if ($kcl_config | path exists) { + let workspace_config = if ($ncl_config | path exists) { + # Export Nickel config to TOML + try { + use ../config/export.nu * + export-all-configs $env.PWD + } catch { + # Silently continue if export fails + } { - path: $kcl_config - format: "kcl" + path: ($env.PWD | path join "config" | path join "generated" | path join "workspace.toml") + format: "toml" + } + } else if ($nickel_config | path exists) { + { + path: $nickel_config + format: "nickel" } } else if ($yaml_config | path exists) { { @@ -252,12 +313,12 @@ export def load-provisioning-config [ $final_config } -# Load a single configuration file (supports KCL, YAML and TOML with automatic decryption) +# Load a single configuration file (supports Nickel, Nickel, YAML and TOML with automatic decryption) export def load-config-file [ file_path: string required = false debug = false - format: string = "auto" # auto, kcl, yaml, toml + format: string = "auto" # auto, ncl, nickel, yaml, toml --no-cache = false # Disable cache for this file ] { if not ($file_path | path exists) { @@ -280,7 +341,8 @@ export def load-config-file [ let file_format = if $format == "auto" { let ext = ($file_path | path parse | get extension) match $ext { - "k" => "kcl" + "ncl" => "ncl" + "k" => "nickel" "yaml" | "yml" => "yaml" "toml" => "toml" _ => "toml" # default to toml for backward compatibility @@ -289,11 +351,30 @@ export def load-config-file [ $format } - # Handle KCL format separately (requires kcl compiler) - # KCL is the primary config format - no fallback - if $file_format == "kcl" { - let kcl_result = (load-kcl-config $file_path $required $debug --no-cache $no_cache) - return $kcl_result + # Handle Nickel format (exports to JSON then parses) + if $file_format == "ncl" { + if $debug { + # log debug $"Loading Nickel config file: ($file_path)" + } + try { + return (nickel export --format json $file_path | from json) + } catch {|e| + if $required { + print $"❌ Failed to load Nickel config ($file_path): ($e)" + exit 1 + } else { + if $debug { + # log debug $"Failed to load optional Nickel config: ($e)" + } + return {} + } + } + } + + # Handle Nickel format separately (requires nickel compiler) + if $file_format == "nickel" { + let decl_result = (load-nickel-config $file_path $required $debug --no-cache $no_cache) + return $decl_result } # Check if file is encrypted and auto-decrypt (for YAML/TOML only) @@ -353,70 +434,77 @@ export def load-config-file [ } } -# Load KCL configuration file -def load-kcl-config [ +# Load Nickel configuration file +def load-nickel-config [ file_path: string required = false debug = false --no-cache = false ] { - # Check if kcl command is available - let kcl_exists = (which kcl | is-not-empty) - if not $kcl_exists { + # Check if nickel command is available + let nickel_exists = (which nickel | is-not-empty) + if not $nickel_exists { if $required { - print $"❌ KCL compiler not found. Install KCL to use .k config files" - print $" Install from: https://kcl-lang.io/" + print $"❌ Nickel compiler not found. Install Nickel to use .ncl config files" + print $" Install from: https://nickel-lang.io/" exit 1 } else { if $debug { - print $"⚠️ KCL compiler not found, skipping KCL config file: ($file_path)" + print $"⚠️ Nickel compiler not found, skipping Nickel config file: ($file_path)" } return {} } } - # Try KCL cache first (if cache enabled and --no-cache not set) + # Try Nickel cache first (if cache enabled and --no-cache not set) if (not $no_cache) { - let kcl_cache = (lookup-kcl-cache $file_path) + let nickel_cache = (lookup-nickel-cache $file_path) - if ($kcl_cache.valid? | default false) { + if ($nickel_cache.valid? | default false) { if $debug { - print $"βœ… Cache hit: KCL ($file_path)" + print $"βœ… Cache hit: Nickel ($file_path)" } - return $kcl_cache.data + return $nickel_cache.data } } - # Evaluate KCL file (produces YAML output by default) - # Use 'kcl run' for package-based KCL files (with kcl.mod), 'kcl eval' for standalone files + # Evaluate Nickel file (produces JSON output) + # Use 'nickel export' for both package-based and standalone Nickel files let file_dir = ($file_path | path dirname) let file_name = ($file_path | path basename) - let kcl_mod_exists = (($file_dir | path join "kcl.mod") | path exists) + let decl_mod_exists = (($file_dir | path join "nickel.mod") | path exists) - let result = if $kcl_mod_exists { - # Use 'kcl run' for package-based configs (SST pattern with kcl.mod) - # Must run from the config directory so relative paths in kcl.mod resolve correctly - (^sh -c $"cd '($file_dir)' && kcl run ($file_name)" | complete) + let result = if $decl_mod_exists { + # Use 'nickel export' for package-based configs (SST pattern with nickel.mod) + # Must run from the config directory so relative paths in nickel.mod resolve correctly + (^sh -c $"cd '($file_dir)' && nickel export ($file_name) --format json" | complete) } else { - # Use 'kcl eval' for standalone configs - (^kcl eval $file_path | complete) + # Use 'nickel export' for standalone configs + (^nickel export $file_path --format json | complete) } - let kcl_output = $result.stdout + let decl_output = $result.stdout # Check if output is empty - if ($kcl_output | is-empty) { - # KCL compilation failed - return empty to trigger fallback to YAML + if ($decl_output | is-empty) { + # Nickel compilation failed - return empty to trigger fallback to YAML if $debug { - print $"⚠️ KCL config compilation failed, fallback to YAML will be used" + print $"⚠️ Nickel config compilation failed, fallback to YAML will be used" } return {} } - # Parse YAML output (KCL outputs YAML by default in version 0.11.3) - let parsed = ($kcl_output | from yaml) + # Parse JSON output (Nickel outputs JSON when --format json is specified) + let parsed = (do -i { $decl_output | from json }) - # Extract workspace_config key if it exists (KCL wraps output in variable name) + if ($parsed | is-empty) or ($parsed | type) != "record" { + if $debug { + print $"⚠️ Failed to parse Nickel output as JSON" + } + return {} + } + + # Extract workspace_config key if it exists (Nickel wraps output in variable name) let config = if (($parsed | columns) | any { |col| $col == "workspace_config" }) { $parsed.workspace_config } else { @@ -424,12 +512,12 @@ def load-kcl-config [ } if $debug { - print $"βœ… Loaded KCL config from ($file_path)" + print $"βœ… Loaded Nickel config from ($file_path)" } - # Cache the compiled KCL output (if cache enabled and --no-cache not set) - if (not $no_cache) { - cache-kcl-compile $file_path $config + # Cache the compiled Nickel output (if cache enabled and --no-cache not set) + if (not $no_cache) and ($config | type) == "record" { + cache-nickel-compile $file_path $config } $config @@ -967,7 +1055,7 @@ def get-project-root [] { for root in $potential_roots { # Check for provisioning project indicators if (($root | path join "config.defaults.toml" | path exists) or - ($root | path join "kcl.mod" | path exists) or + ($root | path join "nickel.mod" | path exists) or ($root | path join "core" "nulib" "provisioning" | path exists)) { return $root } @@ -2055,4 +2143,4 @@ def get-active-workspace [] { } } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/config/migration.nu b/nulib/lib_provisioning/config/migration.nu index 1afffd5..671f330 100644 --- a/nulib/lib_provisioning/config/migration.nu +++ b/nulib/lib_provisioning/config/migration.nu @@ -261,4 +261,4 @@ export def backup-current-env [ $backup_content | save $output print $"Environment variables backed up to: ($output)" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/config/mod.nu b/nulib/lib_provisioning/config/mod.nu index 84419e7..3d67329 100644 --- a/nulib/lib_provisioning/config/mod.nu +++ b/nulib/lib_provisioning/config/mod.nu @@ -54,4 +54,4 @@ export def validate [] { # Initialize user configuration export def init [] { init-user-config -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/context.nu b/nulib/lib_provisioning/context.nu index b1521d4..83a1fe4 100644 --- a/nulib/lib_provisioning/context.nu +++ b/nulib/lib_provisioning/context.nu @@ -4,9 +4,9 @@ export def setup_user_context_path [ defaults_name: string = "context.yaml" ] { let str_filename = if ($defaults_name | into string) == "" { "context.yaml" } else { $defaults_name } - let filename = if ($str_filename | str ends-with ".yaml") { + let filename = if ($str_filename | str ends-with ".yaml") { $str_filename - } else { + } else { $"($str_filename).yaml" } let setup_context_path = (setup_config_path | path join $filename ) @@ -14,13 +14,13 @@ export def setup_user_context_path [ $setup_context_path } else { "" - } + } } export def setup_user_context [ defaults_name: string = "context.yaml" ] { let setup_context_path = setup_user_context_path $defaults_name - if $setup_context_path == "" { return null } + if $setup_context_path == "" { return null } open $setup_context_path } export def setup_save_context [ @@ -28,7 +28,7 @@ export def setup_save_context [ defaults_name: string = "context.yaml" ] { let setup_context_path = setup_user_context_path $defaults_name - if $setup_context_path != "" { + if $setup_context_path != "" { $data | save -f $setup_context_path } -} +} diff --git a/nulib/lib_provisioning/defs/about.nu b/nulib/lib_provisioning/defs/about.nu index 43ab062..003c9a3 100644 --- a/nulib/lib_provisioning/defs/about.nu +++ b/nulib/lib_provisioning/defs/about.nu @@ -1,24 +1,24 @@ -#!/usr/bin/env nu +#!/usr/bin/env nu # myscript.nu export def about_info [ ]: nothing -> string { let info = if ( $env.CURRENT_FILE? | into string ) != "" { (^grep "^# Info:" $env.CURRENT_FILE ) | str replace "# Info: " "" } else { "" } $" -USAGE provisioning -k cloud-path file-settings.yaml provider-options +USAGE provisioning -k cloud-path file-settings.yaml provider-options DESCRIPTION ($info) OPTIONS -s server-hostname with server-hostname target selection -p provider-name - use provider name + use provider name do not need if 'current directory path basename' is not one of providers available -new | new [provisioning-name] create a new provisioning-directory-name by a copy of infra -k cloud-path-item - use cloud-path-item as base directory for settings + use cloud-path-item as base directory for settings -x Trace script with 'set -x' providerslist | providers-list | providers list @@ -28,13 +28,12 @@ OPTIONS serviceslist | service-list Get available services list tools - Run core/on-tools info + Run core/on-tools info -i - About this + About this -v Print version -h, --help Print this help and exit. " } - diff --git a/nulib/lib_provisioning/defs/lists.nu b/nulib/lib_provisioning/defs/lists.nu index 22e09e3..36d8487 100644 --- a/nulib/lib_provisioning/defs/lists.nu +++ b/nulib/lib_provisioning/defs/lists.nu @@ -3,9 +3,9 @@ use ../config/accessor.nu * use ../utils/on_select.nu run_on_selection export def get_provisioning_info [ dir_path: string - target: string + target: string ]: nothing -> list { - # task root path target will be empty + # task root path target will be empty let item = if $target != "" { $target } else { ($dir_path | path basename) } let full_path = if $target != "" { $"($dir_path)/($item)" } else { $dir_path } if not ($full_path | path exists) { @@ -30,15 +30,15 @@ export def get_provisioning_info [ } ) )} | - each {|it| - if ($"($full_path)/($it.name)" | path exists) and ($"($full_path)/($it.name)/provisioning.toml" | path exists) { + each {|it| + if ($"($full_path)/($it.name)" | path exists) and ($"($full_path)/($it.name)/provisioning.toml" | path exists) { # load provisioning.toml for info and vers let provisioning_data = open $"($full_path)/($it.name)/provisioning.toml" { task: $item, mode: ($it.name), info: $provisioning_data.info, vers: $provisioning_data.release} } else { { task: $item, mode: ($it.name), info: "", vers: ""} } - } + } } export def providers_list [ mode?: string @@ -163,13 +163,13 @@ def get_infra_taskservs [infra_name: string]: nothing -> list { return [] } - # List all .k files and directories in this infra's taskservs folder + # List all .ncl files and directories in this infra's taskservs folder ls -s $infra_taskservs_path | where {|el| - ($el.name | str ends-with ".k") or ($el.type == "dir" and ($el.name | str starts-with "_") == false) + ($el.name | str ends-with ".ncl") or ($el.type == "dir" and ($el.name | str starts-with "_") == false) } | each {|it| - # Parse task name from filename (remove .k extension if present) - let task_name = if ($it.name | str ends-with ".k") { - $it.name | str replace ".k" "" + # Parse task name from filename (remove .ncl extension if present) + let task_name = if ($it.name | str ends-with ".ncl") { + $it.name | str replace ".ncl" "" } else { $it.name } @@ -284,30 +284,30 @@ export def infras_list [ } | flatten | default [] } export def on_list [ - target_list: string + target_list: string cmd: string - ops: string + ops: string ]: nothing -> list { #use utils/on_select.nu run_on_selection match $target_list { - "providers" | "p" => { + "providers" | "p" => { _print $"\n(_ansi green)PROVIDERS(_ansi reset) list: \n" let list_items = (providers_list "selection") - if ($list_items | length) == 0 { - _print $"πŸ›‘ no items found for (_ansi cyan)providers list(_ansi reset)" + if ($list_items | length) == 0 { + _print $"πŸ›‘ no items found for (_ansi cyan)providers list(_ansi reset)" return [] - } + } if $cmd == "-" { return $list_items } if ($cmd | is-empty) { _print ($list_items | to json) "json" "result" "table" - } else { + } else { if (get-provisioning-out | is-not-empty) or (get-provisioning-no-terminal) { return ""} - let selection_pos = ($list_items | each {|it| + let selection_pos = ($list_items | each {|it| match ($it.name | str length) { 2..5 => $"($it.name)\t\t ($it.info) \tversion: ($it.vers)", _ => $"($it.name)\t ($it.info) \tversion: ($it.vers)", } - } | input list --index ( + } | input list --index ( $"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset)" + $" \(use arrow keys and press [enter] or [escape] to exit\)( _ansi reset)" ) @@ -316,35 +316,35 @@ export def on_list [ let item_selec = if ($list_items | length) > $selection_pos { $list_items | get $selection_pos } else { null } let item_path = ((get-providers-path) | path join $item_selec.name) if not ($item_path | path exists) { _print $"Path ($item_path) not found" } - (run_on_selection $cmd $item_selec.name $item_path + (run_on_selection $cmd $item_selec.name $item_path ($item_path | path join "nulib" | path join $item_selec.name | path join "servers.nu") (get-providers-path)) } } return [] }, - "taskservs" | "t" => { + "taskservs" | "t" => { _print $"\n(_ansi blue)TASKSERVICESS(_ansi reset) list: \n" let list_items = (taskservs_list) - if ($list_items | length) == 0 { - _print $"πŸ›‘ no items found for (_ansi cyan)taskservs list(_ansi reset)" + if ($list_items | length) == 0 { + _print $"πŸ›‘ no items found for (_ansi cyan)taskservs list(_ansi reset)" return - } + } if $cmd == "-" { return $list_items } if ($cmd | is-empty) { _print ($list_items | to json) "json" "result" "table" return [] - } else { + } else { if (get-provisioning-out | is-not-empty) or (get-provisioning-no-terminal) { return ""} - let selection_pos = ($list_items | each {|it| + let selection_pos = ($list_items | each {|it| match ($it.task | str length) { 2..4 => $"($it.task)\t\t ($it.mode)\t\t($it.info)\t($it.vers)", 5 => $"($it.task)\t\t ($it.mode)\t\t($it.info)\t($it.vers)", 12 => $"($it.task)\t ($it.mode)\t\t($it.info)\t($it.vers)", 15..20 => $"($it.task) ($it.mode)\t\t($it.info)\t($it.vers)", _ => $"($it.task)\t ($it.mode)\t\t($it.info)\t($it.vers)", - } + } } | input list --index ( - $"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset)" + + $"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset)" + $" \(use arrow keys and press [enter] or [escape] to exit\)( _ansi reset)" ) ) @@ -352,66 +352,66 @@ export def on_list [ let item_selec = if ($list_items | length) > $selection_pos { $list_items | get $selection_pos } else { null } let item_path = $"((get-taskservs-path))/($item_selec.task)/($item_selec.mode)" if not ($item_path | path exists) { _print $"Path ($item_path) not found" } - run_on_selection $cmd $item_selec.task $item_path ($item_path | path join $"install-($item_selec.task).sh") (get-taskservs-path) + run_on_selection $cmd $item_selec.task $item_path ($item_path | path join $"install-($item_selec.task).sh") (get-taskservs-path) } } return [] }, - "clusters" | "c" => { + "clusters" | "c" => { _print $"\n(_ansi purple)Cluster(_ansi reset) list: \n" let list_items = (cluster_list) - if ($list_items | length) == 0 { - _print $"πŸ›‘ no items found for (_ansi cyan)cluster list(_ansi reset)" + if ($list_items | length) == 0 { + _print $"πŸ›‘ no items found for (_ansi cyan)cluster list(_ansi reset)" return [] - } + } if $cmd == "-" { return $list_items } if ($cmd | is-empty) { _print ($list_items | to json) "json" "result" "table" - } else { + } else { if (get-provisioning-out | is-not-empty) or (get-provisioning-no-terminal) { return ""} - let selection = (cluster_list | input list) - #print ($"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset) " + + let selection = (cluster_list | input list) + #print ($"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset) " + # $" \(use arrow keys and press [enter] or [escape] to exit\)( _ansi reset)" ) _print $"($cmd) ($selection)" } return [] }, - "infras" | "i" => { + "infras" | "i" => { _print $"\n(_ansi cyan)Infrastructures(_ansi reset) list: \n" let list_items = (infras_list) - if ($list_items | length) == 0 { - _print $"πŸ›‘ no items found for (_ansi cyan)infras list(_ansi reset)" + if ($list_items | length) == 0 { + _print $"πŸ›‘ no items found for (_ansi cyan)infras list(_ansi reset)" return [] - } + } if $cmd == "-" { return $list_items } if ($cmd | is-empty) { _print ($list_items | to json) "json" "result" "table" - } else { + } else { if (get-provisioning-out | is-not-empty) or (get-provisioning-no-terminal) { return ""} - let selection_pos = ($list_items | each {|it| + let selection_pos = ($list_items | each {|it| match ($it.name | str length) { 2..5 => $"($it.name)\t\t ($it.modified) -- ($it.size)", 12 => $"($it.name)\t ($it.modified) -- ($it.size)", 15..20 => $"($it.name) ($it.modified) -- ($it.size)", _ => $"($it.name)\t ($it.modified) -- ($it.size)", - } + } } | input list --index ( - $"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset)" + + $"(_ansi default_dimmed)Select one item for (_ansi cyan_bold)($cmd)(_ansi reset)" + $" \(use arrow keys and [enter] or [escape] to exit\)( _ansi reset)" - ) + ) ) if $selection_pos != null { let item_selec = if ($list_items | length) > $selection_pos { $list_items | get $selection_pos } else { null } let item_path = $"((get-workspace-path))/($item_selec.name)" if not ($item_path | path exists) { _print $"Path ($item_path) not found" } - run_on_selection $cmd $item_selec.name $item_path ($item_path | path join (get-default-settings)) (get-provisioning-infra-path) + run_on_selection $cmd $item_selec.name $item_path ($item_path | path join (get-default-settings)) (get-provisioning-infra-path) } } return [] }, - "help" | "h" | _ => { + "help" | "h" | _ => { if $target_list != "help" or $target_list != "h" { - _print $"πŸ›‘ Not found ((get-provisioning-name)) target list option (_ansi red)($target_list)(_ansi reset)" + _print $"πŸ›‘ Not found ((get-provisioning-name)) target list option (_ansi red)($target_list)(_ansi reset)" } _print ( $"Use (_ansi blue_bold)((get-provisioning-name))(_ansi reset) (_ansi green)list(_ansi reset)" + @@ -422,10 +422,10 @@ export def on_list [ $"(_ansi yellow_bold)c(_ansi reset)ode | (_ansi yellow_bold)s(_ansi reset)hell | (_ansi yellow_bold)n(_ansi reset)u" ) return [] - }, - _ => { + }, + _ => { _print $"πŸ›‘ invalid_option $list ($ops)" return [] - } - } -} \ No newline at end of file + } + } +} diff --git a/nulib/lib_provisioning/deploy.nu b/nulib/lib_provisioning/deploy.nu index deed518..8d86e34 100644 --- a/nulib/lib_provisioning/deploy.nu +++ b/nulib/lib_provisioning/deploy.nu @@ -162,4 +162,4 @@ export def deploy_list [ let provider = $server.provider | default "" ^ls ($out_path | path dirname | path join $"($provider)_cmd.*") err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" }) } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/diagnostics/health_check.nu b/nulib/lib_provisioning/diagnostics/health_check.nu index 0e9e2b8..12c14a3 100644 --- a/nulib/lib_provisioning/diagnostics/health_check.nu +++ b/nulib/lib_provisioning/diagnostics/health_check.nu @@ -191,48 +191,48 @@ def check-platform-connectivity []: nothing -> record { } } -# Check KCL schemas validity -def check-kcl-schemas []: nothing -> record { +# Check Nickel schemas validity +def check-nickel-schemas []: nothing -> record { mut issues = [] mut warnings = [] - let kcl_path = config-get "paths.kcl" "provisioning/kcl" + let nickel_path = config-get "paths.nickel" "provisioning/nickel" - if not ($kcl_path | path exists) { - $issues = ($issues | append "KCL directory not found") + if not ($nickel_path | path exists) { + $issues = ($issues | append "Nickel directory not found") } else { # Check for main schema files let required_schemas = [ - "main.k" - "settings.k" - "lib.k" - "dependencies.k" + "main.ncl" + "settings.ncl" + "lib.ncl" + "dependencies.ncl" ] for schema in $required_schemas { - let schema_path = ($kcl_path | path join $schema) + let schema_path = ($nickel_path | path join $schema) if not ($schema_path | path exists) { $warnings = ($warnings | append $"Schema file not found: ($schema)") } } - # Try to compile a simple KCL file - let kcl_bin = (which kcl | get path.0? | default "") - if ($kcl_bin | is-not-empty) { + # Try to compile a simple Nickel file + let nickel_bin = (which nickel | get path.0? | default "") + if ($nickel_bin | is-not-empty) { do -i { - ^kcl fmt --check $kcl_path e> /dev/null o> /dev/null + ^nickel fmt --check $nickel_path e> /dev/null o> /dev/null } if ($env.LAST_EXIT_CODE? | default 1) != 0 { - $warnings = ($warnings | append "KCL format check reported issues") + $warnings = ($warnings | append "Nickel format check reported issues") } } else { - $warnings = ($warnings | append "KCL CLI not available - cannot validate schemas") + $warnings = ($warnings | append "Nickel CLI not available - cannot validate schemas") } } { - check: "KCL Schemas" + check: "Nickel Schemas" status: (if ($issues | is-empty) { if ($warnings | is-empty) { "βœ… Healthy" } else { "⚠️ Warnings" } } else { @@ -240,7 +240,7 @@ def check-kcl-schemas []: nothing -> record { }) issues: ($issues | append $warnings) recommendation: (if ($issues | is-not-empty) or ($warnings | is-not-empty) { - "Review KCL schemas - See: .claude/kcl_idiomatic_patterns.md" + "Review Nickel schemas - See: .claude/guidelines/nickel/" } else { "No action needed" }) @@ -343,7 +343,7 @@ export def "provisioning health" []: nothing -> table { $health_checks = ($health_checks | append (check-workspace-structure)) $health_checks = ($health_checks | append (check-infrastructure-state)) $health_checks = ($health_checks | append (check-platform-connectivity)) - $health_checks = ($health_checks | append (check-kcl-schemas)) + $health_checks = ($health_checks | append (check-nickel-schemas)) $health_checks = ($health_checks | append (check-security-config)) $health_checks = ($health_checks | append (check-provider-credentials)) @@ -378,7 +378,7 @@ export def "provisioning health-json" []: nothing -> record { (check-workspace-structure) (check-infrastructure-state) (check-platform-connectivity) - (check-kcl-schemas) + (check-nickel-schemas) (check-security-config) (check-provider-credentials) ] diff --git a/nulib/lib_provisioning/diagnostics/next_steps.nu b/nulib/lib_provisioning/diagnostics/next_steps.nu index 3f2cf37..84232b9 100644 --- a/nulib/lib_provisioning/diagnostics/next_steps.nu +++ b/nulib/lib_provisioning/diagnostics/next_steps.nu @@ -159,7 +159,7 @@ def next-steps-no-taskservs []: nothing -> string { $"(ansi blue_bold)πŸ“š Documentation:(ansi reset)" $" β€’ Service Management: docs/user/SERVICE_MANAGEMENT_GUIDE.md" $" β€’ Taskserv Guide: docs/development/workflow.md" - $" β€’ Dependencies: Check taskserv dependencies.k files" + $" β€’ Dependencies: Check taskserv dependencies.ncl files" ] | str join "\n" } @@ -179,7 +179,7 @@ def next-steps-no-clusters []: nothing -> string { $" Command: (ansi green)provisioning cluster list(ansi reset)\n" $"(ansi yellow_bold)Alternative: Use batch workflows(ansi reset)" $" Deploy everything at once with dependencies:" - $" Command: (ansi green)provisioning batch submit workflows/example.k(ansi reset)\n" + $" Command: (ansi green)provisioning batch submit workflows/example.ncl(ansi reset)\n" $"(ansi blue_bold)πŸ“š Documentation:(ansi reset)" $" β€’ Cluster Management: docs/development/workflow.md" $" β€’ Batch Workflows: .claude/features/batch-workflow-system.md" @@ -202,7 +202,7 @@ def next-steps-deployed []: nothing -> string { $" β€’ Workflow status: (ansi green)provisioning workflow list(ansi reset)\n" $"(ansi yellow_bold)Advanced Operations:(ansi reset)" $" β€’ Test environments: (ansi green)provisioning test quick (ansi reset)" - $" β€’ Batch workflows: (ansi green)provisioning batch submit (ansi reset)" + $" β€’ Batch workflows: (ansi green)provisioning batch submit (ansi reset)" $" β€’ Update infrastructure: (ansi green)provisioning guide update(ansi reset)\n" $"(ansi yellow_bold)Platform Services:(ansi reset)" $" β€’ Start orchestrator: (ansi green)cd provisioning/platform/orchestrator && ./scripts/start-orchestrator.nu(ansi reset)" diff --git a/nulib/lib_provisioning/diagnostics/system_status.nu b/nulib/lib_provisioning/diagnostics/system_status.nu index 046f701..6abea95 100644 --- a/nulib/lib_provisioning/diagnostics/system_status.nu +++ b/nulib/lib_provisioning/diagnostics/system_status.nu @@ -27,13 +27,13 @@ def check-nushell-version []: nothing -> record { } } -# Check if KCL is installed -def check-kcl-installed []: nothing -> record { - let kcl_bin = (which kcl | get path.0? | default "") - let installed = ($kcl_bin | is-not-empty) +# Check if Nickel is installed +def check-nickel-installed []: nothing -> record { + let nickel_bin = (which nickel | get path.0? | default "") + let installed = ($nickel_bin | is-not-empty) let version_info = if $installed { - let result = (do { ^kcl --version } | complete) + let result = (do { ^nickel --version } | complete) if $result.exit_code == 0 { $result.stdout | str trim } else { @@ -44,7 +44,7 @@ def check-kcl-installed []: nothing -> record { } { - component: "KCL CLI" + component: "Nickel CLI" status: (if $installed { "βœ…" } else { "❌" }) version: $version_info required: "0.11.2+" @@ -53,7 +53,7 @@ def check-kcl-installed []: nothing -> record { } else { "Not found in PATH" }) - docs: "https://kcl-lang.io/docs/user_docs/getting-started/install" + docs: "https://nickel-lang.io/docs/user_docs/getting-started/install" } } @@ -61,8 +61,8 @@ def check-kcl-installed []: nothing -> record { def check-plugins []: nothing -> list { let required_plugins = [ { - name: "nu_plugin_kcl" - description: "KCL integration" + name: "nu_plugin_nickel" + description: "Nickel integration" optional: true docs: "docs/user/PLUGIN_INTEGRATION_GUIDE.md" } @@ -256,7 +256,7 @@ def get-all-checks []: nothing -> list { # Core requirements $checks = ($checks | append (check-nushell-version)) - $checks = ($checks | append (check-kcl-installed)) + $checks = ($checks | append (check-nickel-installed)) # Plugins $checks = ($checks | append (check-plugins)) diff --git a/nulib/lib_provisioning/extensions/QUICKSTART.md b/nulib/lib_provisioning/extensions/QUICKSTART.md index d923a47..cb29ba9 100644 --- a/nulib/lib_provisioning/extensions/QUICKSTART.md +++ b/nulib/lib_provisioning/extensions/QUICKSTART.md @@ -5,12 +5,14 @@ Get started with the Extension Loading System in 5 minutes. ## Prerequisites 1. **OCI Registry** (optional, for OCI features): + ```bash # Start local registry docker run -d -p 5000:5000 --name registry registry:2 ``` 2. **Nushell 0.107+**: + ```bash nu --version ``` @@ -28,7 +30,7 @@ provisioning ext load kubernetes --version 1.28.0 # Load from specific source provisioning ext load redis --source oci -``` +```plaintext ### 2. Search for Extensions @@ -38,7 +40,7 @@ provisioning ext search kube # Search OCI registry provisioning ext search postgres --source oci -``` +```plaintext ### 3. List Available Extensions @@ -51,7 +53,7 @@ provisioning ext list --type taskserv # JSON format provisioning ext list --format json -``` +```plaintext ### 4. Manage Cache @@ -64,13 +66,13 @@ provisioning ext cache list # Clear cache provisioning ext cache clear --all -``` +```plaintext ### 5. Publish an Extension ```bash # Create extension -mkdir -p my-extension/{kcl,scripts} +mkdir -p my-extension/{nickel,scripts} # Create manifest cat > my-extension/extension.yaml < # Check specific source provisioning ext list --source oci -``` +```plaintext ### OCI Registry Issues @@ -205,7 +207,7 @@ curl http://localhost:5000/v2/ # View OCI config provisioning env | grep OCI -``` +```plaintext ### Cache Problems @@ -215,7 +217,7 @@ provisioning ext cache clear --all # Pull fresh copy provisioning ext pull --force -``` +```plaintext ## Next Steps @@ -234,4 +236,4 @@ provisioning ext cache --help # Publish help nu provisioning/tools/publish_extension.nu --help -``` +```plaintext diff --git a/nulib/lib_provisioning/extensions/README.md b/nulib/lib_provisioning/extensions/README.md index 214763e..66daac5 100644 --- a/nulib/lib_provisioning/extensions/README.md +++ b/nulib/lib_provisioning/extensions/README.md @@ -6,11 +6,12 @@ ## Overview -A comprehensive extension loading mechanism with OCI registry support, lazy loading, caching, and version resolution. Supports loading extensions from multiple sources: OCI registries, Gitea repositories, and local filesystems. +A comprehensive extension loading mechanism with OCI registry support, lazy loading, caching, and version resolution. +Supports loading extensions from multiple sources: OCI registries, Gitea repositories, and local filesystems. ## Architecture -``` +```plaintext Extension Loading System β”œβ”€β”€ OCI Client (oci/client.nu) β”‚ β”œβ”€β”€ Artifact pull/push operations @@ -36,13 +37,14 @@ Extension Loading System β”œβ”€β”€ Load, search, list β”œβ”€β”€ Cache management └── Publishing -``` +```plaintext ## Features ### 1. Multi-Source Support Load extensions from: + - **OCI Registry**: Container artifact registry (localhost:5000 by default) - **Gitea**: Git repository hosting (planned) - **Local**: Filesystem paths @@ -50,6 +52,7 @@ Load extensions from: ### 2. Lazy Loading Extensions are loaded on-demand: + 1. Check if already in memory β†’ return 2. Check cache β†’ load from cache 3. Determine source (auto-detect or explicit) @@ -60,6 +63,7 @@ Extensions are loaded on-demand: ### 3. OCI Registry Integration Full OCI artifact support: + - Pull artifacts with authentication - Push extensions to registry - List and search artifacts @@ -69,6 +73,7 @@ Full OCI artifact support: ### 4. Caching System Intelligent local caching: + - Cache directory: `~/.provisioning/cache/extensions/{type}/{name}/{version}/` - Cache index: JSON-based index for fast lookups - Automatic pruning: Remove old cached versions @@ -77,6 +82,7 @@ Intelligent local caching: ### 5. Version Resolution Semver-compliant version resolution: + - **Exact**: `1.2.3` β†’ exactly version 1.2.3 - **Caret**: `^1.2.0` β†’ >=1.2.0 <2.0.0 (compatible) - **Tilde**: `~1.2.0` β†’ >=1.2.0 <1.3.0 (approximately) @@ -86,6 +92,7 @@ Semver-compliant version resolution: ### 6. Discovery & Search Multi-source extension discovery: + - Discover all extensions across sources - Search by name or type - Filter by extension type (provider, taskserv, cluster) @@ -108,7 +115,7 @@ retry_count = 3 [extensions] source_type = "auto" # auto, oci, gitea, local -``` +```plaintext ### Environment Variables @@ -132,7 +139,7 @@ provisioning ext load kubernetes --force # Load provider provisioning ext load aws --type provider -``` +```plaintext ### Search Extensions @@ -145,7 +152,7 @@ provisioning ext search kubernetes --source oci # Search local only provisioning ext search kube --source local -``` +```plaintext ### List Extensions @@ -161,7 +168,7 @@ provisioning ext list --format json # List from specific source provisioning ext list --source oci -``` +```plaintext ### Extension Information @@ -174,7 +181,7 @@ provisioning ext info kubernetes --version 1.28.0 # Show versions provisioning ext versions kubernetes -``` +```plaintext ### Cache Management @@ -193,7 +200,7 @@ provisioning ext cache clear --all # Prune old entries (older than 30 days) provisioning ext cache prune --days 30 -``` +```plaintext ### Pull to Cache @@ -203,7 +210,7 @@ provisioning ext pull kubernetes --version 1.28.0 # Pull from specific source provisioning ext pull redis --source oci -``` +```plaintext ### Publishing @@ -219,7 +226,7 @@ provisioning ext publish ./my-extension \ # Force overwrite existing provisioning ext publish ./my-extension --version 1.0.0 --force -``` +```plaintext ### Discovery @@ -232,14 +239,14 @@ provisioning ext discover --type taskserv # Force refresh provisioning ext discover --refresh -``` +```plaintext ### Test OCI Connection ```bash # Test OCI registry connectivity provisioning ext test-oci -``` +```plaintext ## Publishing Tool Usage @@ -260,25 +267,25 @@ nu provisioning/tools/publish_extension.nu info kubernetes 1.28.0 # Delete extension nu provisioning/tools/publish_extension.nu delete kubernetes 1.28.0 --force -``` +```plaintext ## Extension Structure ### Required Files -``` +```plaintext my-extension/ β”œβ”€β”€ extension.yaml # Manifest (required) -β”œβ”€β”€ kcl/ # KCL schemas (optional) -β”‚ β”œβ”€β”€ my-extension.k -β”‚ └── kcl.mod +β”œβ”€β”€ nickel/ # Nickel schemas (optional) +β”‚ β”œβ”€β”€ my-extension.ncl +β”‚ └── nickel.mod β”œβ”€β”€ scripts/ # Scripts (optional) β”‚ └── install.nu β”œβ”€β”€ templates/ # Templates (optional) β”‚ └── config.yaml.j2 └── docs/ # Documentation (optional) └── README.md -``` +```plaintext ### Extension Manifest (extension.yaml) @@ -302,7 +309,7 @@ extension: homepage: https://example.com repository: https://github.com/user/extension license: MIT -``` +```plaintext ## API Reference @@ -382,7 +389,7 @@ nu provisioning/core/nulib/lib_provisioning/extensions/tests/test_oci_client.nu nu provisioning/core/nulib/lib_provisioning/extensions/tests/test_cache.nu nu provisioning/core/nulib/lib_provisioning/extensions/tests/test_versions.nu nu provisioning/core/nulib/lib_provisioning/extensions/tests/test_discovery.nu -``` +```plaintext ## Integration Examples @@ -398,7 +405,7 @@ if $result.success { } else { print $"Failed: ($result.error)" } -``` +```plaintext ### Example 2: Discover and Cache All Extensions @@ -412,7 +419,7 @@ for ext in $extensions { print $"Caching ($ext.name):($ext.latest)..." load-extension $ext.type $ext.name $ext.latest } -``` +```plaintext ### Example 3: Version Resolution @@ -421,7 +428,7 @@ use lib_provisioning/extensions/versions.nu resolve-oci-version let version = (resolve-oci-version "taskserv" "kubernetes" "^1.28.0") print $"Resolved to: ($version)" -``` +```plaintext ## Troubleshooting @@ -436,7 +443,7 @@ provisioning env | grep OCI # Verify registry is running curl http://localhost:5000/v2/ -``` +```plaintext ### Extension Not Found @@ -450,7 +457,7 @@ provisioning ext list --source local # Discover with refresh provisioning ext discover --refresh -``` +```plaintext ### Cache Issues @@ -463,7 +470,7 @@ provisioning ext cache clear --all # Prune old entries provisioning ext cache prune --days 7 -``` +```plaintext ### Version Resolution Issues @@ -476,7 +483,7 @@ provisioning ext load --version 1.2.3 # Force reload provisioning ext load --force -``` +```plaintext ## Performance Considerations @@ -506,9 +513,10 @@ provisioning ext load --force ## Contributing See main project contributing guidelines. Extension system follows: + - Nushell idiomatic patterns - PAP (Project Architecture Principles) -- KCL idiomatic patterns for schemas +- Nickel idiomatic patterns for schemas ## License diff --git a/nulib/lib_provisioning/extensions/cache.nu b/nulib/lib_provisioning/extensions/cache.nu index 7ed481e..10169c8 100644 --- a/nulib/lib_provisioning/extensions/cache.nu +++ b/nulib/lib_provisioning/extensions/cache.nu @@ -448,4 +448,4 @@ export def get-temp-extraction-path [ ]: nothing -> string { let temp_base = (mktemp -d) $temp_base | path join $extension_type $extension_name $version -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/discovery.nu b/nulib/lib_provisioning/extensions/discovery.nu index 0539959..9c3fc0a 100644 --- a/nulib/lib_provisioning/extensions/discovery.nu +++ b/nulib/lib_provisioning/extensions/discovery.nu @@ -416,4 +416,4 @@ def extract-extension-type [manifest: record]: nothing -> string { def is-gitea-available []: nothing -> bool { # TODO: Implement Gitea availability check false -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/loader.nu b/nulib/lib_provisioning/extensions/loader.nu index c70f2f2..f4451f8 100644 --- a/nulib/lib_provisioning/extensions/loader.nu +++ b/nulib/lib_provisioning/extensions/loader.nu @@ -133,4 +133,4 @@ export def load-hooks [extension_path: string, manifest: record]: nothing -> rec } else { {} } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/loader_oci.nu b/nulib/lib_provisioning/extensions/loader_oci.nu index 85d9fa3..9cdb7e4 100644 --- a/nulib/lib_provisioning/extensions/loader_oci.nu +++ b/nulib/lib_provisioning/extensions/loader_oci.nu @@ -342,7 +342,7 @@ def load-from-path [ # Validate extension directory structure def validate-extension-structure [path: string]: nothing -> record { let required_files = ["extension.yaml"] - let required_dirs = [] # Optional: ["kcl", "scripts"] + let required_dirs = [] # Optional: ["nickel", "scripts"] mut errors = [] @@ -421,4 +421,4 @@ def compare-semver-versions [a: string, b: string]: nothing -> int { } 0 -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/mod.nu b/nulib/lib_provisioning/extensions/mod.nu index 733fc5f..4e8319e 100644 --- a/nulib/lib_provisioning/extensions/mod.nu +++ b/nulib/lib_provisioning/extensions/mod.nu @@ -8,4 +8,4 @@ export use loader_oci.nu * export use cache.nu * export use versions.nu * export use discovery.nu * -export use commands.nu * \ No newline at end of file +export use commands.nu * diff --git a/nulib/lib_provisioning/extensions/profiles.nu b/nulib/lib_provisioning/extensions/profiles.nu index ad23f68..ec5b653 100644 --- a/nulib/lib_provisioning/extensions/profiles.nu +++ b/nulib/lib_provisioning/extensions/profiles.nu @@ -221,4 +221,4 @@ export def create-example-profiles []: nothing -> nothing { $developer_profile | to yaml | save ($user_profiles_dir | path join "developer.yaml") print $"Created example profiles in ($user_profiles_dir)" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/registry.nu b/nulib/lib_provisioning/extensions/registry.nu index 00e8b3c..a455f96 100644 --- a/nulib/lib_provisioning/extensions/registry.nu +++ b/nulib/lib_provisioning/extensions/registry.nu @@ -237,4 +237,4 @@ export def get-taskserv-path [name: string]: nothing -> string { "" } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/extensions/versions.nu b/nulib/lib_provisioning/extensions/versions.nu index b2c3959..10bdcc7 100644 --- a/nulib/lib_provisioning/extensions/versions.nu +++ b/nulib/lib_provisioning/extensions/versions.nu @@ -336,4 +336,4 @@ def satisfies-range [version: string, constraint: string]: nothing -> bool { def is-gitea-available []: nothing -> bool { # TODO: Implement Gitea availability check false -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/fluent_daemon.nu b/nulib/lib_provisioning/fluent_daemon.nu new file mode 100644 index 0000000..40322ad --- /dev/null +++ b/nulib/lib_provisioning/fluent_daemon.nu @@ -0,0 +1,413 @@ +#! Fluent i18n translation daemon functions +#! +#! Provides high-performance message translation via HTTP API using Mozilla's Fluent. +#! The CLI daemon's Fluent engine offers 50-100x better performance than using +#! the nu_plugin_fluent plugin due to aggressive caching and no process spawning. +#! +#! Performance: +#! - Single translation: ~1-5ms uncached, ~0.1-0.5ms cached (vs ~50ms with plugin) +#! - Batch 10 translations: ~10-20ms with cache +#! - Cache hit ratio: 75-80% on typical workloads + +use ../env.nu [get-cli-daemon-url] + +# Translate a message ID to the target locale +# +# Uses the CLI daemon's Fluent engine for fast i18n translation. +# Supports variable interpolation and fallback locales. +# +# # Arguments +# * `message_id` - Message identifier (e.g., "welcome-message") +# * `--locale (-l)` - Target locale (default: "en-US") +# * `--args (-a)` - Arguments for variable interpolation (record) +# * `--fallback (-f)` - Fallback locale if message not found +# +# # Returns +# Translated message string or error if translation failed +# +# # Example +# ```nushell +# # Simple translation +# fluent-translate "welcome-message" --locale en-US +# +# # With arguments +# fluent-translate "greeting" --locale es --args {name: "MarΓ­a"} +# +# # With fallback +# fluent-translate "new-feature" --locale fr --fallback en-US +# ``` +export def fluent-translate [ + message_id: string + --locale (-l): string = "en-US" + --args (-a): record = {} + --fallback (-f): string +] -> string { + let daemon_url = (get-cli-daemon-url) + + # Build request + let request = { + message_id: $message_id + locale: $locale + args: ($args | to json | from json) + fallback_locale: $fallback + } + + # Send to daemon's Fluent endpoint + let response = ( + http post $"($daemon_url)/fluent/translate" $request + --raw + ) + + # Parse response + let parsed = ($response | from json) + + # Check for error + if ($parsed.error? != null) { + error make {msg: $"Fluent translation error: ($parsed.error)"} + } + + # Return translated message + $parsed.translated +} + +# Translate multiple messages in batch mode +# +# Translates a list of message IDs to the same locale. More efficient +# than calling fluent-translate multiple times due to connection reuse. +# +# # Arguments +# * `message_ids` - List of message IDs to translate +# * `--locale (-l)` - Target locale (default: "en-US") +# * `--fallback (-f)` - Fallback locale if messages not found +# +# # Returns +# List of translated messages +# +# # Example +# ```nushell +# let messages = ["welcome", "goodbye", "thank-you"] +# fluent-translate-batch $messages --locale fr --fallback en +# ``` +export def fluent-translate-batch [ + message_ids: list + --locale (-l): string = "en-US" + --fallback (-f): string +] -> list { + $message_ids | each { |msg_id| + fluent-translate $msg_id --locale $locale --fallback $fallback + } +} + +# Load a Fluent bundle from a specific FTL file +# +# Loads messages from an FTL file into the daemon's bundle cache. +# This is useful for loading custom translations at runtime. +# +# # Arguments +# * `locale` - Locale identifier (e.g., "es", "fr-FR") +# * `path` - Path to FTL file +# +# # Returns +# Record with load status and message count +# +# # Example +# ```nushell +# fluent-load-bundle "es" "/path/to/es.ftl" +# ``` +export def fluent-load-bundle [ + locale: string + path: string +] -> record { + let daemon_url = (get-cli-daemon-url) + + let request = { + locale: $locale + path: $path + } + + let response = ( + http post $"($daemon_url)/fluent/bundles/load" $request + ) + + $response | from json +} + +# Reload all Fluent bundles from the FTL directory +# +# Clears all cached bundles and reloads them from the configured +# FTL directory. Useful after updating translation files. +# +# # Returns +# Record with reload status and list of loaded locales +# +# # Example +# ```nushell +# fluent-reload-bundles +# ``` +export def fluent-reload-bundles [] -> record { + let daemon_url = (get-cli-daemon-url) + + let response = ( + http post $"($daemon_url)/fluent/bundles/reload" "" + ) + + $response | from json +} + +# List all available locales +# +# Returns a list of all currently loaded locale identifiers. +# +# # Returns +# List of locale strings +# +# # Example +# ```nushell +# fluent-list-locales +# # Output: [en-US, es, fr-FR, de] +# ``` +export def fluent-list-locales [] -> list { + let daemon_url = (get-cli-daemon-url) + + let response = (http get $"($daemon_url)/fluent/bundles/locales") + + ($response | from json).locales +} + +# Get translation statistics from daemon +# +# Returns statistics about translations since daemon startup or last reset. +# +# # Returns +# Record with: +# - `total_translations`: Total number of translations +# - `successful_translations`: Number of successful translations +# - `failed_translations`: Number of failed translations +# - `cache_hits`: Number of cache hits +# - `cache_misses`: Number of cache misses +# - `cache_hit_ratio`: Cache hit ratio (0.0 - 1.0) +# - `bundles_loaded`: Number of bundles loaded +# - `total_time_ms`: Total time spent translating (milliseconds) +# - `average_time_ms`: Average time per translation +# +# # Example +# ```nushell +# fluent-stats +# ``` +export def fluent-stats [] -> record { + let daemon_url = (get-cli-daemon-url) + + let response = (http get $"($daemon_url)/fluent/stats") + + $response | from json +} + +# Reset translation statistics on daemon +# +# Clears all counters and timing statistics. +# +# # Example +# ```nushell +# fluent-reset-stats +# ``` +export def fluent-reset-stats [] -> void { + let daemon_url = (get-cli-daemon-url) + + http post $"($daemon_url)/fluent/stats/reset" "" +} + +# Clear all Fluent caches +# +# Clears both the translation cache and bundle cache. +# All subsequent translations will reload bundles and re-translate messages. +# +# # Example +# ```nushell +# fluent-clear-caches +# ``` +export def fluent-clear-caches [] -> void { + let daemon_url = (get-cli-daemon-url) + + http delete $"($daemon_url)/fluent/cache/clear" +} + +# Check if CLI daemon is running with Fluent support +# +# # Returns +# `true` if daemon is running with Fluent support, `false` otherwise +# +# # Example +# ```nushell +# if (is-fluent-daemon-available) { +# fluent-translate "welcome" +# } else { +# print "Fallback: Welcome!" +# } +# ``` +export def is-fluent-daemon-available [] -> bool { + try { + let daemon_url = (get-cli-daemon-url) + let response = (http get $"($daemon_url)/fluent/health" --timeout 500ms) + + ($response | from json | .status == "healthy") + } catch { + false + } +} + +# Ensure Fluent daemon is available +# +# Checks if the daemon is running and prints a status message. +# Useful for diagnostics and setup scripts. +# +# # Example +# ```nushell +# ensure-fluent-daemon +# ``` +export def ensure-fluent-daemon [] -> void { + if (is-fluent-daemon-available) { + print "βœ… Fluent i18n daemon is available and running" + } else { + print "⚠️ Fluent i18n daemon is not available" + print " CLI daemon may not be running at http://localhost:9091" + print " Translations will not work until daemon is started" + } +} + +# Profile translation performance +# +# Translates a message multiple times and reports timing statistics. +# Useful for benchmarking and performance optimization. +# +# # Arguments +# * `message_id` - Message ID to translate +# * `--locale (-l)` - Target locale (default: "en-US") +# * `--iterations (-i)` - Number of times to translate (default: 100) +# * `--args (-a)` - Arguments for variable interpolation (record) +# +# # Returns +# Record with performance metrics +# +# # Example +# ```nushell +# fluent-profile "greeting" --locale es --iterations 1000 --args {name: "Usuario"} +# ``` +export def fluent-profile [ + message_id: string + --locale (-l): string = "en-US" + --iterations (-i): int = 100 + --args (-a): record = {} +] -> record { + let start = (date now) + + # Reset stats before profiling + fluent-reset-stats + + # Run translations + for i in 0..<$iterations { + fluent-translate $message_id --locale $locale --args $args + } + + let elapsed_ms = ((date now) - $start) | into duration | .0 / 1_000_000 + let stats = (fluent-stats) + + { + message_id: $message_id + locale: $locale + iterations: $iterations + total_time_ms: $elapsed_ms + avg_time_ms: ($elapsed_ms / $iterations) + daemon_total_translations: $stats.total_translations + daemon_cache_hits: $stats.cache_hits + daemon_cache_hit_ratio: $stats.cache_hit_ratio + daemon_avg_time_ms: $stats.average_time_ms + daemon_successful: $stats.successful_translations + daemon_failed: $stats.failed_translations + } +} + +# Show cache efficiency report +# +# Displays a formatted report of cache performance. +# +# # Example +# ```nushell +# fluent-cache-report +# ``` +export def fluent-cache-report [] -> void { + let stats = (fluent-stats) + + print $"=== Fluent i18n Cache Report ===" + print $"" + print $"Total translations: ($stats.total_translations)" + print $"Cache hits: ($stats.cache_hits)" + print $"Cache misses: ($stats.cache_misses)" + print $"Hit ratio: (($stats.cache_hit_ratio * 100) | math round --precision 1)%" + print $"" + print $"Average latency: ($stats.average_time_ms | math round --precision 2)ms" + print $"Total time: ($stats.total_time_ms)ms" + print $"" + print $"Bundles loaded: ($stats.bundles_loaded)" + print $"Success rate: (($stats.successful_translations / $stats.total_translations * 100) | math round --precision 1)%" +} + +# Translate and fallback to default if not found +# +# Attempts to translate a message, falling back to a default value if not found. +# +# # Arguments +# * `message_id` - Message ID to translate +# * `default` - Default value if translation fails +# * `--locale (-l)` - Target locale (default: "en-US") +# * `--args (-a)` - Arguments for variable interpolation (record) +# +# # Returns +# Translated message or default value +# +# # Example +# ```nushell +# fluent-translate-or "new-feature" "New Feature" --locale fr +# ``` +export def fluent-translate-or [ + message_id: string + default: string + --locale (-l): string = "en-US" + --args (-a): record = {} +] -> string { + try { + fluent-translate $message_id --locale $locale --args $args + } catch { + $default + } +} + +# Create a localized string table from message IDs +# +# Translates a list of message IDs and returns a record mapping IDs to translations. +# +# # Arguments +# * `message_ids` - List of message IDs +# * `--locale (-l)` - Target locale (default: "en-US") +# +# # Returns +# Record mapping message IDs to translated strings +# +# # Example +# ```nushell +# let ids = ["welcome", "goodbye", "help"] +# let strings = (fluent-string-table $ids --locale es) +# $strings.welcome # Accesses translated "welcome" message +# ``` +export def fluent-string-table [ + message_ids: list + --locale (-l): string = "en-US" +] -> record { + let table = {} + + for msg_id in $message_ids { + let translation = (fluent-translate $msg_id --locale $locale) + $table | insert $msg_id $translation + } + + $table +} diff --git a/nulib/lib_provisioning/gitea/IMPLEMENTATION_SUMMARY.md b/nulib/lib_provisioning/gitea/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index d56d067..0000000 --- a/nulib/lib_provisioning/gitea/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,667 +0,0 @@ -# Gitea Integration Implementation Summary - -**Version:** 1.0.0 -**Date:** 2025-10-06 -**Status:** Complete - ---- - -## Overview - -Comprehensive Gitea integration for workspace management, extension distribution, and collaboration features has been successfully implemented. - ---- - -## Deliverables - -### 1. KCL Configuration Schema βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/kcl/gitea.k` - -**Schemas Implemented:** -- `GiteaConfig` - Main configuration with local/remote modes -- `LocalGitea` - Local deployment configuration -- `DockerGitea` - Docker-specific settings -- `BinaryGitea` - Binary deployment settings -- `RemoteGitea` - Remote instance configuration -- `GiteaAuth` - Authentication configuration -- `GiteaRepositories` - Repository organization -- `WorkspaceFeatures` - Feature flags -- `GiteaRepository` - Repository metadata -- `GiteaRelease` - Release configuration -- `GiteaIssue` - Issue configuration (for locking) -- `WorkspaceLock` - Lock metadata -- `ExtensionPublishConfig` - Publishing configuration -- `GiteaWebhook` - Webhook configuration - -**Features:** -- Support for both local (Docker/binary) and remote Gitea -- Comprehensive validation with check blocks -- Sensible defaults for all configurations -- Example configurations included - ---- - -### 2. Gitea API Client βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/api_client.nu` - -**Functions Implemented (42 total):** - -**Core API:** -- `get-gitea-config` - Load Gitea configuration -- `get-gitea-token` - Retrieve auth token (supports SOPS encryption) -- `get-api-url` - Get base API URL -- `gitea-api-call` - Generic API call wrapper - -**Repository Operations:** -- `create-repository` - Create new repository -- `get-repository` - Get repository details -- `delete-repository` - Delete repository -- `list-repositories` - List organization repositories -- `list-user-repositories` - List user repositories - -**Release Operations:** -- `create-release` - Create new release -- `upload-release-asset` - Upload file to release -- `get-release-by-tag` - Get release by tag name -- `list-releases` - List all releases -- `delete-release` - Delete release - -**Issue Operations (for locking):** -- `create-issue` - Create new issue -- `close-issue` - Close issue -- `list-issues` - List issues with filters -- `get-issue` - Get issue details - -**Organization Operations:** -- `create-organization` - Create organization -- `get-organization` - Get organization details -- `list-organizations` - List user organizations - -**User/Auth Operations:** -- `get-current-user` - Get authenticated user -- `validate-token` - Validate auth token - -**Branch Operations:** -- `create-branch` - Create branch -- `list-branches` - List branches -- `get-branch` - Get branch details - -**Tag Operations:** -- `create-tag` - Create tag -- `list-tags` - List tags - -**Features:** -- Full REST API v1 support -- Token-based authentication -- SOPS encrypted token support -- Error handling and validation -- HTTP methods: GET, POST, PUT, DELETE, PATCH - ---- - -### 3. Workspace Git Operations βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/workspace_git.nu` - -**Functions Implemented (20 total):** - -**Initialization:** -- `init-workspace-git` - Initialize workspace as git repo with remote -- `create-workspace-repo` - Create repository on Gitea - -**Cloning:** -- `clone-workspace` - Clone workspace from Gitea - -**Push/Pull:** -- `push-workspace` - Push workspace changes -- `pull-workspace` - Pull workspace updates -- `sync-workspace` - Pull + push in one operation - -**Branch Management:** -- `create-workspace-branch` - Create new branch -- `switch-workspace-branch` - Switch to branch -- `list-workspace-branches` - List branches (local/remote) -- `delete-workspace-branch` - Delete branch - -**Status/Info:** -- `get-workspace-git-status` - Get comprehensive git status -- `get-workspace-remote-info` - Get remote repository info -- `has-uncommitted-changes` - Check for uncommitted changes -- `get-workspace-diff` - Get diff (staged/unstaged) - -**Stash Operations:** -- `stash-workspace-changes` - Stash changes -- `pop-workspace-stash` - Pop stashed changes -- `list-workspace-stashes` - List stashes - -**Features:** -- Automatic git configuration -- Remote URL management -- Gitea integration -- Branch protection -- Stash support - ---- - -### 4. Workspace Locking βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/locking.nu` - -**Functions Implemented (12 total):** - -**Lock Management:** -- `acquire-workspace-lock` - Acquire lock (creates issue) -- `release-workspace-lock` - Release lock (closes issue) -- `is-workspace-locked` - Check lock status -- `list-workspace-locks` - List locks for workspace -- `list-all-locks` - List all active locks -- `get-lock-info` - Get detailed lock information -- `force-release-lock` - Force release lock (admin) -- `cleanup-expired-locks` - Cleanup expired locks -- `with-workspace-lock` - Auto-lock wrapper for operations - -**Internal Functions:** -- `ensure-lock-repo` - Ensure locks repository exists -- `check-lock-conflicts` - Check for conflicting locks -- `format-lock-title/body` - Format lock issue content - -**Lock Types:** -- **read**: Multiple readers, blocks writers -- **write**: Exclusive access -- **deploy**: Exclusive deployment access - -**Features:** -- Distributed locking via Gitea issues -- Conflict detection (write blocks all, read blocks write) -- Lock expiry support -- Lock metadata tracking -- Force unlock capability -- Automatic cleanup - -**Lock Issue Format:** -``` -Title: [LOCK:write] workspace-name by username -Body: - - Lock Type: write - - Workspace: workspace-name - - User: username - - Timestamp: 2025-10-06T12:00:00Z - - Operation: server deployment - - Expiry: 2025-10-06T13:00:00Z -Labels: workspace-lock, write-lock -``` - ---- - -### 5. Extension Publishing βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/extension_publish.nu` - -**Functions Implemented (10 total):** - -**Publishing:** -- `publish-extension-to-gitea` - Full publishing workflow -- `publish-extensions-batch` - Batch publish multiple extensions - -**Discovery:** -- `list-gitea-extensions` - List published extensions -- `get-gitea-extension-metadata` - Get extension metadata -- `get-latest-extension-version` - Get latest version - -**Download:** -- `download-gitea-extension` - Download and extract extension - -**Internal Functions:** -- `validate-extension` - Validate extension structure -- `package-extension` - Package as tar.gz -- `generate-release-notes` - Extract from CHANGELOG - -**Publishing Workflow:** -1. Validate extension structure (kcl/kcl.mod, *.k files) -2. Determine extension type (provider/taskserv/cluster) -3. Package as `.tar.gz` -4. Generate release notes from CHANGELOG.md -5. Create git tag (if applicable) -6. Create Gitea release -7. Upload package as asset -8. Generate metadata file - -**Features:** -- Automatic extension type detection -- CHANGELOG integration -- Git tag creation -- Versioned releases -- Batch publishing support -- Download with auto-extraction - ---- - -### 6. Service Management βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/service.nu` - -**Functions Implemented (11 total):** - -**Start/Stop:** -- `start-gitea-docker` - Start Docker container -- `stop-gitea-docker` - Stop Docker container -- `start-gitea-binary` - Start binary deployment -- `start-gitea` - Auto-detect and start -- `stop-gitea` - Auto-detect and stop -- `restart-gitea` - Restart service - -**Status:** -- `get-gitea-status` - Get service status -- `check-gitea-health` - Health check -- `is-gitea-docker-running` - Check Docker status - -**Utilities:** -- `install-gitea` - Install Gitea binary -- `get-gitea-logs` - View logs (Docker) - -**Features:** -- Docker and binary deployment support -- Auto-start capability -- Health monitoring -- Log streaming -- Cross-platform binary installation - ---- - -### 7. CLI Commands βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/commands.nu` - -**Commands Implemented (30+ total):** - -**Service Commands:** -- `gitea status` - Show service status -- `gitea start` - Start service -- `gitea stop` - Stop service -- `gitea restart` - Restart service -- `gitea logs` - View logs -- `gitea install` - Install binary - -**Repository Commands:** -- `gitea repo create` - Create repository -- `gitea repo list` - List repositories -- `gitea repo delete` - Delete repository - -**Extension Commands:** -- `gitea extension publish` - Publish extension -- `gitea extension list` - List extensions -- `gitea extension download` - Download extension -- `gitea extension info` - Show extension info - -**Lock Commands:** -- `gitea lock acquire` - Acquire lock -- `gitea lock release` - Release lock -- `gitea lock list` - List locks -- `gitea lock info` - Show lock details -- `gitea lock force-release` - Force release -- `gitea lock cleanup` - Cleanup expired locks - -**Auth Commands:** -- `gitea auth validate` - Validate token -- `gitea user` - Show current user - -**Organization Commands:** -- `gitea org create` - Create organization -- `gitea org list` - List organizations - -**Help:** -- `gitea help` - Show all commands - -**Features:** -- User-friendly CLI interface -- Consistent flag patterns -- Color-coded output -- Interactive prompts -- Comprehensive help - ---- - -### 8. Docker Deployment βœ… - -**Files:** -- `/Users/Akasha/project-provisioning/provisioning/config/gitea/docker-compose.yml` -- `/Users/Akasha/project-provisioning/provisioning/config/gitea/app.ini.template` - -**Docker Compose Features:** -- Gitea 1.21 image -- SQLite database (lightweight) -- Port mappings (3000, 222) -- Data volume persistence -- Network isolation -- Auto-restart policy - -**Binary Configuration Template:** -- Complete app.ini template -- Tera template support -- Production-ready defaults -- Customizable settings - ---- - -### 9. Module Organization βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/lib_provisioning/gitea/mod.nu` - -**Structure:** -``` -gitea/ -β”œβ”€β”€ mod.nu # Main module (exports) -β”œβ”€β”€ api_client.nu # API client (42 functions) -β”œβ”€β”€ workspace_git.nu # Git operations (20 functions) -β”œβ”€β”€ locking.nu # Locking mechanism (12 functions) -β”œβ”€β”€ extension_publish.nu # Publishing (10 functions) -β”œβ”€β”€ service.nu # Service management (11 functions) -β”œβ”€β”€ commands.nu # CLI commands (30+ commands) -└── IMPLEMENTATION_SUMMARY.md # This file -``` - ---- - -### 10. Testing βœ… - -**File:** `/Users/Akasha/project-provisioning/provisioning/core/nulib/tests/test_gitea.nu` - -**Test Suites:** -- `test-api-client` - API client operations -- `test-repository-operations` - Repository CRUD -- `test-release-operations` - Release management -- `test-issue-operations` - Issue operations -- `test-workspace-locking` - Lock acquisition/release -- `test-service-management` - Service status/health -- `test-workspace-git-mock` - Git operations (mock) -- `test-extension-publishing-mock` - Extension validation (mock) -- `run-all-tests` - Execute all tests - -**Features:** -- Setup/cleanup automation -- Assertion helpers -- Integration and mock tests -- Comprehensive coverage - ---- - -### 11. Documentation βœ… - -**File:** `/Users/Akasha/project-provisioning/docs/user/GITEA_INTEGRATION_GUIDE.md` - -**Sections:** -- Overview and architecture -- Setup and configuration -- Workspace git integration -- Workspace locking -- Extension publishing -- Service management -- API reference -- Troubleshooting -- Best practices -- Advanced usage - -**Features:** -- Complete user guide (600+ lines) -- Step-by-step examples -- Troubleshooting scenarios -- Best practices -- API reference -- Architecture diagrams - ---- - -## Integration Points - -### 1. Configuration System -- KCL schema: `provisioning/kcl/gitea.k` -- Config loader integration via `get-gitea-config()` -- SOPS encrypted token support - -### 2. Workspace System -- Git integration for workspaces -- Locking for concurrent access -- Remote repository management - -### 3. Extension System -- Publishing to Gitea releases -- Download from releases -- Version management - -### 4. Mode System -- Gitea configuration per mode -- Local vs remote deployment -- Environment-specific settings - ---- - -## Technical Features - -### API Client -- βœ… Full REST API v1 support -- βœ… Token-based authentication -- βœ… SOPS encrypted tokens -- βœ… HTTP methods: GET, POST, PUT, DELETE, PATCH -- βœ… Error handling -- βœ… Response parsing - -### Workspace Git -- βœ… Repository initialization -- βœ… Clone operations -- βœ… Push/pull synchronization -- βœ… Branch management -- βœ… Status tracking -- βœ… Stash operations - -### Locking -- βœ… Distributed locking via issues -- βœ… Lock types: read, write, deploy -- βœ… Conflict detection -- βœ… Lock expiry -- βœ… Force unlock -- βœ… Automatic cleanup - -### Extension Publishing -- βœ… Structure validation -- βœ… Packaging (tar.gz) -- βœ… Release creation -- βœ… Asset upload -- βœ… Metadata generation -- βœ… Batch publishing - -### Service Management -- βœ… Docker deployment -- βœ… Binary deployment -- βœ… Start/stop/restart -- βœ… Health monitoring -- βœ… Log streaming -- βœ… Auto-start - ---- - -## File Summary - -| Category | File | Lines | Functions/Schemas | -|----------|------|-------|-------------------| -| Schema | `kcl/gitea.k` | 380 | 13 schemas | -| API Client | `gitea/api_client.nu` | 450 | 42 functions | -| Workspace Git | `gitea/workspace_git.nu` | 420 | 20 functions | -| Locking | `gitea/locking.nu` | 380 | 12 functions | -| Extension Publishing | `gitea/extension_publish.nu` | 380 | 10 functions | -| Service Management | `gitea/service.nu` | 420 | 11 functions | -| CLI Commands | `gitea/commands.nu` | 380 | 30+ commands | -| Module | `gitea/mod.nu` | 10 | 6 exports | -| Docker | `config/gitea/docker-compose.yml` | 35 | N/A | -| Config Template | `config/gitea/app.ini.template` | 60 | N/A | -| Tests | `tests/test_gitea.nu` | 350 | 8 test suites | -| Documentation | `docs/user/GITEA_INTEGRATION_GUIDE.md` | 650 | N/A | -| **Total** | **12 files** | **3,915 lines** | **95+ functions** | - ---- - -## Usage Examples - -### Basic Workflow - -```bash -# 1. Start Gitea -provisioning gitea start - -# 2. Initialize workspace with git -provisioning workspace init my-workspace --git --remote gitea - -# 3. Acquire lock -provisioning gitea lock acquire my-workspace write --operation "Deploy servers" - -# 4. Make changes -cd workspace_my-workspace -# ... edit configs ... - -# 5. Push changes -provisioning workspace push --message "Updated server configs" - -# 6. Release lock -provisioning gitea lock release my-workspace 42 -``` - -### Extension Publishing - -```bash -# Publish taskserv -provisioning gitea extension publish \ - ./extensions/taskservs/database/postgres \ - 1.2.0 \ - --release-notes "Added connection pooling" - -# Download extension -provisioning gitea extension download postgres 1.2.0 -``` - -### Collaboration - -```bash -# Developer 1: Clone workspace -provisioning workspace clone workspaces/production ./prod-workspace - -# Developer 2: Check locks before changes -provisioning gitea lock list production - -# Developer 2: Acquire lock if free -provisioning gitea lock acquire production write -``` - ---- - -## Testing - -### Run Tests - -```bash -# All tests (requires running Gitea) -nu provisioning/core/nulib/tests/test_gitea.nu run-all-tests - -# Unit tests only (no integration) -nu provisioning/core/nulib/tests/test_gitea.nu run-all-tests --skip-integration -``` - -### Test Coverage - -- βœ… API client operations -- βœ… Repository CRUD -- βœ… Release management -- βœ… Issue operations (locking) -- βœ… Workspace locking logic -- βœ… Service management -- βœ… Git operations (mock) -- βœ… Extension validation (mock) - ---- - -## Next Steps - -### Recommended Enhancements - -1. **Webhooks Integration** - - Implement webhook handlers - - Automated workflows on git events - - CI/CD integration - -2. **Advanced Locking** - - Lock priority system - - Lock queuing - - Lock notifications - -3. **Extension Marketplace** - - Web UI for browsing extensions - - Extension ratings/reviews - - Dependency resolution - -4. **Workspace Templates** - - Template repository system - - Workspace scaffolding - - Best practices templates - -5. **Collaboration Features** - - Pull request workflows - - Code review integration - - Team management - ---- - -## Known Limitations - -1. **Comment API**: Gitea basic API doesn't support adding comments to issues directly -2. **SSH Keys**: SSH key management not yet implemented -3. **Webhooks**: Webhook creation supported in schema but not automated -4. **Binary Deployment**: Process management for binary mode is basic - ---- - -## Security Considerations - -1. **Token Storage**: Always use SOPS encryption for tokens -2. **Repository Privacy**: Default to private repositories -3. **Lock Validation**: Validate lock ownership before release -4. **Token Rotation**: Implement regular token rotation -5. **Audit Logging**: All lock operations are tracked via issues - ---- - -## Performance Notes - -1. **API Rate Limiting**: Gitea has rate limits, batch operations may need throttling -2. **Large Files**: Git LFS not yet integrated for large workspace files -3. **Lock Cleanup**: Run cleanup periodically to prevent issue buildup -4. **Docker Resources**: Monitor container resources for local deployments - ---- - -## Conclusion - -The Gitea integration is **complete and production-ready** with: - -- βœ… 95+ functions across 6 modules -- βœ… 13 KCL schemas for configuration -- βœ… 30+ CLI commands -- βœ… Comprehensive testing suite -- βœ… Complete documentation (650+ lines) -- βœ… Docker and binary deployment support -- βœ… Workspace git integration -- βœ… Distributed locking mechanism -- βœ… Extension publishing workflow - -The implementation follows all PAP principles: -- Configuration-driven (KCL schemas) -- Modular architecture (6 focused modules) -- Idiomatic Nushell (explicit types, pure functions) -- Comprehensive documentation -- Extensive testing - ---- - -**Version:** 1.0.0 -**Implementation Date:** 2025-10-06 -**Status:** βœ… Complete -**Next Review:** 2025-11-06 diff --git a/nulib/lib_provisioning/gitea/extension_publish.nu b/nulib/lib_provisioning/gitea/extension_publish.nu index 134f54a..64d46df 100644 --- a/nulib/lib_provisioning/gitea/extension_publish.nu +++ b/nulib/lib_provisioning/gitea/extension_publish.nu @@ -20,20 +20,20 @@ def validate-extension [ } # Check for required files - let has_kcl_mod = $"($ext_path)/kcl/kcl.mod" | path exists + let has_nickel_mod = $"($ext_path)/nickel/nickel.mod" | path exists let has_main_file = ( - ls $"($ext_path)/kcl/*.k" | where name !~ ".*test.*" | length + ls $"($ext_path)/nickel/*.ncl" | where name !~ ".*test.*" | length ) > 0 - if not $has_kcl_mod { + if not $has_nickel_mod { error make { - msg: "Extension missing kcl/kcl.mod" + msg: "Extension missing nickel/nickel.mod" } } if not $has_main_file { error make { - msg: "Extension missing main KCL file" + msg: "Extension missing main Nickel file" } } @@ -377,4 +377,4 @@ export def publish-extensions-batch [ null } } | where {|x| $x != null} -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/gitea/locking.nu b/nulib/lib_provisioning/gitea/locking.nu index 04647e5..3414c2e 100644 --- a/nulib/lib_provisioning/gitea/locking.nu +++ b/nulib/lib_provisioning/gitea/locking.nu @@ -424,4 +424,4 @@ export def with-workspace-lock [ release-workspace-lock $workspace_name $lock.lock_id $cmd_result.stdout -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/agent_interface.nu b/nulib/lib_provisioning/infra_validator/agent_interface.nu index 50817bb..a938f88 100644 --- a/nulib/lib_provisioning/infra_validator/agent_interface.nu +++ b/nulib/lib_provisioning/infra_validator/agent_interface.nu @@ -371,4 +371,4 @@ export def webhook_validate [ } $response -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/config_loader.nu b/nulib/lib_provisioning/infra_validator/config_loader.nu index c2d606e..8345b5c 100644 --- a/nulib/lib_provisioning/infra_validator/config_loader.nu +++ b/nulib/lib_provisioning/infra_validator/config_loader.nu @@ -240,4 +240,4 @@ export def create_rule_context [ rule_timeout: ($rule.timeout | default 30) auto_fix_enabled: (($rule.auto_fix | default false) and ($global_context.fix_mode | default false)) } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/report_generator.nu b/nulib/lib_provisioning/infra_validator/report_generator.nu index 7f8097f..c37badf 100644 --- a/nulib/lib_provisioning/infra_validator/report_generator.nu +++ b/nulib/lib_provisioning/infra_validator/report_generator.nu @@ -325,4 +325,4 @@ export def generate_enhancement_report [results: record, context: record]: nothi } $report -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/rules_engine.nu b/nulib/lib_provisioning/infra_validator/rules_engine.nu index 56830d6..422cb26 100644 --- a/nulib/lib_provisioning/infra_validator/rules_engine.nu +++ b/nulib/lib_provisioning/infra_validator/rules_engine.nu @@ -27,19 +27,19 @@ export def get_yaml_syntax_rule []: nothing -> record { } } -# KCL Compilation Rule -export def get_kcl_compilation_rule []: nothing -> record { +# Nickel Compilation Rule +export def get_nickel_compilation_rule []: nothing -> record { { id: "VAL002" category: "compilation" severity: "critical" - name: "KCL Compilation Check" - description: "Validate KCL files compile successfully" - files_pattern: '.*\.k$' - validator: "validate_kcl_compilation" + name: "Nickel Compilation Check" + description: "Validate Nickel files compile successfully" + files_pattern: '.*\.ncl$' + validator: "validate_nickel_compilation" auto_fix: false fix_function: null - tags: ["kcl", "compilation", "critical"] + tags: ["nickel", "compilation", "critical"] } } @@ -154,7 +154,7 @@ export def execute_rule [ # Execute the validation function based on the rule configuration match $function_name { "validate_yaml_syntax" => (validate_yaml_syntax $file) - "validate_kcl_compilation" => (validate_kcl_compilation $file) + "validate_nickel_compilation" => (validate_nickel_compilation $file) "validate_quoted_variables" => (validate_quoted_variables $file) "validate_required_fields" => (validate_required_fields $file) "validate_naming_conventions" => (validate_naming_conventions $file) @@ -263,13 +263,13 @@ export def validate_quoted_variables [file: string]: nothing -> record { } } -export def validate_kcl_compilation [file: string]: nothing -> record { - # Check if KCL compiler is available - let kcl_check = (do { - ^bash -c "type -P kcl" | ignore +export def validate_nickel_compilation [file: string]: nothing -> record { + # Check if Nickel compiler is available + let decl_check = (do { + ^bash -c "type -P nickel" | ignore } | complete) - if $kcl_check.exit_code != 0 { + if $nickel_check.exit_code != 0 { { passed: false issue: { @@ -277,16 +277,16 @@ export def validate_kcl_compilation [file: string]: nothing -> record { severity: "critical" file: $file line: null - message: "KCL compiler not available" - details: "kcl command not found in PATH" - suggested_fix: "Install KCL compiler or add to PATH" + message: "Nickel compiler not available" + details: "nickel command not found in PATH" + suggested_fix: "Install Nickel compiler or add to PATH" auto_fixable: false } } } else { - # Try to compile the KCL file + # Try to compile the Nickel file let compile_result = (do { - ^kcl $file | ignore + ^nickel $file | ignore } | complete) if $compile_result.exit_code != 0 { @@ -297,9 +297,9 @@ export def validate_kcl_compilation [file: string]: nothing -> record { severity: "critical" file: $file line: null - message: "KCL compilation failed" + message: "Nickel compilation failed" details: $compile_result.stderr - suggested_fix: "Fix KCL syntax and compilation errors" + suggested_fix: "Fix Nickel syntax and compilation errors" auto_fixable: false } } @@ -314,8 +314,8 @@ export def validate_required_fields [file: string]: nothing -> record { let content = (open $file --raw) # Check for common required fields based on file type - if ($file | str ends-with ".k") { - # KCL server configuration checks + if ($file | str ends-with ".ncl") { + # Nickel server configuration checks if ($content | str contains "servers") and (not ($content | str contains "hostname")) { { passed: false @@ -390,4 +390,4 @@ export def fix_unquoted_variables [file: string, issue: record]: nothing -> reco export def fix_naming_conventions [file: string, issue: record]: nothing -> record { # Placeholder for naming convention fixes { success: false, message: "Naming convention auto-fix not implemented yet" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/schema_validator.nu b/nulib/lib_provisioning/infra_validator/schema_validator.nu index 1aa2676..7be8b51 100644 --- a/nulib/lib_provisioning/infra_validator/schema_validator.nu +++ b/nulib/lib_provisioning/infra_validator/schema_validator.nu @@ -311,4 +311,4 @@ export def get_taskserv_schema []: nothing -> record { target_save_path: "string" } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/infra_validator/validation_config.toml b/nulib/lib_provisioning/infra_validator/validation_config.toml index aea7090..bddb61a 100644 --- a/nulib/lib_provisioning/infra_validator/validation_config.toml +++ b/nulib/lib_provisioning/infra_validator/validation_config.toml @@ -55,7 +55,6 @@ enabled = true auto_fix = false files_pattern = '.*\.k$' validator_function = "validate_kcl_compilation" -fix_function = null execution_order = 2 tags = ["kcl", "compilation", "critical"] dependencies = ["kcl"] # Required system dependencies @@ -84,7 +83,6 @@ enabled = true auto_fix = false files_pattern = '.*\.(k|ya?ml)$' validator_function = "validate_required_fields" -fix_function = null execution_order = 10 tags = ["schema", "required", "fields"] @@ -112,7 +110,6 @@ enabled = true auto_fix = false files_pattern = '.*\.(k|ya?ml)$' validator_function = "validate_security_basics" -fix_function = null execution_order = 15 tags = ["security", "ssh", "ports"] @@ -126,7 +123,6 @@ enabled = true auto_fix = false files_pattern = '.*\.(k|ya?ml|toml)$' validator_function = "validate_version_compatibility" -fix_function = null execution_order = 25 tags = ["versions", "compatibility", "deprecation"] @@ -140,7 +136,6 @@ enabled = true auto_fix = false files_pattern = '.*\.(k|ya?ml)$' validator_function = "validate_network_config" -fix_function = null execution_order = 18 tags = ["networking", "cidr", "ip"] @@ -223,4 +218,4 @@ custom_rules = ["K8S001", "K8S002"] [taskservs.containerd] enabled_rules = ["VAL001", "VAL004", "VAL006"] -custom_rules = ["CONTAINERD001"] \ No newline at end of file +custom_rules = ["CONTAINERD001"] diff --git a/nulib/lib_provisioning/infra_validator/validator.nu b/nulib/lib_provisioning/infra_validator/validator.nu index bd343fa..d39811a 100644 --- a/nulib/lib_provisioning/infra_validator/validator.nu +++ b/nulib/lib_provisioning/infra_validator/validator.nu @@ -140,8 +140,8 @@ def load_validation_rules [context?: record]: nothing -> list { def discover_infrastructure_files [infra_path: string]: nothing -> list { mut files = [] - # KCL files - $files = ($files | append (glob $"($infra_path)/**/*.k")) + # Nickel files + $files = ($files | append (glob $"($infra_path)/**/*.ncl")) # YAML files $files = ($files | append (glob $"($infra_path)/**/*.yaml")) @@ -293,9 +293,9 @@ def determine_exit_code [results: record]: nothing -> int { def detect_provider [infra_path: string]: nothing -> string { # Try to detect provider from file structure or configuration - let kcl_files = (glob ($infra_path | path join "**/*.k")) + let nickel_files = (glob ($infra_path | path join "**/*.ncl")) - for file in $kcl_files { + for file in $decl_files { let content = (open $file --raw) if ($content | str contains "upcloud") { return "upcloud" @@ -321,10 +321,10 @@ def detect_provider [infra_path: string]: nothing -> string { def detect_taskservs [infra_path: string]: nothing -> list { mut taskservs = [] - let kcl_files = (glob ($infra_path | path join "**/*.k")) + let nickel_files = (glob ($infra_path | path join "**/*.ncl")) let yaml_files = (glob ($infra_path | path join "**/*.yaml")) - let all_files = ($kcl_files | append $yaml_files) + let all_files = ($decl_files | append $yaml_files) for file in $all_files { let content = (open $file --raw) @@ -344,4 +344,4 @@ def detect_taskservs [infra_path: string]: nothing -> list { } $taskservs | uniq -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/integrations/iac/iac_orchestrator.nu b/nulib/lib_provisioning/integrations/iac/iac_orchestrator.nu index d7ca1dd..43deb63 100644 --- a/nulib/lib_provisioning/integrations/iac/iac_orchestrator.nu +++ b/nulib/lib_provisioning/integrations/iac/iac_orchestrator.nu @@ -192,8 +192,8 @@ def generate-workflow-phases [ [$phase1_tasks, $phase2_tasks, $phase3_tasks] | flatten } -# Export workflow to KCL format for orchestrator -export def export-workflow-kcl [workflow] { +# Export workflow to Nickel format for orchestrator +export def export-workflow-nickel [workflow] { # Handle both direct workflow and nested structure let w = ( try { $workflow.workflow } catch { $workflow } @@ -462,4 +462,4 @@ export def orchestrate-from-iac [ print $" Error: ($submission.message)" $submission } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/kms/client.nu b/nulib/lib_provisioning/kms/client.nu index 9f7fed6..efbf7b1 100644 --- a/nulib/lib_provisioning/kms/client.nu +++ b/nulib/lib_provisioning/kms/client.nu @@ -675,4 +675,4 @@ export def main [] { print "" print "Supported Backends:" print " age, aws-kms, vault, cosmian" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/kms/lib.nu b/nulib/lib_provisioning/kms/lib.nu index 72150b8..c913573 100644 --- a/nulib/lib_provisioning/kms/lib.nu +++ b/nulib/lib_provisioning/kms/lib.nu @@ -96,11 +96,11 @@ export def run_cmd_kms [ } export def on_kms [ - task: string - source_path: string - output_path?: string - ...args - --check (-c) + task: string + source_path: string + output_path?: string + ...args + --check (-c) --error_exit --quiet ]: nothing -> string { @@ -202,18 +202,18 @@ def build_kms_command [ config: record ]: nothing -> string { mut cmd_parts = [] - + # Base command - using curl to interact with Cosmian KMS REST API $cmd_parts = ($cmd_parts | append "curl") - + # SSL verification if not $config.verify_ssl { $cmd_parts = ($cmd_parts | append "-k") } - + # Timeout $cmd_parts = ($cmd_parts | append $"--connect-timeout ($config.timeout)") - + # Authentication match $config.auth_method { "certificate" => { @@ -236,7 +236,7 @@ def build_kms_command [ } } } - + # Operation specific parameters match $operation { "encrypt" => { @@ -252,7 +252,7 @@ def build_kms_command [ $cmd_parts = ($cmd_parts | append $"($config.server_url)/decrypt") } } - + ($cmd_parts | str join " ") } @@ -279,4 +279,4 @@ export def get_def_kms_config [ exit 1 } ($provisioning_kms | default "") -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/kms/mod.nu b/nulib/lib_provisioning/kms/mod.nu index 66cb1fe..c0ccfad 100644 --- a/nulib/lib_provisioning/kms/mod.nu +++ b/nulib/lib_provisioning/kms/mod.nu @@ -1,2 +1,2 @@ export use lib.nu * -export use client.nu * \ No newline at end of file +export use client.nu * diff --git a/nulib/lib_provisioning/layers/resolver.nu b/nulib/lib_provisioning/layers/resolver.nu index 537c45b..2a404ad 100644 --- a/nulib/lib_provisioning/layers/resolver.nu +++ b/nulib/lib_provisioning/layers/resolver.nu @@ -82,7 +82,7 @@ def resolve-system-module [name: string, type: string]: nothing -> record { let result = (do { let info = (get-taskserv-info $name) { - path: $info.kcl_path + path: $info.schema_path layer: "system" layer_number: 1 name: $name @@ -102,7 +102,7 @@ def resolve-system-module [name: string, type: string]: nothing -> record { let result = (do { let info = (get-provider-info $name) { - path: $info.kcl_path + path: $info.schema_path layer: "system" layer_number: 1 name: $name @@ -122,7 +122,7 @@ def resolve-system-module [name: string, type: string]: nothing -> record { let result = (do { let info = (get-cluster-info $name) { - path: $info.kcl_path + path: $info.schema_path layer: "system" layer_number: 1 name: $name @@ -310,4 +310,4 @@ export def print-resolution [resolution: record]: nothing -> nothing { } else { print $"❌ Module ($resolution.name) not found in any layer" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/mod.nu b/nulib/lib_provisioning/mod.nu index cb282f5..7c33b82 100644 --- a/nulib/lib_provisioning/mod.nu +++ b/nulib/lib_provisioning/mod.nu @@ -15,3 +15,5 @@ export use providers.nu * export use workspace * export use config * export use diagnostics * +#export use tera_daemon * +#export use fluent_daemon * diff --git a/nulib/lib_provisioning/mode/validator.nu b/nulib/lib_provisioning/mode/validator.nu index c2d101d..c4f375f 100644 --- a/nulib/lib_provisioning/mode/validator.nu +++ b/nulib/lib_provisioning/mode/validator.nu @@ -1,5 +1,5 @@ # Mode Configuration Validator -# Validates mode configurations against KCL schemas and runtime requirements +# Validates mode configurations against Nickel schemas and runtime requirements use ../utils/logging.nu * @@ -230,7 +230,7 @@ def validate-services-config [services: record] -> record { if "namespaces" in $oci { let ns = $oci.namespaces - let required_ns = ["extensions", "kcl_packages", "platform_images", "test_images"] + let required_ns = ["extensions", "nickel_packages", "platform_images", "test_images"] for n in $required_ns { if not ($n in $ns) { $warnings = ($warnings | append $"OCI registry namespace missing: ($n)") diff --git a/nulib/lib_provisioning/kcl_module_loader.nu b/nulib/lib_provisioning/module_loader.nu similarity index 70% rename from nulib/lib_provisioning/kcl_module_loader.nu rename to nulib/lib_provisioning/module_loader.nu index 9aea7b6..d029c60 100644 --- a/nulib/lib_provisioning/kcl_module_loader.nu +++ b/nulib/lib_provisioning/module_loader.nu @@ -1,5 +1,5 @@ -# KCL Module Loader Library -# Provides functions for discovering, syncing, and managing KCL modules +# Nickel Module Loader Library +# Provides functions for discovering, syncing, and managing Nickel modules # Used by CLI commands and other components # Author: JesusPerezLorenzo # Date: 2025-09-29 @@ -8,50 +8,46 @@ use config/accessor.nu * use config/cache/simple-cache.nu * use utils * -# Discover KCL modules from extensions (providers, taskservs, clusters) -export def "discover-kcl-modules" [ +# Discover Nickel modules from extensions (providers, taskservs, clusters) +export def "discover-nickel-modules" [ type: string # "providers" | "taskservs" | "clusters" ]: nothing -> table { - # Get base paths from config using config-get with proper fallback - let configured_path = (config-get $"paths.($type)" "") - let base_path = if ($configured_path | is-not-empty) { - $configured_path - } else { - # Fallback to system extensions path - let proj_root = ($env.PROVISIONING_ROOT? | default "/Users/Akasha/project-provisioning") - ($proj_root | path join "provisioning" "extensions" $type) - } + # Fast path: don't load config, just use extensions path directly + # This avoids Nickel evaluation which can hang the system + let proj_root = ($env.PROVISIONING_ROOT? | default "/Users/Akasha/project-provisioning") + let base_path = ($proj_root | path join "provisioning" "extensions" $type) if not ($base_path | path exists) { return [] } # Discover modules using directory structure + # Use proper Nushell ls with null stdin to avoid hanging let modules = (ls $base_path | where type == "dir" | get name | path basename) - # Build table with KCL information + # Build table with Nickel information $modules | each {|module_name| let module_path = ($base_path | path join $module_name) - let kcl_path = ($module_path | path join "kcl") + let schema_path = ($module_path | path join "nickel") - # Check if KCL directory exists - if not ($kcl_path | path exists) { + # Check if Nickel directory exists + if not ($schema_path | path exists) { return null } - # Read kcl.mod for metadata - let kcl_mod_path = ($kcl_path | path join "kcl.mod") - let metadata = if ($kcl_mod_path | path exists) { - parse-kcl-mod $kcl_mod_path + # Read nickel.mod for metadata + let mod_path = ($schema_path | path join "nickel.mod") + let metadata = if ($mod_path | path exists) { + parse-nickel-mod $mod_path } else { {name: "", version: "0.0.1", edition: "v0.11.3"} } - # Determine KCL module name based on type - let kcl_module_name = match $type { + # Determine Nickel module name based on type + let module_name = match $type { "providers" => $"($module_name)_prov" "taskservs" => $"($module_name)_task" "clusters" => $"($module_name)_cluster" @@ -62,31 +58,31 @@ export def "discover-kcl-modules" [ name: $module_name type: $type path: $module_path - kcl_path: $kcl_path - kcl_module_name: $kcl_module_name + schema_path: $schema_path + module_name: $module_name version: $metadata.version edition: $metadata.edition - has_kcl: true + has_nickel: true } } | compact } -# Cached version of discover-kcl-modules +# Cached version of discover-nickel-modules # NOTE: In practice, OS filesystem caching (dentry cache, inode cache) is more efficient # than custom caching due to Nushell's JSON serialization overhead. # This function is provided for future optimization when needed. -export def "discover-kcl-modules-cached" [ +export def "discover-nickel-modules-cached" [ type: string # "providers" | "taskservs" | "clusters" ]: nothing -> table { # Direct call - relies on OS filesystem cache for performance - discover-kcl-modules $type + discover-nickel-modules $type } -# Parse kcl.mod file and extract metadata -def "parse-kcl-mod" [ - kcl_mod_path: string +# Parse nickel.mod file and extract metadata +def "parse-nickel-mod" [ + mod_path: string ]: nothing -> record { - let content = (open $kcl_mod_path) + let content = (open $mod_path) # Simple TOML parsing for [package] section let lines = ($content | lines) @@ -107,8 +103,8 @@ def "parse-kcl-mod" [ {name: $name, version: $version, edition: $edition} } -# Sync KCL dependencies for an infrastructure workspace -export def "sync-kcl-dependencies" [ +# Sync Nickel dependencies for an infrastructure workspace +export def "sync-nickel-dependencies" [ infra_path: string --manifest: string = "providers.manifest.yaml" ] { @@ -119,13 +115,13 @@ export def "sync-kcl-dependencies" [ } let manifest = (open $manifest_path) - let modules_dir_name = (config-get "kcl.modules_dir" "kcl") + let modules_dir_name = (config-get "nickel.modules_dir" "nickel") let modules_dir = ($infra_path | path join $modules_dir_name) # Create modules directory if it doesn't exist mkdir $modules_dir - _print $"πŸ”„ Syncing KCL dependencies for ($infra_path | path basename)..." + _print $"πŸ”„ Syncing Nickel dependencies for ($infra_path | path basename)..." # Sync each provider from manifest if ($manifest | get providers? | is-not-empty) { @@ -134,10 +130,10 @@ export def "sync-kcl-dependencies" [ } } - # Update kcl.mod - update-kcl-mod $infra_path $manifest + # Update nickel.mod + update-nickel-mod $infra_path $manifest - _print "βœ… KCL dependencies synced successfully" + _print "βœ… Nickel dependencies synced successfully" } # Sync a single provider module (create symlink) @@ -145,7 +141,7 @@ def "sync-provider-module" [ provider: record modules_dir: string ] { - let discovered = (discover-kcl-modules-cached "providers" + let discovered = (discover-nickel-modules-cached "providers" | where name == $provider.name) if ($discovered | is-empty) { @@ -153,7 +149,7 @@ def "sync-provider-module" [ } let module_info = ($discovered | first) - let link_path = ($modules_dir | path join $module_info.kcl_module_name) + let link_path = ($modules_dir | path join $module_info.module_name) # Remove existing symlink if present if ($link_path | path exists) { @@ -161,7 +157,7 @@ def "sync-provider-module" [ } # Create symlink (relative path for portability) - let relative_path = (get-relative-path $modules_dir $module_info.kcl_path) + let relative_path = (get-relative-path $modules_dir $module_info.schema_path) # Use ln -sf for symlink ^ln -sf $relative_path $link_path @@ -175,41 +171,41 @@ def "get-relative-path" [ to: string ]: nothing -> string { # Calculate relative path - # For now, use absolute path (KCL handles this fine) + # For now, use absolute path (Nickel handles this fine) $to } -# Update kcl.mod with provider dependencies -export def "update-kcl-mod" [ +# Update nickel.mod with provider dependencies +export def "update-nickel-mod" [ infra_path: string manifest: record ] { - let kcl_mod_path = ($infra_path | path join "kcl.mod") + let mod_path = ($infra_path | path join "nickel.mod") - if not ($kcl_mod_path | path exists) { - error make {msg: $"kcl.mod not found at ($kcl_mod_path)"} + if not ($mod_path | path exists) { + error make {msg: $"nickel.mod not found at ($mod_path)"} } - let current_mod = (open $kcl_mod_path) - let modules_dir_name = (get-config | get kcl.modules_dir) + let current_mod = (open $mod_path) + let modules_dir_name = (get-config | get nickel.modules_dir) # Generate provider dependencies let provider_deps = if ($manifest | get providers? | is-not-empty) { # Load all providers once to cache them - let all_providers = (discover-kcl-modules-cached "providers") + let all_providers = (discover-nickel-modules-cached "providers") $manifest.providers | each {|provider| let discovered = ($all_providers | where name == $provider.name) if ($discovered | is-empty) { return "" } let module_info = ($discovered | first) - $"($module_info.kcl_module_name) = { path = \"./($modules_dir_name)/($module_info.kcl_module_name)\", version = \"($provider.version)\" }" + $"($module_info.module_name) = { path = \"./($modules_dir_name)/($module_info.module_name)\", version = \"($provider.version)\" }" } | str join "\n" } else { "" } - # Parse current kcl.mod and update dependencies section + # Parse current nickel.mod and update dependencies section let lines = ($current_mod | lines) mut in_deps = false mut new_lines = [] @@ -249,10 +245,10 @@ export def "update-kcl-mod" [ } } - # Write updated kcl.mod - $new_lines | str join "\n" | save -f $kcl_mod_path + # Write updated nickel.mod + $new_lines | str join "\n" | save -f $mod_path - _print $" βœ“ Updated kcl.mod with provider dependencies" + _print $" βœ“ Updated nickel.mod with provider dependencies" } # Install a provider to an infrastructure @@ -262,7 +258,7 @@ export def "install-provider" [ --version: string = "0.0.1" ] { # Discover provider using cached version - let available = (discover-kcl-modules-cached "providers" | where name == $provider_name) + let available = (discover-nickel-modules-cached "providers" | where name == $provider_name) if ($available | is-empty) { error make {msg: $"Provider '($provider_name)' not found"} @@ -275,8 +271,8 @@ export def "install-provider" [ # Update or create manifest update-manifest $infra_path $provider_name $version - # Sync KCL dependencies - sync-kcl-dependencies $infra_path + # Sync Nickel dependencies + sync-nickel-dependencies $infra_path _print $"βœ… Provider ($provider_name) installed successfully" } @@ -339,13 +335,13 @@ export def "remove-provider" [ $updated_manifest | to yaml | save -f $manifest_path # Remove symlink - let modules_dir_name = (get-config | get kcl.modules_dir) + let modules_dir_name = (get-config | get nickel.modules_dir) let modules_dir = ($infra_path | path join $modules_dir_name) - let discovered = (discover-kcl-modules-cached "providers" | where name == $provider_name) + let discovered = (discover-nickel-modules-cached "providers" | where name == $provider_name) if not ($discovered | is-empty) { let module_info = ($discovered | first) - let link_path = ($modules_dir | path join $module_info.kcl_module_name) + let link_path = ($modules_dir | path join $module_info.module_name) if ($link_path | path exists) { rm -f $link_path @@ -353,23 +349,23 @@ export def "remove-provider" [ } } - # Sync to update kcl.mod - sync-kcl-dependencies $infra_path + # Sync to update nickel.mod + sync-nickel-dependencies $infra_path _print $"βœ… Provider ($provider_name) removed successfully" } -# List all available KCL modules -export def "list-kcl-modules" [ +# List all available Nickel modules +export def "list-nickel-modules" [ type: string # "providers" | "taskservs" | "clusters" | "all" ]: nothing -> table { if $type == "all" { - let providers = (discover-kcl-modules-cached "providers" | insert module_type "provider") - let taskservs = (discover-kcl-modules-cached "taskservs" | insert module_type "taskserv") - let clusters = (discover-kcl-modules-cached "clusters" | insert module_type "cluster") + let providers = (discover-nickel-modules-cached "providers" | insert module_type "provider") + let taskservs = (discover-nickel-modules-cached "taskservs" | insert module_type "taskserv") + let clusters = (discover-nickel-modules-cached "clusters" | insert module_type "cluster") $providers | append $taskservs | append $clusters } else { - discover-kcl-modules-cached $type | insert module_type $type + discover-nickel-modules-cached $type | insert module_type $type } } diff --git a/nulib/lib_provisioning/nickel/migration_helper.nu b/nulib/lib_provisioning/nickel/migration_helper.nu new file mode 100644 index 0000000..4bd0988 --- /dev/null +++ b/nulib/lib_provisioning/nickel/migration_helper.nu @@ -0,0 +1,281 @@ +# | Nickel to Nickel Migration Helper +# | Automates pattern detection and application +# | Follows: .claude/kcl_to_nickel_migration_framework.md +# | Author: Migration Framework +# | Date: 2025-12-15 + +# ============================================================ +# Pattern Detection +# ============================================================ + +# Detect if Nickel file uses schema inheritance pattern +export def "detect-inheritance" [decl_file: path] -> bool { + let content = open $decl_file | into string + ($content | str contains "schema ") and ($content | str contains "(") +} + +# Detect if Nickel file exports global instances +export def "detect-exports" [decl_file: path] -> list { + let content = open $decl_file | into string + $content + | split row "\n" + | filter { |line| ($line | str contains ": ") and not ($line | str contains "schema") } + | filter { |line| ($line | str contains " = ") } + | map { |line| $line | str trim } +} + +# Detect if Nickel file only defines schemas (no exports) +export def "is-schema-only" [decl_file: path] -> bool { + let exports = (detect-exports $decl_file) + ($exports | length) == 0 +} + +# Get migration template type for Nickel file +export def "get-template-type" [decl_file: path] -> string { + let has_inheritance = (detect-inheritance $decl_file) + let is_empty_export = (is-schema-only $decl_file) + let exports = (detect-exports $decl_file) + let export_count = ($exports | length) + + if $is_empty_export { + "template-1-schema-only" + } else if $has_inheritance { + "template-4-inheritance" + } else if $export_count == 1 { + "template-2-single-instance" + } else if $export_count > 1 { + "template-5-multiple-schemas" + } else { + "template-3-complex-nesting" + } +} + +# ============================================================ +# Value Conversion +# ============================================================ + +# Convert Nickel boolean to Nickel +export def "convert-boolean" [value: string] -> string { + match ($value | str trim) { + "True" => "true", + "False" => "false", + "true" => "true", + "false" => "false", + _ => $value, + } +} + +# Convert Nickel None to Nickel null +export def "convert-none" [value: string] -> string { + match ($value | str trim) { + "None" => "null", + "null" => "null", + _ => $value, + } +} + +# Convert Nickel value to Nickel value +export def "convert-value" [decl_value: string] -> string { + let trimmed = ($decl_value | str trim) + let bool_converted = (convert-boolean $trimmed) + (convert-none $bool_converted) +} + +# ============================================================ +# JSON Equivalence Validation +# ============================================================ + +# Export Nickel file to JSON for comparison +export def "nickel-to-json" [decl_file: path] { + if not ($decl_file | path exists) { + error make {msg: $"Nickel file not found: ($decl_file)"} + } + + nickel export $decl_file --format json 2>&1 +} + +# Export Nickel file to JSON for comparison +export def "nickel-to-json" [nickel_file: path] { + if not ($nickel_file | path exists) { + error make {msg: $"Nickel file not found: ($nickel_file)"} + } + + nickel export $nickel_file 2>&1 | from json | to json +} + +# Compare Nickel and Nickel JSON outputs for equivalence +export def "compare-equivalence" [decl_file: path, nickel_file: path] -> bool { + let source_json = (nickel-to-json $decl_file | from json) + let nickel_json = (nickel-to-json $nickel_file | from json) + + $source_json == $nickel_json +} + +# Show detailed comparison between Nickel and Nickel +export def "show-comparison" [decl_file: path, nickel_file: path] { + print $"Comparing: ($decl_file) ⇄ ($nickel_file)\n" + + let source_json = (nickel-to-json $decl_file) + let nickel_json = (nickel-to-json $nickel_file) + + print "=== Source Output (JSON) ===" + print $source_json + print "" + print "=== Target Output (JSON) ===" + print $nickel_json + print "" + + let equivalent = ($source_json == $nickel_json) + if $equivalent { + print "βœ… Outputs are EQUIVALENT" + } else { + print "❌ Outputs DIFFER" + print "\nDifferences:" + diff <(print $source_json | jq -S .) <(print $nickel_json | jq -S .) + } +} + +# ============================================================ +# Migration Workflow +# ============================================================ + +# Analyze Nickel file and recommend migration approach +export def "analyze-nickel" [decl_file: path] { + if not ($decl_file | path exists) { + error make {msg: $"File not found: ($decl_file)"} + } + + let template = (get-template-type $decl_file) + let has_inheritance = (detect-inheritance $decl_file) + let exports = (detect-exports $decl_file) + let is_empty = (is-schema-only $decl_file) + + print $"File: ($decl_file)" + print $"Template Type: ($template)" + print $"Has Schema Inheritance: ($has_inheritance)" + print $"Is Schema-Only (no exports): ($is_empty)" + print $"Exported Instances: ($exports | length)" + + if ($exports | length) > 0 { + print "\nExported instances:" + $exports | each { |exp| print $" - ($exp)" } + } +} + +# Generate skeleton Nickel file from Nickel template +export def "generate-nickel-skeleton" [decl_file: path, output_file: path] { + let template = (get-template-type $decl_file) + let source_name = ($decl_file | path basename | str replace ".ncl" "") + + let skeleton = match $template { + "template-1-schema-only" => { + $"# | Schema definitions migrated from ($source_name).ncl\n# | Migrated: 2025-12-15\n\n{{}}" + }, + "template-2-single-instance" => { + let exports = (detect-exports $decl_file) + let instance = ($exports | get 0 | str split " " | get 0) + $"# | Configuration migrated from ($source_name).ncl\n\n{\n ($instance) = {\n # TODO: Fill in fields\n },\n}" + }, + _ => { + $"# | Migrated from ($source_name).ncl\n# | Template: ($template)\n\n{\n # TODO: Implement\n}" + }, + } + + print $skeleton + print $"\nTo save: print output to ($output_file)" +} + +# ============================================================ +# Batch Migration +# ============================================================ + +# Migrate multiple Nickel files to Nickel using templates +export def "batch-migrate" [ + source_dir: path, + nickel_dir: path, + --pattern: string = "*.ncl", + --dry-run: bool = false, +] { + let source_files = (glob $"($source_dir)/($pattern)") + + print $"Found ($source_files | length) Nickel files matching pattern: ($pattern)" + print "" + + $source_files | each { |source_file| + let relative_path = ($source_file | str replace $"($source_dir)/" "") + let nickel_file = $"($nickel_dir)/($relative_path | str replace ".ncl" ".ncl")" + + print $"[$relative_path]" + let template = (get-template-type $source_file) + print $" Template: ($template)" + + if not $dry_run { + if ($nickel_file | path exists) { + print $" ⚠️ Already exists: ($nickel_file)" + } else { + print $" β†’ Would migrate to: ($nickel_file)" + } + } + } +} + +# ============================================================ +# Validation +# ============================================================ + +# Validate Nickel file syntax +export def "validate-nickel" [nickel_file: path] -> bool { + try { + nickel export $nickel_file | null + true + } catch { + false + } +} + +# Full migration validation for a file pair +export def "validate-migration" [decl_file: path, nickel_file: path] -> record { + let source_exists = ($decl_file | path exists) + let nickel_exists = ($nickel_file | path exists) + let nickel_valid = if $nickel_exists { (validate-nickel $nickel_file) } else { false } + let equivalent = if ($source_exists and $nickel_valid) { + (compare-equivalence $decl_file $nickel_file) + } else { + false + } + + { + source_exists: $source_exists, + nickel_exists: $nickel_exists, + nickel_valid: $nickel_valid, + outputs_equivalent: $equivalent, + status: if $equivalent { "βœ… PASS" } else { "❌ FAIL" }, + } +} + +# Validation report for all migrated files +export def "validation-report" [source_dir: path, nickel_dir: path] { + let nickel_files = (glob $"($nickel_dir)/**/*.ncl") + + print $"Validation Report: ($nickel_files | length) Nickel files\n" + + let results = $nickel_files | map { |nickel_file| + let relative = ($nickel_file | str replace $"($nickel_dir)/" "") + let source_file = $"($source_dir)/($relative | str replace ".ncl" ".ncl")" + let validation = (validate-migration $source_file $nickel_file) + + print $"($validation.status) $relative" + if not $validation.nickel_valid { + print " ⚠️ Nickel syntax error" + } + if not $validation.outputs_equivalent { + print " ⚠️ JSON outputs differ" + } + + $validation + } + + let passed = ($results | where {|r| $r.outputs_equivalent} | length) + let total = ($results | length) + print $"\nSummary: ($passed)/($total) files PASS equivalence check" +} diff --git a/nulib/lib_provisioning/kcl_packaging.nu b/nulib/lib_provisioning/packaging.nu similarity index 87% rename from nulib/lib_provisioning/kcl_packaging.nu rename to nulib/lib_provisioning/packaging.nu index 4caff17..a921ef6 100644 --- a/nulib/lib_provisioning/kcl_packaging.nu +++ b/nulib/lib_provisioning/packaging.nu @@ -1,12 +1,12 @@ -# KCL Packaging Library -# Functions for packaging and distributing KCL modules +# Nickel Packaging Library +# Functions for packaging and distributing Nickel modules # Author: JesusPerezLorenzo # Date: 2025-09-29 use config/accessor.nu * use utils * -# Package core provisioning KCL schemas +# Package core provisioning Nickel schemas export def "pack-core" [ --output: string = "" # Output directory (from config if not specified) --version: string = "" # Version override @@ -15,7 +15,7 @@ export def "pack-core" [ # Get config let dist_config = (get-distribution-config) - let kcl_config = (get-kcl-config) + let nickel_config = (get-nickel-config) # Get pack path from config or use provided output let pack_path = if ($output | is-empty) { @@ -29,12 +29,12 @@ export def "pack-core" [ if ($base_path | is-empty) { error make {msg: "PROVISIONING_CONFIG or PROVISIONING environment variable must be set"} } - let core_module = ($kcl_config.core_module | str replace --all "{{paths.base}}" $base_path) + let core_module = ($nickel_config.core_module | str replace --all "{{paths.base}}" $base_path) let core_path = $core_module # Get version from config or use provided let core_version = if ($version | is-empty) { - $kcl_config.core_version + $nickel_config.core_version } else { $version } @@ -43,37 +43,37 @@ export def "pack-core" [ mkdir $pack_path let abs_pack_path = ($pack_path | path expand) - # Change to the KCL module directory to run packaging from inside + # Change to the Nickel module directory to run packaging from inside cd $core_path - # Check if kcl mod pkg is supported - let help_result = (^kcl mod --help | complete) + # Check if nickel mod pkg is supported + let help_result = (^nickel mod --help | complete) let has_pkg = ($help_result.stdout | str contains "pkg") if not $has_pkg { - _print $" ⚠️ KCL does not support 'kcl mod pkg'" - _print $" πŸ’‘ Please upgrade to KCL 0.11.3+ for packaging support" - error make {msg: "KCL packaging not supported in this version"} + _print $" ⚠️ Nickel does not support 'nickel mod pkg'" + _print $" πŸ’‘ Please upgrade to Nickel 0.11.3+ for packaging support" + error make {msg: "Nickel packaging not supported in this version"} } - # Run kcl mod pkg from inside the module directory with --target - _print $" Running: kcl mod pkg --target ($abs_pack_path)" - let result = (^kcl mod pkg --target $abs_pack_path | complete) + # Run nickel mod pkg from inside the module directory with --target + _print $" Running: nickel mod pkg --target ($abs_pack_path)" + let result = (^nickel mod pkg --target $abs_pack_path | complete) if $result.exit_code != 0 { error make {msg: $"Failed to package core: ($result.stderr)"} } - _print $" βœ“ KCL packaging completed" + _print $" βœ“ Nickel packaging completed" - # Find the generated package in the target directory (kcl creates .tar files) + # Find the generated package in the target directory (nickel creates .tar files) cd $abs_pack_path let package_files = (glob *.tar) if ($package_files | is-empty) { _print $" ⚠️ No .tar file created in ($abs_pack_path)" - _print $" πŸ’‘ Check if kcl.mod is properly configured" - error make {msg: "KCL packaging did not create output file"} + _print $" πŸ’‘ Check if nickel.mod is properly configured" + error make {msg: "Nickel packaging did not create output file"} } let package_file = ($package_files | first) @@ -104,7 +104,7 @@ export def "pack-provider" [ # Get provider path from config let config = (get-config) let providers_base = ($config | get paths.providers) - let provider_path = ($providers_base | path join $provider "kcl") + let provider_path = ($providers_base | path join $provider "nickel") if not ($provider_path | path exists) { error make {msg: $"Provider not found: ($provider) at ($provider_path)"} @@ -114,12 +114,12 @@ export def "pack-provider" [ mkdir $pack_path let abs_pack_path = ($pack_path | path expand) - # Change to the provider KCL directory to run packaging from inside + # Change to the provider Nickel directory to run packaging from inside cd $provider_path - # Run kcl mod pkg with target directory - _print $" Running: kcl mod pkg --target ($abs_pack_path)" - let result = (^kcl mod pkg --target $abs_pack_path | complete) + # Run nickel mod pkg with target directory + _print $" Running: nickel mod pkg --target ($abs_pack_path)" + let result = (^nickel mod pkg --target $abs_pack_path | complete) if $result.exit_code != 0 { error make {msg: $"Failed to package provider: ($result.stderr)"} @@ -138,11 +138,11 @@ export def "pack-provider" [ let package_file = ($package_files | first) _print $" βœ“ Package: ($package_file)" - # Read version from kcl.mod if not provided + # Read version from nickel.mod if not provided let pkg_version = if ($version | is-empty) { - let kcl_mod = ($provider_path | path join "kcl.mod") - if ($kcl_mod | path exists) { - parse-kcl-version $kcl_mod + let mod_file = ($provider_path | path join "nickel.mod") + if ($mod_file | path exists) { + parse-nickel-version $mod_file } else { "0.0.1" } @@ -160,7 +160,7 @@ export def "pack-provider" [ export def "pack-all-providers" [ --output: string = "" # Output directory ] { - use kcl_module_loader.nu * + use module_loader.nu * let dist_config = (get-distribution-config) let pack_path = if ($output | is-empty) { @@ -171,7 +171,7 @@ export def "pack-all-providers" [ _print "πŸ“¦ Packaging all providers..." - let providers = (discover-kcl-modules "providers") + let providers = (discover-nickel-modules "providers") mut packaged = [] @@ -226,11 +226,11 @@ def "generate-package-metadata" [ _print $" βœ“ Metadata: ($metadata_file)" } -# Parse version from kcl.mod -def "parse-kcl-version" [ - kcl_mod_path: string +# Parse version from nickel.mod +def "parse-nickel-version" [ + mod_path: string ]: nothing -> string { - let content = (open $kcl_mod_path) + let content = (open $mod_path) let lines = ($content | lines) for line in $lines { @@ -478,4 +478,4 @@ export def "clean-all-packages" [ } else { _print $"βœ… Cleaned ($all_packages | length) packages and ($all_metadata | length) metadata files" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/plugins/auth.nu b/nulib/lib_provisioning/plugins/auth.nu index 5b69bdd..f3639f4 100644 --- a/nulib/lib_provisioning/plugins/auth.nu +++ b/nulib/lib_provisioning/plugins/auth.nu @@ -3,15 +3,18 @@ # name = "auth login" # group = "authentication" # tags = ["authentication", "jwt", "interactive", "login"] -# version = "2.1.0" -# requires = ["forminquire.nu:1.0.0", "nushell:0.109.0"] -# note = "Migrated to FormInquire interactive forms for login and MFA enrollment" +# version = "3.0.0" +# requires = ["nushell:0.109.0"] +# note = "MIGRATION: ForminQuire (Jinja2 templates) archived. Use TypeDialog forms for auth flows" +# migration = "See: provisioning/.coder/archive/forminquire/ (deprecated) β†’ provisioning/.typedialog/provisioning/fragments/auth-*.toml (new)" # Authentication Plugin Wrapper with HTTP Fallback # Provides graceful degradation to HTTP API when nu_plugin_auth is unavailable use ../config/accessor.nu * -use ../../../forminquire/nulib/forminquire.nu * +# ARCHIVED: use ../../../forminquire/nulib/forminquire.nu * +# ForminQuire has been archived to: provisioning/.coder/archive/forminquire/ +# New solution: Use TypeDialog for authentication forms (auth-api-key.toml, auth-jwt.toml) use ../commands/traits.nu * # Check if auth plugin is available diff --git a/nulib/lib_provisioning/plugins/mod.nu b/nulib/lib_provisioning/plugins/mod.nu index 8116db6..041abbe 100644 --- a/nulib/lib_provisioning/plugins/mod.nu +++ b/nulib/lib_provisioning/plugins/mod.nu @@ -3,6 +3,7 @@ export use auth.nu * export use kms.nu * +export use secretumvault.nu * # Plugin management utilities use ../config/accessor.nu * @@ -32,16 +33,18 @@ export def list-plugins []: nothing -> table { ($plugin.name | str contains "auth") or ($plugin.name | str contains "kms") or ($plugin.name | str contains "orchestrator") or + ($plugin.name | str contains "secretumvault") or ($plugin.name | str contains "tera") or - ($plugin.name | str contains "kcl") + ($plugin.name | str contains "nickel") ) let status = if $is_core { "enabled" } else { "active" } let description = match $plugin.name { "auth" => "JWT authentication with MFA support" "kms" => "Key Management Service integration" + "secretumvault" => "SecretumVault KMS integration" "tera" => "Template rendering engine" - "kcl" => "KCL configuration language" + "nickel" => "Nickel configuration language" "clipboard" => "Clipboard operations" "desktop_notifications" => "Desktop notifications" "qr_maker" => "QR code generation" @@ -109,7 +112,7 @@ export def register-plugin [ # Test plugin functionality export def test-plugin [ - plugin_name: string # auth, kms, tera, kcl + plugin_name: string # auth, kms, secretumvault, tera, nickel ]: nothing -> record { match $plugin_name { "auth" => { @@ -129,6 +132,17 @@ export def test-plugin [ print $"Mode: ($info.mode)" $info } + "secretumvault" => { + print $"(_ansi cyan)Testing SecretumVault plugin...(_ansi reset)" + let info = (plugin-secretumvault-info) + print $"Plugin available: ($info.plugin_available)" + print $"Plugin enabled: ($info.plugin_enabled)" + print $"Service URL: ($info.service_url)" + print $"Mount point: ($info.mount_point)" + print $"Default key: ($info.default_key)" + print $"Mode: ($info.mode)" + $info + } "tera" => { print $"(_ansi cyan)Testing tera plugin...(_ansi reset)" let installed = (version).installed_plugins @@ -136,10 +150,10 @@ export def test-plugin [ print $"Plugin registered: ($available)" {plugin_available: $available} } - "kcl" => { - print $"(_ansi cyan)Testing KCL plugin...(_ansi reset)" + "nickel" => { + print $"(_ansi cyan)Testing Nickel plugin...(_ansi reset)" let installed = (version).installed_plugins - let available = ($installed | str contains "kcl") + let available = ($installed | str contains "nickel") print $"Plugin registered: ($available)" {plugin_available: $available} } @@ -147,7 +161,7 @@ export def test-plugin [ error make { msg: $"❌ Unknown plugin: ($plugin_name)" label: { - text: "Valid plugins: auth, kms, tera, kcl" + text: "Valid plugins: auth, kms, secretumvault, tera, nickel" span: (metadata $plugin_name).span } } diff --git a/nulib/lib_provisioning/plugins/orchestrator_test.nu b/nulib/lib_provisioning/plugins/orchestrator_test.nu index 9f81908..903c2f8 100644 --- a/nulib/lib_provisioning/plugins/orchestrator_test.nu +++ b/nulib/lib_provisioning/plugins/orchestrator_test.nu @@ -135,14 +135,14 @@ export def test_health_check [] { } } -# Test KCL validation -export def test_kcl_validation [] { - print " Testing KCL validation..." +# Test Nickel validation +export def test_nickel_validation [] { + print " Testing Nickel validation..." use orchestrator.nu * - # Create simple test KCL content - let kcl_content = ''' + # Create simple test Nickel content + let nickel_content = ''' schema TestSchema: name: str value: int @@ -154,13 +154,13 @@ config: TestSchema = { ''' let result = (do { - plugin-orch-validate-kcl $kcl_content + plugin-orch-validate-nickel $nickel_content } | complete) if $result.exit_code == 0 { - print " βœ… KCL validation succeeded" + print " βœ… Nickel validation succeeded" } else { - print " ⚠️ KCL validation failed (might need orchestrator running)" + print " ⚠️ Nickel validation failed (might need orchestrator running)" } } @@ -287,7 +287,7 @@ export def main [] { test_workflow_status test_batch_operations test_statistics - test_kcl_validation + test_nickel_validation test_config_integration test_error_handling test_orch_performance diff --git a/nulib/lib_provisioning/plugins/secretumvault.nu b/nulib/lib_provisioning/plugins/secretumvault.nu new file mode 100644 index 0000000..3acf78b --- /dev/null +++ b/nulib/lib_provisioning/plugins/secretumvault.nu @@ -0,0 +1,498 @@ +# SecretumVault Plugin Wrapper with HTTP Fallback +# Provides high-level functions for SecretumVault operations with graceful HTTP fallback + +use ../config/accessor.nu * + +# Check if SecretumVault plugin is available +def is-plugin-available []: nothing -> bool { + (which secretumvault | length) > 0 +} + +# Check if SecretumVault plugin is enabled in config +def is-plugin-enabled []: nothing -> bool { + config-get "plugins.secretumvault_enabled" true +} + +# Get SecretumVault service URL +def get-secretumvault-url []: nothing -> string { + config-get "kms.secretumvault.server_url" "http://localhost:8200" +} + +# Get SecretumVault auth token +def get-secretumvault-token []: nothing -> string { + let token = ( + if ($env.SECRETUMVAULT_TOKEN? != null) { + $env.SECRETUMVAULT_TOKEN + } else { + "" + } + ) + if ($token | is-empty) { + config-get "kms.secretumvault.auth_token" "" + } else { + $token + } +} + +# Get SecretumVault mount point +def get-secretumvault-mount-point []: nothing -> string { + config-get "kms.secretumvault.mount_point" "transit" +} + +# Get default SecretumVault key name +def get-secretumvault-key-name []: nothing -> string { + config-get "kms.secretumvault.key_name" "provisioning-master" +} + +# Helper to safely execute a closure and return null on error +def try-plugin [callback: closure]: nothing -> any { + do -i $callback +} + +# Encrypt data using SecretumVault plugin +export def plugin-secretumvault-encrypt [ + plaintext: string + --key-id: string = "" # Encryption key ID +] { + let enabled = is-plugin-enabled + let available = is-plugin-available + let key_name = if ($key_id | is-empty) { get-secretumvault-key-name } else { $key_id } + + if $enabled and $available { + let plugin_result = (try-plugin { + let args = if ($key_id | is-empty) { + [encrypt $plaintext] + } else { + [encrypt $plaintext --key-id $key_id] + } + + secretumvault ...$args + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault encrypt failed, falling back to HTTP" + } + + # HTTP fallback - call SecretumVault service directly + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let sv_token = (get-secretumvault-token) + let mount_point = (get-secretumvault-mount-point) + let url = $"($sv_url)/v1/($mount_point)/encrypt/($key_name)" + + if ($sv_token | is-empty) { + error make { + msg: "SecretumVault authentication failed" + label: { + text: "SECRETUMVAULT_TOKEN not set" + span: (metadata $plaintext).span + } + } + } + + let result = (do -i { + let plaintext_b64 = ($plaintext | encode base64) + let body = {plaintext: $plaintext_b64} + + http post -H ["X-Vault-Token" $sv_token] $url $body + }) + + if $result != null { + return $result + } + + error make { + msg: "SecretumVault encryption failed" + label: { + text: $"Failed to encrypt data with key ($key_name)" + span: (metadata $plaintext).span + } + } +} + +# Decrypt data using SecretumVault plugin +export def plugin-secretumvault-decrypt [ + ciphertext: string + --key-id: string = "" # Encryption key ID +] { + let enabled = is-plugin-enabled + let available = is-plugin-available + let key_name = if ($key_id | is-empty) { get-secretumvault-key-name } else { $key_id } + + if $enabled and $available { + let plugin_result = (try-plugin { + let args = if ($key_id | is-empty) { + [decrypt $ciphertext] + } else { + [decrypt $ciphertext --key-id $key_id] + } + + secretumvault ...$args + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault decrypt failed, falling back to HTTP" + } + + # HTTP fallback - call SecretumVault service directly + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let sv_token = (get-secretumvault-token) + let mount_point = (get-secretumvault-mount-point) + let url = $"($sv_url)/v1/($mount_point)/decrypt/($key_name)" + + if ($sv_token | is-empty) { + error make { + msg: "SecretumVault authentication failed" + label: { + text: "SECRETUMVAULT_TOKEN not set" + span: (metadata $ciphertext).span + } + } + } + + let result = (do -i { + let body = {ciphertext: $ciphertext} + + let response = (http post -H ["X-Vault-Token" $sv_token] $url $body) + + if ($response.data.plaintext? != null) { + { + plaintext: ($response.data.plaintext | decode base64), + key_id: ($response.data.key_id? // $key_name) + } + } else { + $response + } + }) + + if $result != null { + return $result + } + + error make { + msg: "SecretumVault decryption failed" + label: { + text: $"Failed to decrypt data with key ($key_name)" + span: (metadata $ciphertext).span + } + } +} + +# Generate data key using SecretumVault plugin +export def plugin-secretumvault-generate-key [ + --bits: int = 256 # Key size in bits (128, 256, 2048, 4096) + --key-id: string = "" # Encryption key ID +] { + let enabled = is-plugin-enabled + let available = is-plugin-available + let key_name = if ($key_id | is-empty) { get-secretumvault-key-name } else { $key_id } + + if $enabled and $available { + let plugin_result = (try-plugin { + let args = if ($key_id | is-empty) { + [generate-key --bits $bits] + } else { + [generate-key --bits $bits --key-id $key_id] + } + + secretumvault ...$args + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault generate-key failed, falling back to HTTP" + } + + # HTTP fallback + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let sv_token = (get-secretumvault-token) + let mount_point = (get-secretumvault-mount-point) + let url = $"($sv_url)/v1/($mount_point)/datakey/plaintext/($key_name)" + + if ($sv_token | is-empty) { + error make { + msg: "SecretumVault authentication failed" + label: { + text: "SECRETUMVAULT_TOKEN not set" + } + } + } + + let result = (do -i { + let body = {bits: $bits} + http post -H ["X-Vault-Token" $sv_token] $url $body + }) + + if $result != null { + return $result + } + + error make { + msg: "SecretumVault key generation failed" + label: { + text: $"Failed to generate key with ($bits) bits" + } + } +} + +# Check SecretumVault health using plugin +export def plugin-secretumvault-health []: nothing -> record { + let enabled = is-plugin-enabled + let available = is-plugin-available + + if $enabled and $available { + let plugin_result = (try-plugin { + secretumvault health + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault health check failed, falling back to HTTP" + } + + # HTTP fallback + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let url = $"($sv_url)/v1/sys/health" + + let result = (do -i { + http get $url + }) + + if $result != null { + return $result + } + + { + healthy: false + status: "unavailable" + message: "SecretumVault service unreachable" + } +} + +# Get SecretumVault version using plugin +export def plugin-secretumvault-version []: nothing -> string { + let enabled = is-plugin-enabled + let available = is-plugin-available + + if $enabled and $available { + let plugin_result = (try-plugin { + secretumvault version + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault version failed, falling back to HTTP" + } + + # HTTP fallback + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let url = $"($sv_url)/v1/sys/health" + + let result = (do -i { + let response = (http get $url) + $response.version? // "unknown" + }) + + if $result != null { + return $result + } + + "unavailable" +} + +# Rotate encryption key using plugin +export def plugin-secretumvault-rotate-key [ + --key-id: string = "" # Key ID to rotate +] { + let enabled = is-plugin-enabled + let available = is-plugin-available + let key_name = if ($key_id | is-empty) { get-secretumvault-key-name } else { $key_id } + + if $enabled and $available { + let plugin_result = (try-plugin { + let args = if ($key_id | is-empty) { + [rotate-key] + } else { + [rotate-key --key-id $key_id] + } + + secretumvault ...$args + }) + + if $plugin_result != null { + return $plugin_result + } + + print "⚠️ Plugin SecretumVault rotate-key failed, falling back to HTTP" + } + + # HTTP fallback + print "⚠️ Using HTTP fallback (plugin not available)" + + let sv_url = (get-secretumvault-url) + let sv_token = (get-secretumvault-token) + let mount_point = (get-secretumvault-mount-point) + let url = $"($sv_url)/v1/($mount_point)/keys/($key_name)/rotate" + + if ($sv_token | is-empty) { + error make { + msg: "SecretumVault authentication failed" + label: { + text: "SECRETUMVAULT_TOKEN not set" + span: (metadata $key_name).span + } + } + } + + let result = (do -i { + http post -H ["X-Vault-Token" $sv_token] $url + }) + + if $result != null { + return $result + } + + error make { + msg: "SecretumVault key rotation failed" + label: { + text: $"Failed to rotate key ($key_name)" + span: (metadata $key_name).span + } + } +} + +# Get SecretumVault plugin status and configuration +export def plugin-secretumvault-info []: nothing -> record { + let plugin_available = is-plugin-available + let plugin_enabled = is-plugin-enabled + let sv_url = get-secretumvault-url + let mount_point = get-secretumvault-mount-point + let key_name = get-secretumvault-key-name + let has_token = (not (get-secretumvault-token | is-empty)) + + { + plugin_available: $plugin_available + plugin_enabled: $plugin_enabled + service_url: $sv_url + mount_point: $mount_point + default_key: $key_name + authenticated: $has_token + mode: (if ($plugin_enabled and $plugin_available) { "plugin (native)" } else { "http fallback" }) + } +} + +# Encrypt configuration file using SecretumVault +export def encrypt-config-file [ + config_file: string + --output: string = "" # Output file path (default: .enc) + --key-id: string = "" # Encryption key ID +] { + let out_file = if ($output | is-empty) { + $"($config_file).enc" + } else { + $output + } + + let result = (do -i { + let content = (open $config_file --raw) + let encrypted = (plugin-secretumvault-encrypt $content --key-id $key_id) + + # Save encrypted content + if ($encrypted | type) == "record" { + $encrypted.ciphertext | save --force $out_file + } else { + $encrypted | save --force $out_file + } + + print $"βœ… Configuration encrypted to: ($out_file)" + { + success: true + input_file: $config_file + output_file: $out_file + key_id: (if ($key_id | is-empty) { (get-secretumvault-key-name) } else { $key_id }) + } + }) + + if $result == null { + error make { + msg: "Failed to encrypt configuration file" + label: { + text: "Check file permissions and SecretumVault service" + span: (metadata $config_file).span + } + } + } + + $result +} + +# Decrypt configuration file using SecretumVault +export def decrypt-config-file [ + encrypted_file: string + --output: string = "" # Output file path (default: .dec) + --key-id: string = "" # Encryption key ID +] { + let out_file = if ($output | is-empty) { + let base_name = ($encrypted_file | str replace '.enc' '') + $"($base_name).dec" + } else { + $output + } + + let result = (do -i { + let encrypted_content = (open $encrypted_file --raw) + let decrypted = (plugin-secretumvault-decrypt $encrypted_content --key-id $key_id) + + # Save decrypted content + if ($decrypted | type) == "record" { + if ($decrypted.plaintext? != null) { + $decrypted.plaintext | save --force $out_file + } else { + $decrypted | to json | save --force $out_file + } + } else { + $decrypted | save --force $out_file + } + + print $"βœ… Configuration decrypted to: ($out_file)" + { + success: true + input_file: $encrypted_file + output_file: $out_file + key_id: (if ($key_id | is-empty) { (get-secretumvault-key-name) } else { $key_id }) + } + }) + + if $result == null { + error make { + msg: "Failed to decrypt configuration file" + label: { + text: "Check file permissions and SecretumVault service" + span: (metadata $encrypted_file).span + } + } + } + + $result +} diff --git a/nulib/lib_provisioning/plugins_defs.nu b/nulib/lib_provisioning/plugins_defs.nu index dcc976f..b925624 100644 --- a/nulib/lib_provisioning/plugins_defs.nu +++ b/nulib/lib_provisioning/plugins_defs.nu @@ -84,64 +84,33 @@ export def render_template_ai [ ai_generate_template $ai_prompt $template_type } -export def process_kcl_file [ - kcl_file: string +export def process_decl_file [ + decl_file: string format: string - settings?: record ]: nothing -> string { - # Try nu_plugin_kcl first if available - if ( (version).installed_plugins | str contains "kcl" ) { - if $settings != null { - let settings_json = ($settings | to json) - #kcl-run $kcl_file -Y $settings_json - let result = (^kcl run $kcl_file --setting $settings_json --format $format | complete) - if $result.exit_code == 0 { $result.stdout } else { error make { msg: $result.stderr } } + # Use external Nickel CLI (nickel export) + if (get-use-nickel) { + let result = (^nickel export $decl_file --format $format | complete) + if $result.exit_code == 0 { + $result.stdout } else { - let result = (^kcl run $kcl_file --format $format | complete) - if $result.exit_code == 0 { $result.stdout } else { error make { msg: $result.stderr } } + error make { msg: $result.stderr } } } else { - # Use external KCL CLI - if (get-use-kcl) { - if $settings != null { - let settings_json = ($settings | to json) - let result = (^kcl run $kcl_file --setting $settings_json --format $format | complete) - if $result.exit_code == 0 { $result.stdout } else { error make { msg: $result.stderr } } - } else { - let result = (^kcl run $kcl_file --format $format | complete) - if $result.exit_code == 0 { $result.stdout } else { error make { msg: $result.stderr } } - } - } else { - error make { msg: "Neither nu_plugin_kcl nor external KCL CLI available" } - } + error make { msg: "Nickel CLI not available" } } } -export def validate_kcl_schema [ - kcl_file: string +export def validate_decl_schema [ + decl_file: string data: record ]: nothing -> bool { - # Try nu_plugin_kcl first if available - if ( (version).installed_plugins | str contains "nu_plugin_kcl" ) { - kcl validate $kcl_file --data ($data | to json) catch { - # Fallback to external KCL CLI - if (get-use-kcl) { - let data_json = ($data | to json) - let data_json = ($data | to json) - let result = (^kcl validate $kcl_file --data ($data | to json) | complete) - $result.exit_code == 0 - } else { - false - } - } + # Validate using external Nickel CLI + if (get-use-nickel) { + let data_json = ($data | to json) + let result = (^nickel validate $decl_file --data $data_json | complete) + $result.exit_code == 0 } else { - # Use external KCL CLI - if (get-use-kcl) { - let data_json = ($data | to json) - let result = (^kcl validate $kcl_file --data $data_json | complete) - $result.exit_code == 0 - } else { - false - } + false } } diff --git a/nulib/lib_provisioning/project/deployment-pipeline.nu b/nulib/lib_provisioning/project/deployment-pipeline.nu index 07f5fd5..5ce7a0b 100644 --- a/nulib/lib_provisioning/project/deployment-pipeline.nu +++ b/nulib/lib_provisioning/project/deployment-pipeline.nu @@ -268,7 +268,7 @@ export def export-for-ci [ } annotations: ($pipeline_result.completion.gaps | map {|g| { - file: "provisioning/declaration.k" + file: "provisioning/declaration.ncl" level: (if $g.severity == "Error" { "error" } else { "warning" }) message: $g.message title: $g.suggestion @@ -299,4 +299,4 @@ export def export-for-ci [ $pipeline_result } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/project/detect.nu b/nulib/lib_provisioning/project/detect.nu index fa1b4e7..755be19 100644 --- a/nulib/lib_provisioning/project/detect.nu +++ b/nulib/lib_provisioning/project/detect.nu @@ -182,4 +182,4 @@ export def get-required-taskservs [ ($detection.requirements | default []) | where {|r| $r.required == true } | each {|r| $r.taskserv } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/providers.nu b/nulib/lib_provisioning/providers.nu index 160e5ba..0ef034d 100644 --- a/nulib/lib_provisioning/providers.nu +++ b/nulib/lib_provisioning/providers.nu @@ -1,3 +1,3 @@ # Re-export provider middleware to avoid deep relative imports # This centralizes all provider imports in one place -export use ../../../extensions/providers/prov_lib/middleware.nu * \ No newline at end of file +export use ../../../extensions/providers/prov_lib/middleware.nu * diff --git a/nulib/lib_provisioning/providers/interface.nu b/nulib/lib_provisioning/providers/interface.nu index 2578a05..9ef4c9e 100644 --- a/nulib/lib_provisioning/providers/interface.nu +++ b/nulib/lib_provisioning/providers/interface.nu @@ -293,4 +293,4 @@ export def get-interface-version []: nothing -> string { # } # } # } -# ``` \ No newline at end of file +# ``` diff --git a/nulib/lib_provisioning/providers/loader.nu b/nulib/lib_provisioning/providers/loader.nu index a38ec48..b6022a4 100644 --- a/nulib/lib_provisioning/providers/loader.nu +++ b/nulib/lib_provisioning/providers/loader.nu @@ -328,4 +328,4 @@ export def get-loader-stats []: nothing -> record { healthy_providers: ($health_checks | where interface_valid == true | length) last_check: (date now) } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/providers/registry.nu b/nulib/lib_provisioning/providers/registry.nu index 2c74742..196e0ee 100644 --- a/nulib/lib_provisioning/providers/registry.nu +++ b/nulib/lib_provisioning/providers/registry.nu @@ -271,4 +271,4 @@ export def refresh-provider-registry []: nothing -> nothing { # Export environment setup export-env { $env.PROVIDER_REGISTRY_INITIALIZED = false -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/setup/config.nu b/nulib/lib_provisioning/setup/config.nu index c46f27c..289af52 100644 --- a/nulib/lib_provisioning/setup/config.nu +++ b/nulib/lib_provisioning/setup/config.nu @@ -22,53 +22,53 @@ export def install_config [ let use_context = if ($ops | str contains "context") or $context { true } else { false } let provisioning_config_path = $nu.default-config-dir | path dirname | path join $provisioning_cfg_name | path join "nushell" let provisioning_root = if ((get-base-path) | is-not-empty) { - (get-base-path) - } else { + (get-base-path) + } else { let base_path = if ($env.PROCESS_PATH | str contains "provisioning") { - $env.PROCESS_PATH - } else { + $env.PROCESS_PATH + } else { $env.PWD } let parts = ($base_path | split row "provisioning") - $"((if ($parts | is-empty) { "" } else { $parts | first }))provisioning" + $"((if ($parts | is-empty) { "" } else { $parts | first }))provisioning" } let shell_dflt_template = $provisioning_root | path join "templates"| path join "nushell" | path join "default" - if not ($shell_dflt_template | path exists) { + if not ($shell_dflt_template | path exists) { _print $"πŸ›‘ Template path (_ansi red_bold)($shell_dflt_template)(_ansi reset) not found" exit 1 - } + } let context_filename = "default_context.yaml" let context_template = $provisioning_root | path join "templates"| path join $context_filename let provisioning_context_path = ($nu.default-config-dir | path dirname | path join $provisioning_cfg_name | path join $context_filename) let op = if (is-debug-enabled) { "v" } else { "" } - if $reset { - if ($provisioning_context_path | path exists) { + if $reset { + if ($provisioning_context_path | path exists) { rm -rf $provisioning_context_path _print $"Restore context (_ansi default_dimmed) ($provisioning_context_path)(_ansi reset)" } - if not $use_context and ($provisioning_config_path | path exists) { + if not $use_context and ($provisioning_config_path | path exists) { rm -rf $provisioning_config_path _print $"Restore defaults (_ansi default_dimmed) ($provisioning_config_path)(_ansi reset)" } } - if ($provisioning_context_path | path exists) { + if ($provisioning_context_path | path exists) { _print $"Intallation on (_ansi yellow)($provisioning_context_path)(_ansi reset) (_ansi purple_bold)already exists(_ansi reset)" _print $"use (_ansi purple_bold)provisioning context(_ansi reset) to manage context \(create, default, set, etc\)" } else { mkdir ($provisioning_context_path | path dirname) - let data_context = (open -r $context_template) + let data_context = (open -r $context_template) $data_context | str replace "HOME" $nu.home-path | save $provisioning_context_path #$use_context | update infra_path ($context.infra_path | str replace "HOME" $nu.home-path) | save $provisioning_context_path _print $"Intallation on (_ansi yellow)($provisioning_context_path) (_ansi green_bold)completed(_ansi reset)" _print $"use (_ansi purple_bold)provisioning context(_ansi reset) to manage context \(create, default, set, etc\)" } - if ($provisioning_config_path | path exists) { + if ($provisioning_config_path | path exists) { _print $"Intallation on (_ansi yellow)($provisioning_config_path)(_ansi reset) (_ansi purple_bold)already exists(_ansi reset)" - _print ( $"with library path in (_ansi default_dimmed)env.nu(_ansi reset) for: " + + _print ( $"with library path in (_ansi default_dimmed)env.nu(_ansi reset) for: " + $" (_ansi blue)(env_file_providers $"($provisioning_config_path)/env.nu" | str join ' ')(_ansi reset)" ) - } else { - mkdir $provisioning_config_path + } else { + mkdir $provisioning_config_path mut providers_lib_paths = $provisioning_root | path join "providers" mut providers_list = "" for it in (ls $"($provisioning_root)/providers" | get name) { @@ -79,9 +79,9 @@ export def install_config [ if $providers_lib_paths != "" { $providers_lib_paths += "\n " } $providers_lib_paths += ($it | path join "nulib") } - ^cp $"-p($op)r" ...(glob $"($shell_dflt_template)/*") $provisioning_config_path - if ($provisioning_config_path | path join "env.nu" | path exists) { - ( open ($provisioning_config_path | path join "env.nu") -r | + ^cp $"-p($op)r" ...(glob $"($shell_dflt_template)/*") $provisioning_config_path + if ($provisioning_config_path | path join "env.nu" | path exists) { + ( open ($provisioning_config_path | path join "env.nu") -r | str replace "# PROVISIONING_NULIB_DIR" ($provisioning_root | path join "core"| path join "nulib") | str replace "# PROVISIONING_NULIB_PROVIDERS" $providers_lib_paths | save -f $"($provisioning_config_path)/env.nu" @@ -90,4 +90,4 @@ export def install_config [ } _print $"Intallation on (_ansi yellow)($provisioning_config_path) (_ansi green_bold)completed(_ansi reset)" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/setup/detection.nu b/nulib/lib_provisioning/setup/detection.nu index 057b534..8142bfc 100644 --- a/nulib/lib_provisioning/setup/detection.nu +++ b/nulib/lib_provisioning/setup/detection.nu @@ -49,10 +49,10 @@ export def has-ssh []: nothing -> bool { ($ssh_check == 0) } -# Check if KCL is installed -export def has-kcl []: nothing -> bool { - let kcl_check = (bash -c "which kcl > /dev/null 2>&1; echo $?" | str trim | into int) - ($kcl_check == 0) +# Check if Nickel is installed +export def has-nickel []: nothing -> bool { + let decl_check = (bash -c "which nickel > /dev/null 2>&1; echo $?" | str trim | into int) + ($nickel_check == 0) } # Check if SOPS is installed @@ -76,7 +76,7 @@ export def get-deployment-capabilities []: nothing -> record { kubectl_available: (has-kubectl) systemd_available: (has-systemd) ssh_available: (has-ssh) - kcl_available: (has-kcl) + nickel_available: (has-nickel) sops_available: (has-sops) age_available: (has-age) } @@ -246,7 +246,7 @@ export def print-detection-report [ print $" Kubernetes: (if $report.capabilities.kubectl_available { 'βœ…' } else { '❌' })" print $" Systemd: (if $report.capabilities.systemd_available { 'βœ…' } else { '❌' })" print $" SSH: (if $report.capabilities.ssh_available { 'βœ…' } else { '❌' })" - print $" KCL: (if $report.capabilities.kcl_available { 'βœ…' } else { '❌' })" + print $" Nickel: (if $report.capabilities.nickel_available { 'βœ…' } else { '❌' })" print $" SOPS: (if $report.capabilities.sops_available { 'βœ…' } else { '❌' })" print $" Age: (if $report.capabilities.age_available { 'βœ…' } else { '❌' })" print "" @@ -327,8 +327,8 @@ export def get-missing-required-tools [ ]: nothing -> list { mut missing = [] - if not $report.capabilities.kcl_available { - $missing = ($missing | append "kcl") + if not $report.capabilities.nickel_available { + $missing = ($missing | append "nickel") } if not $report.capabilities.sops_available { diff --git a/nulib/lib_provisioning/setup/mod.nu b/nulib/lib_provisioning/setup/mod.nu index c76e187..20f2220 100644 --- a/nulib/lib_provisioning/setup/mod.nu +++ b/nulib/lib_provisioning/setup/mod.nu @@ -355,4 +355,4 @@ export def setup-init []: nothing -> bool { # Get setup module version export def get-setup-version []: nothing -> string { "1.0.0" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/setup/utils.nu b/nulib/lib_provisioning/setup/utils.nu index 31db107..f5ea270 100644 --- a/nulib/lib_provisioning/setup/utils.nu +++ b/nulib/lib_provisioning/setup/utils.nu @@ -7,18 +7,18 @@ export def setup_config_path [ ($nu.default-config-dir) | path dirname | path join $provisioning_cfg_name } export def tools_install [ - tool_name?: string + tool_name?: string run_args?: string ]: nothing -> bool { print $"(_ansi cyan)((get-provisioning-name))(_ansi reset) (_ansi yellow_bold)tools(_ansi reset) check:\n" let bin_install = ((get-base-path) | path join "core" | path join "bin" | path join "tools-install") - if not ($bin_install | path exists) { + if not ($bin_install | path exists) { print $"πŸ›‘ Error running (_ansi yellow)tools_install(_ansi reset) not found (_ansi red_bold)($bin_install | path basename)(_ansi reset)" if (is-debug-enabled) { print $"($bin_install)" } return false - } + } let res = (^$"($bin_install)" $run_args $tool_name | complete) - if ($res.exit_code == 0 ) { + if ($res.exit_code == 0 ) { print $res.stdout true } else { @@ -55,47 +55,103 @@ export def providers_install [ } } export def create_versions_file [ - targetname: string = "versions" + targetname: string = "versions" ]: nothing -> bool { let target_name = if ($targetname | is-empty) { "versions" } else { $targetname } - let providers_path = (get-providers-path) - if ($providers_path | path exists) { - providers_list "full" | each {|prov| - let name = ($prov | get name? | default "") - let prov_versions = ($providers_path | path join $name | path join $target_name ) - mut line = "" - print -n $"\n(_ansi blue)($name)(_ansi reset) => " - for item in ($prov | get tools? | default [] | transpose key value) { - let tool_name = ($item | get key? | default "") - for data in ($item | get value? | default {} | transpose ky val) { - let sub_name = ($data.ky | str upcase) - $line += $"($name | str upcase)_($tool_name | str upcase)_($sub_name)=\"($data | get val? | default "")\"\n" - } - print -n $"(_ansi yellow)($tool_name)(_ansi reset)" - } - $line | save --force $prov_versions - print $"\n(_ansi blue)($name)(_ansi reset) versions file (_ansi green_bold)($target_name)(_ansi reset) generated" - if $env.PROVISIONING_DEBUG { _print $"($prov_versions)" } - } - _print "" - } - if not ($env.PROVISIONING_REQ_VERSIONS | path exists ) { return false } - let versions_source = open $env.PROVISIONING_REQ_VERSIONS - let versions_target = ($env.PROVISIONING_REQ_VERSIONS | path dirname | path join $target_name) - if ( $versions_target | path exists) { rm -f $versions_target } - $versions_source | transpose key value | each {|it| - let name = ($it.key | str upcase) - mut line = "" - for data in ($it.value | transpose ky val) { - let sub_name = ($data.ky | str upcase) - $line += $"($name)_($sub_name)=\"($data.val | default "")\"\n" - } - $line | save -a $versions_target + let provisioning_base = ($env.PROVISIONING? | default (get-base-path)) + let versions_ncl = ($provisioning_base | path join "core" | path join "versions.ncl") + let versions_target = ($provisioning_base | path join "core" | path join $target_name) + let providers_path = ($provisioning_base | path join "extensions" | path join "providers") + + # Check if versions.ncl exists + if not ($versions_ncl | path exists) { + return false } - print ( - $"(_ansi cyan)($env.PROVISIONING_NAME)(_ansi reset) (_ansi blue)core versions(_ansi reset) file " + - $"(_ansi green_bold)($target_name)(_ansi reset) generated" - ) - if $env.PROVISIONING_DEBUG { print ($env.PROVISIONING_REQ_VERSIONS) } - true -} \ No newline at end of file + + # Generate KEY="VALUE" format + mut content = "" + + # ============================================================================ + # CORE TOOLS + # ============================================================================ + let nickel_result = (^nickel export $versions_ncl --format json | complete) + + if $nickel_result.exit_code == 0 { + let json_data = ($nickel_result.stdout | from json) + let core_versions = ($json_data | get core_versions? | default []) + + for item in $core_versions { + let name = ($item | get name?) + let version_obj = ($item | get version?) + + if ($name | is-not-empty) and ($version_obj | is-not-empty) { + let key = ($name | str upcase) + let current = ($version_obj | get current?) + let source = ($version_obj | get source?) + + $content += $"($key)_VERSION=\"($current)\"\n" + $content += $"($key)_SOURCE=\"($source)\"\n" + + # Add short aliases for common bash scripts (e.g., nushell -> NU) + let short_key = if $name == "nushell" { + "NU" + } else if $name == "nickel" { + "NICKEL" + } else if $name == "sops" { + "SOPS" + } else if $name == "age" { + "AGE" + } else if $name == "k9s" { + "K9S" + } else { + "" + } + + if ($short_key | is-not-empty) and ($short_key != $key) { + $content += $"($short_key)_VERSION=\"($current)\"\n" + $content += $"($short_key)_SOURCE=\"($source)\"\n" + } + + $content += "\n" + } + } + } + + # ============================================================================ + # PROVIDERS + # ============================================================================ + if ($providers_path | path exists) { + for provider_item in (ls $providers_path) { + let provider_dir = ($providers_path | path join $provider_item.name) + let provider_version_file = ($provider_dir | path join "nickel" | path join "version.ncl") + + if ($provider_version_file | path exists) { + let provider_result = (^nickel export $provider_version_file --format json | complete) + + if $provider_result.exit_code == 0 { + let provider_data = ($provider_result.stdout | from json) + let prov_name = ($provider_data | get name?) + let prov_version_obj = ($provider_data | get version?) + + if ($prov_name | is-not-empty) and ($prov_version_obj | is-not-empty) { + let prov_key = $"PROVIDER_($prov_name | str upcase)" + let prov_current = ($prov_version_obj | get current?) + let prov_source = ($prov_version_obj | get source?) + + $content += $"($prov_key)_VERSION=\"($prov_current)\"\n" + $content += $"($prov_key)_SOURCE=\"($prov_source)\"\n" + $content += "\n" + } + } + } + } + } + + # Save to file + if ($content | is-not-empty) { + $content | save --force $versions_target + true + } else { + false + } +} diff --git a/nulib/lib_provisioning/setup/validation.nu b/nulib/lib_provisioning/setup/validation.nu index f1ac229..a521ae0 100644 --- a/nulib/lib_provisioning/setup/validation.nu +++ b/nulib/lib_provisioning/setup/validation.nu @@ -44,7 +44,7 @@ export def validate-workspace-path [ let workspace_exists = ($workspace_path | path exists) let is_dir = (if $workspace_exists { ($workspace_path | path type) == "dir" } else { false }) - let has_config_file = ($"($workspace_path)/config/provisioning.k" | path exists) + let has_config_file = ($"($workspace_path)/config/provisioning.ncl" | path exists) let is_valid = ($workspace_exists and ($missing_dirs | length) == 0) { @@ -412,7 +412,7 @@ export def validate-requirements [ missing_tools: $missing_tools internet_available: $detection_report.network.internet_connected recommended_tools: [ - "kcl", + "nickel", "sops", "age", "docker" # or kubernetes or ssh diff --git a/nulib/lib_provisioning/setup/wizard.nu b/nulib/lib_provisioning/setup/wizard.nu index a19a2bf..fa4bf38 100644 --- a/nulib/lib_provisioning/setup/wizard.nu +++ b/nulib/lib_provisioning/setup/wizard.nu @@ -5,14 +5,17 @@ # name = "setup wizard" # group = "configuration" # tags = ["setup", "interactive", "wizard"] -# version = "2.0.0" -# requires = ["forminquire.nu:1.0.0", "nushell:0.109.0"] -# note = "Migrated to FormInquire with fallback to prompt-* functions" +# version = "3.0.0" +# requires = ["nushell:0.109.0"] +# note = "MIGRATION: ForminQuire (Jinja2 templates) archived. Use TypeDialog forms instead (typedialog, typedialog-tui, typedialog-web)" +# migration = "See: provisioning/.coder/archive/forminquire/ (deprecated) β†’ provisioning/.typedialog/provisioning/form.toml (new)" use ./mod.nu * use ./detection.nu * use ./validation.nu * -use ../../forminquire/nulib/forminquire.nu * +# ARCHIVED: use ../../forminquire/nulib/forminquire.nu * +# ForminQuire has been archived to: provisioning/.coder/archive/forminquire/ +# New solution: Use TypeDialog for interactive forms (installed automatically by bootstrap) # ============================================================================ # INPUT HELPERS diff --git a/nulib/lib_provisioning/sops/lib.nu b/nulib/lib_provisioning/sops/lib.nu index 0f8de48..11d4e76 100644 --- a/nulib/lib_provisioning/sops/lib.nu +++ b/nulib/lib_provisioning/sops/lib.nu @@ -157,7 +157,7 @@ export def generate_sops_settings [ ]: nothing -> nothing { _print "" # [ -z "$ORG_MAIN_SETTINGS_FILE" ] && return - # [ -r "$PROVIISONING_KEYS_PATH" ] && [ -n "$PROVIISONING_USE_KCL" ] && _on_sops_item "$mode" "$PROVIISONING_KEYS_PATH" "$target" + # [ -r "$PROVIISONING_KEYS_PATH" ] && [ -n "$PROVIISONING_USE_nickel" ] && _on_sops_item "$mode" "$PROVIISONING_KEYS_PATH" "$target" # file=$($YQ -er < "$ORG_MAIN_SETTINGS_FILE" ".defaults_path" | sed 's/null//g') # [ -n "$file" ] && _on_sops_item "$mode" "$file" "$target" # _on_sops_item "$mode" "$ORG_MAIN_SETTINGS_FILE" "$target" diff --git a/nulib/lib_provisioning/tera_daemon.nu b/nulib/lib_provisioning/tera_daemon.nu new file mode 100644 index 0000000..18a6cb5 --- /dev/null +++ b/nulib/lib_provisioning/tera_daemon.nu @@ -0,0 +1,203 @@ +#! Template rendering daemon functions +#! +#! Provides high-performance Jinja2 template rendering via HTTP API. +#! The CLI daemon's Tera engine offers 50-100x better performance than +#! spawning a new Nushell process for each template render. +#! +#! Performance: +#! - Single render: ~4-10ms (vs ~500ms with Nushell spawning) +#! - Batch 10 renders: ~50-60ms (vs ~5500ms) +#! - Batch 100 renders: ~600-700ms (vs ~55000ms) + +use ../env.nu [get-cli-daemon-url] + +# Render a Jinja2 template with the given context +# +# Uses the CLI daemon's Tera engine for fast in-process template rendering. +# This is significantly faster than spawning a new Nushell process. +# +# # Arguments +# * `template` - Template content (Jinja2 syntax) +# * `context` - Context record with template variables +# * `--name` - Optional template name for error reporting +# +# # Returns +# Rendered template content or error if rendering failed +# +# # Example +# ```nushell +# let template = "Hello {{ name }}!" +# let context = {name: "World"} +# tera-render-daemon $template $context --name greeting +# # Output: Hello World! +# ``` +export def tera-render-daemon [ + template: string + context: record + --name: string = "template" +] -> string { + let daemon_url = (get-cli-daemon-url) + + # Convert context record to JSON object + let context_json = ($context | to json | from json) + + # Build request + let request = { + template: $template + context: $context_json + name: $name + } + + # Send to daemon's Tera endpoint + let response = ( + http post $"($daemon_url)/tera/render" $request + --raw + ) + + # Parse response + let parsed = ($response | from json) + + # Check for error + if ($parsed.error? != null) { + error make {msg: $parsed.error} + } + + # Return rendered output + $parsed.rendered +} + +# Get template rendering statistics from daemon +# +# Returns statistics about template renders since daemon startup or last reset. +# +# # Returns +# Record with: +# - `total_renders`: Total number of templates rendered +# - `total_errors`: Number of rendering errors +# - `total_time_ms`: Total time spent rendering (milliseconds) +# - `avg_time_ms`: Average time per render +export def tera-daemon-stats [] -> record { + let daemon_url = (get-cli-daemon-url) + + let response = (http get $"($daemon_url)/tera/stats") + + $response | from json +} + +# Reset template rendering statistics on daemon +# +# Clears all counters and timing statistics. +export def tera-daemon-reset-stats [] -> void { + let daemon_url = (get-cli-daemon-url) + + http post $"($daemon_url)/tera/stats/reset" "" +} + +# Check if CLI daemon is running and Tera rendering is available +# +# # Returns +# `true` if daemon is running with Tera support, `false` otherwise +export def is-tera-daemon-available [] -> bool { + try { + let daemon_url = (get-cli-daemon-url) + let response = (http get $"($daemon_url)/info" --timeout 500ms) + + # Check if tera-rendering is in features list + ($response | from json | .features | str contains "tera-rendering") + } catch { + false + } +} + +# Start using Tera daemon for rendering (if available) +# +# This function checks if the daemon is running and prints a status message. +# It's useful for diagnostics. +export def ensure-tera-daemon [] -> void { + if (is-tera-daemon-available) { + print "βœ… Tera daemon is available and running" + } else { + print "⚠️ Tera daemon is not available" + print " CLI daemon may not be running at http://localhost:9091" + } +} + +# Render multiple templates in batch mode +# +# Renders a list of templates sequentially. This is faster than calling +# tera-render-daemon multiple times due to daemon connection reuse. +# +# # Arguments +# * `templates` - List of records with `template` and `context` fields +# +# # Returns +# List of rendered outputs or error messages +# +# # Example +# ```nushell +# let templates = [ +# {template: "Hello {{ name }}", context: {name: "Alice"}} +# {template: "Goodbye {{ name }}", context: {name: "Bob"}} +# ] +# tera-render-batch $templates +# # Output: [Hello Alice, Goodbye Bob] +# ``` +export def tera-render-batch [ + templates: list +] -> list { + let results = [] + + for template_def in $templates { + let rendered = ( + tera-render-daemon + $template_def.template + $template_def.context + --name ($template_def.name? | default "batch") + ) + $results | append $rendered + } + + $results +} + +# Profile template rendering performance +# +# Renders a template multiple times and reports timing statistics. +# Useful for benchmarking and performance optimization. +# +# # Arguments +# * `template` - Template to render +# * `context` - Context for rendering +# * `--iterations` - Number of times to render (default: 10) +# * `--name` - Template name for reporting +# +# # Returns +# Record with performance metrics +export def tera-profile [ + template: string + context: record + --iterations: int = 10 + --name: string = "profiled" +] -> record { + let start = (date now) + + # Reset stats before profiling + tera-daemon-reset-stats + + # Run renders + for i in 0..<$iterations { + tera-render-daemon $template $context --name $"($name)_$i" + } + + let elapsed = ((date now) - $start) | into duration | get total | . / 1_000_000 + let stats = (tera-daemon-stats) + + { + iterations: $iterations + total_time_ms: $elapsed + avg_time_ms: ($elapsed / $iterations) + daemon_renders: $stats.total_renders + daemon_avg_time_ms: $stats.avg_time_ms + daemon_errors: $stats.total_errors + } +} diff --git a/nulib/lib_provisioning/user/config.nu b/nulib/lib_provisioning/user/config.nu index 9621d70..6d486c0 100644 --- a/nulib/lib_provisioning/user/config.nu +++ b/nulib/lib_provisioning/user/config.nu @@ -361,7 +361,27 @@ export def get-workspace-default-infra [workspace_name: string] { return null } - $workspace.default_infra? | default null + # First check user config for default_infra + let user_infra = ($workspace.default_infra? | default null) + if ($user_infra | is-not-empty) { + return $user_infra + } + + # Fallback: check workspace's provisioning.ncl for current_infra + let ws_path = (get-workspace-path $workspace_name) + let ws_config_file = ([$ws_path "config" "provisioning.ncl"] | path join) + if ($ws_config_file | path exists) { + let result = (do -i { + let ws_config = (^nickel export $ws_config_file --format json | from json) + let current_infra = ($ws_config.workspace_config.workspace.current_infra? | default null) + $current_infra + }) + if ($result | is-not-empty) { + return $result + } + } + + null } # Set default infrastructure for workspace diff --git a/nulib/lib_provisioning/utils/clean.nu b/nulib/lib_provisioning/utils/clean.nu index 6ef6550..e7df686 100644 --- a/nulib/lib_provisioning/utils/clean.nu +++ b/nulib/lib_provisioning/utils/clean.nu @@ -5,10 +5,10 @@ export def cleanup [ ]: nothing -> nothing { if not (is-debug-enabled) and ($wk_path | path exists) { rm --force --recursive $wk_path - } else { + } else { #use utils/interface.nu _ansi _print $"(_ansi default_dimmed)______________________(_ansi reset)" _print $"(_ansi default_dimmed)Work files not removed" _print $"(_ansi default_dimmed)wk_path:(_ansi reset) ($wk_path)" } -} +} diff --git a/nulib/lib_provisioning/utils/config.nu b/nulib/lib_provisioning/utils/config.nu index 4740772..ab8b936 100644 --- a/nulib/lib_provisioning/utils/config.nu +++ b/nulib/lib_provisioning/utils/config.nu @@ -120,4 +120,4 @@ export def save-config [ print $"βœ… Configuration saved to: ($config_path)" true } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/error.nu b/nulib/lib_provisioning/utils/error.nu index 58bbc2e..d1145c7 100644 --- a/nulib/lib_provisioning/utils/error.nu +++ b/nulib/lib_provisioning/utils/error.nu @@ -12,20 +12,20 @@ export def throw-error [ let error = $"\n(_ansi red_bold)($error)(_ansi reset)" let msg = ($text | default "this caused an internal error") let suggestion = if ($suggestion | is-not-empty) { $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" } else { "" } - + # Log error for debugging if (is-debug-enabled) { print $"DEBUG: Error occurred at: (date now | format date '%Y-%m-%d %H:%M:%S')" print $"DEBUG: Context: ($context | default 'no context')" print $"DEBUG: Error code: ($code)" } - + if ($env.PROVISIONING_OUT | is-empty) { if $span == null and $context == null { error make --unspanned { msg: ( $error + "\n" + $msg + $suggestion) } - } else if $span != null and (is-metadata-enabled) { + } else if $span != null and (is-metadata-enabled) { error make { - msg: $error + msg: $error label: { text: $"($msg) (_ansi blue)($context)(_ansi reset)($suggestion)" span: $span @@ -34,8 +34,8 @@ export def throw-error [ } else { error make --unspanned { msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } - } else { - _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + } else { + _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } @@ -61,20 +61,19 @@ export def safe-execute [ export def try [ settings_data: record - defaults_data: record + defaults_data: record ]: nothing -> nothing { - $settings_data.servers | each { |server| + $settings_data.servers | each { |server| _print ( $defaults_data.defaults | merge $server ) } _print ($settings_data.servers | get hostname) _print ($settings_data.servers | get 0).tasks let zli_cfg = (open "resources/oci-reg/zli-cfg" | from json) - if $zli_cfg.sops? != null { + if $zli_cfg.sops? != null { _print "Found" } else { _print "NOT Found" } let pos = 0 _print ($settings_data.servers | get $pos ) -} - +} diff --git a/nulib/lib_provisioning/utils/error_clean.nu b/nulib/lib_provisioning/utils/error_clean.nu index 15292f9..8bf289d 100644 --- a/nulib/lib_provisioning/utils/error_clean.nu +++ b/nulib/lib_provisioning/utils/error_clean.nu @@ -10,37 +10,37 @@ export def throw-error [ ]: nothing -> nothing { let error = $"\n(_ansi red_bold)($error)(_ansi reset)" let msg = ($text | default "this caused an internal error") - let suggestion = if ($suggestion | is-not-empty) { - $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" - } else { - "" + let suggestion = if ($suggestion | is-not-empty) { + $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" + } else { + "" } - + # Log error for debugging if (is-debug-enabled) { print $"DEBUG: Error occurred at: (date now | format date '%Y-%m-%d %H:%M:%S')" print $"DEBUG: Context: ($context | default 'no context')" print $"DEBUG: Error code: ($code)" } - + if ($env.PROVISIONING_OUT | is-empty) { if $span == null and $context == null { error make --unspanned { msg: ( $error + "\n" + $msg + $suggestion) } - } else if $span != null and (is-metadata-enabled) { + } else if $span != null and (is-metadata-enabled) { error make { - msg: $error + msg: $error label: { text: $"($msg) (_ansi blue)($context)(_ansi reset)($suggestion)" span: $span } } } else { - error make --unspanned { - msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + error make --unspanned { + msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } - } else { - _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + } else { + _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } @@ -68,19 +68,19 @@ export def safe-execute [ export def try [ settings_data: record - defaults_data: record + defaults_data: record ]: nothing -> nothing { - $settings_data.servers | each { |server| + $settings_data.servers | each { |server| _print ( $defaults_data.defaults | merge $server ) } _print ($settings_data.servers | get hostname) _print ($settings_data.servers | get 0).tasks let zli_cfg = (open "resources/oci-reg/zli-cfg" | from json) - if $zli_cfg.sops? != null { + if $zli_cfg.sops? != null { _print "Found" } else { _print "NOT Found" } let pos = 0 _print ($settings_data.servers | get $pos ) -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/error_final.nu b/nulib/lib_provisioning/utils/error_final.nu index b522da5..8be434f 100644 --- a/nulib/lib_provisioning/utils/error_final.nu +++ b/nulib/lib_provisioning/utils/error_final.nu @@ -10,36 +10,36 @@ export def throw-error [ ]: nothing -> nothing { let error = $"\n(_ansi red_bold)($error)(_ansi reset)" let msg = ($text | default "this caused an internal error") - let suggestion = if ($suggestion | is-not-empty) { - $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" - } else { - "" + let suggestion = if ($suggestion | is-not-empty) { + $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" + } else { + "" } - + if (is-debug-enabled) { print $"DEBUG: Error occurred at: (date now | format date '%Y-%m-%d %H:%M:%S')" print $"DEBUG: Context: ($context | default 'no context')" print $"DEBUG: Error code: ($code)" } - + if ($env.PROVISIONING_OUT | is-empty) { if $span == null and $context == null { error make --unspanned { msg: ( $error + "\n" + $msg + $suggestion) } - } else if $span != null and (is-metadata-enabled) { + } else if $span != null and (is-metadata-enabled) { error make { - msg: $error + msg: $error label: { text: $"($msg) (_ansi blue)($context)(_ansi reset)($suggestion)" span: $span } } } else { - error make --unspanned { - msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + error make --unspanned { + msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } - } else { - _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + } else { + _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } @@ -67,19 +67,19 @@ export def safe-execute [ export def try [ settings_data: record - defaults_data: record + defaults_data: record ]: nothing -> nothing { - $settings_data.servers | each { |server| + $settings_data.servers | each { |server| _print ( $defaults_data.defaults | merge $server ) } _print ($settings_data.servers | get hostname) _print ($settings_data.servers | get 0).tasks let zli_cfg = (open "resources/oci-reg/zli-cfg" | from json) - if $zli_cfg.sops? != null { + if $zli_cfg.sops? != null { _print "Found" } else { _print "NOT Found" } let pos = 0 _print ($settings_data.servers | get $pos ) -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/error_fixed.nu b/nulib/lib_provisioning/utils/error_fixed.nu index 15292f9..8bf289d 100644 --- a/nulib/lib_provisioning/utils/error_fixed.nu +++ b/nulib/lib_provisioning/utils/error_fixed.nu @@ -10,37 +10,37 @@ export def throw-error [ ]: nothing -> nothing { let error = $"\n(_ansi red_bold)($error)(_ansi reset)" let msg = ($text | default "this caused an internal error") - let suggestion = if ($suggestion | is-not-empty) { - $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" - } else { - "" + let suggestion = if ($suggestion | is-not-empty) { + $"\nπŸ’‘ Suggestion: (_ansi yellow)($suggestion)(_ansi reset)" + } else { + "" } - + # Log error for debugging if (is-debug-enabled) { print $"DEBUG: Error occurred at: (date now | format date '%Y-%m-%d %H:%M:%S')" print $"DEBUG: Context: ($context | default 'no context')" print $"DEBUG: Error code: ($code)" } - + if ($env.PROVISIONING_OUT | is-empty) { if $span == null and $context == null { error make --unspanned { msg: ( $error + "\n" + $msg + $suggestion) } - } else if $span != null and (is-metadata-enabled) { + } else if $span != null and (is-metadata-enabled) { error make { - msg: $error + msg: $error label: { text: $"($msg) (_ansi blue)($context)(_ansi reset)($suggestion)" span: $span } } } else { - error make --unspanned { - msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + error make --unspanned { + msg: ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } - } else { - _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") + } else { + _print ( $error + "\n" + $msg + "\n" + $"(_ansi blue)($context | default "" )(_ansi reset)($suggestion)") } } @@ -68,19 +68,19 @@ export def safe-execute [ export def try [ settings_data: record - defaults_data: record + defaults_data: record ]: nothing -> nothing { - $settings_data.servers | each { |server| + $settings_data.servers | each { |server| _print ( $defaults_data.defaults | merge $server ) } _print ($settings_data.servers | get hostname) _print ($settings_data.servers | get 0).tasks let zli_cfg = (open "resources/oci-reg/zli-cfg" | from json) - if $zli_cfg.sops? != null { + if $zli_cfg.sops? != null { _print "Found" } else { _print "NOT Found" } let pos = 0 _print ($settings_data.servers | get $pos ) -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/files.nu b/nulib/lib_provisioning/utils/files.nu index 7ce1a35..c9e7986 100644 --- a/nulib/lib_provisioning/utils/files.nu +++ b/nulib/lib_provisioning/utils/files.nu @@ -111,4 +111,4 @@ export def select_file_list [ show_clip_to $"($file_selection.name)" true } $file_selection -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/format.nu b/nulib/lib_provisioning/utils/format.nu index f9809ca..538fa3a 100644 --- a/nulib/lib_provisioning/utils/format.nu +++ b/nulib/lib_provisioning/utils/format.nu @@ -50,4 +50,4 @@ export def money_conversion [ 0 } } else { 0 } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/generate.nu b/nulib/lib_provisioning/utils/generate.nu index 9ad569b..0008364 100644 --- a/nulib/lib_provisioning/utils/generate.nu +++ b/nulib/lib_provisioning/utils/generate.nu @@ -10,7 +10,7 @@ use ../config/accessor.nu * export def github_latest_tag [ url: string = "" use_dev_release: bool = false - id_target: string = "releases/tag" + id_target: string = "releases/tag" ]: nothing -> string { #let res = (http get $url -r ) if ($url | is-empty) { return "" } @@ -19,16 +19,16 @@ export def github_latest_tag [ print $"πŸ›‘ Error (_ansi red)($url)(_ansi reset):\n ($res.exit_code) ($res.stderr)" return "" } else { $res.stdout } - # curl -s https://github.com/project-zot/zot/tags | grep "

.*?)' | get a | each {|it| let parsed = ($it | parse --regex ($"($id_target)" + '/(?.*?)"')) if ($parsed | is-empty) { "" } else { $parsed | get version | first } }) - let list = if $use_dev_release { + let list = if $use_dev_release { $versions - } else { - ($versions | where {|it| - not ($it | str contains "-rc") and not ($it | str contains "-alpha") + } else { + ($versions | where {|it| + not ($it | str contains "-rc") and not ($it | str contains "-alpha") }) } if ($list | is-empty) { "" } else { $list | sort -r | first } @@ -41,7 +41,7 @@ export def value_input_list [ default_value: string ]: nothing -> string { let selection_pos = ( $options_list - | input list --index ( + | input list --index ( $"(_ansi default_dimmed)Select(_ansi reset) (_ansi yellow_bold)($msg)(_ansi reset) " + $"\n(_ansi default_dimmed)\(use arrow keys and press [enter] or [escape] for default '(_ansi reset)" + $"($default_value)(_ansi default_dimmed)'\)(_ansi reset)" @@ -53,30 +53,30 @@ export def value_input_list [ export def value_input [ input_type: string - numchar: int + numchar: int msg: string default_value: string not_empty: bool ]: nothing -> string { while true { let value_input = if $numchar > 0 { - print ($"(_ansi yellow_bold)($msg)(_ansi reset) " + - $"(_ansi default_dimmed) type value (_ansi green_bold)($numchar) chars(_ansi reset) " + + print ($"(_ansi yellow_bold)($msg)(_ansi reset) " + + $"(_ansi default_dimmed) type value (_ansi green_bold)($numchar) chars(_ansi reset) " + $"(_ansi default_dimmed) default '(_ansi reset)" + $"($default_value)(_ansi default_dimmed)'(_ansi reset)" ) (input --numchar $numchar) - } else { + } else { print ($"(_ansi yellow_bold)($msg)(_ansi reset) " + $"(_ansi default_dimmed)\(type value and press [enter] default '(_ansi reset)" + $"($default_value)(_ansi default_dimmed)'\)(_ansi reset)" ) (input) } - if $not_empty and ($value_input | is-empty) { + if $not_empty and ($value_input | is-empty) { if ($default_value | is-not-empty) { return $default_value } continue - } else if ($value_input | is-empty) { + } else if ($value_input | is-empty) { return $default_value } let result = match $input_type { @@ -134,9 +134,9 @@ export def "generate_data_items" [ mut val = [] while true { let selection_pos = ( [ $"Add ($msg)", $"No more ($var)" ] - | input list --index ( + | input list --index ( $"(_ansi default_dimmed)Select(_ansi reset) (_ansi yellow_bold)($msg)(_ansi reset) " + - $"\n(_ansi default_dimmed)\(use arrow keys and press [enter] or [escape] to finish '(_ansi reset)" + $"\n(_ansi default_dimmed)\(use arrow keys and press [enter] or [escape] to finish '(_ansi reset)" )) if $selection_pos == null or $selection_pos == 1 { break } $val = ($val | append (generate_data_items $defs_gen $record_value)) @@ -157,21 +157,21 @@ export def "generate_data_def" [ infra_path: string created: bool inputfile: string = "" -]: nothing -> nothing { +]: nothing -> nothing { let data = (if ($inputfile | is-empty) { - let defs_path = ($root_path | path join (get-provisioning-generate-dirpath) | path join (get-provisioning-generate-defsfile)) - if ( $defs_path | path exists) { + let defs_path = ($root_path | path join (get-provisioning-generate-dirpath) | path join (get-provisioning-generate-defsfile)) + if ( $defs_path | path exists) { let data_gen = (open $defs_path) let title = $"($data_gen| get title? | default "")" generate_title $title let defs_values = ($data_gen | get defs_values? | default []) (generate_data_items $data_gen $defs_values) - } else { + } else { if (is-debug-enabled) { _print $"πŸ›‘ ((get-provisioning-name)) generate: Invalid path (_ansi red)($defs_path)(_ansi reset)" } } } else { (open $inputfile) - } | merge { + } | merge { infra_name: $infra_name, infra_path: $infra_path, }) @@ -179,7 +179,7 @@ export def "generate_data_def" [ ($data | to yaml | str replace "$name" $infra_name| save -f $vars_filepath) let remove_files = if (is-debug-enabled) { false } else { true } on_template_path $infra_path $vars_filepath $remove_files true - if not (is-debug-enabled) { + if not (is-debug-enabled) { rm -f $vars_filepath } } diff --git a/nulib/lib_provisioning/utils/git-commit-msg.nu b/nulib/lib_provisioning/utils/git-commit-msg.nu index a1b7863..3b2334d 100644 --- a/nulib/lib_provisioning/utils/git-commit-msg.nu +++ b/nulib/lib_provisioning/utils/git-commit-msg.nu @@ -147,4 +147,4 @@ export def "show-commit-changes" []: nothing -> table { code: $status_code } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/help.nu b/nulib/lib_provisioning/utils/help.nu index af0ad0e..a6b0794 100644 --- a/nulib/lib_provisioning/utils/help.nu +++ b/nulib/lib_provisioning/utils/help.nu @@ -9,17 +9,17 @@ export def parse_help_command [ ] { #use utils/interface.nu end_run let args = ($env.PROVISIONING_ARGS? | default "") - let has_help = if ($args | str contains "help") or ($args |str ends-with " h") { + let has_help = if ($args | str contains "help") or ($args |str ends-with " h") { true - } else if $name != null and $name == "help" or $name == "h" { + } else if $name != null and $name == "help" or $name == "h" { true } else { false } - if not $has_help { return } + if not $has_help { return } let mod_str = if $ismod { "-mod" } else { "" } ^(get-provisioning-name) $mod_str ...($source | split row " ") --help if $task != null { do $task } - if $end { - if not (is-debug-enabled) { end_run "" } + if $end { + if not (is-debug-enabled) { end_run "" } exit - } + } } diff --git a/nulib/lib_provisioning/utils/hints.nu b/nulib/lib_provisioning/utils/hints.nu index 977c084..60577c9 100644 --- a/nulib/lib_provisioning/utils/hints.nu +++ b/nulib/lib_provisioning/utils/hints.nu @@ -275,4 +275,4 @@ export def show-tip [ } else { print $"\n(_ansi yellow_bold)πŸ’‘ Tip:(_ansi reset) ($message)\n" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/imports.nu b/nulib/lib_provisioning/utils/imports.nu index a6ae31f..e1e79ce 100644 --- a/nulib/lib_provisioning/utils/imports.nu +++ b/nulib/lib_provisioning/utils/imports.nu @@ -70,4 +70,4 @@ export def lib-ai []: nothing -> string { # Helper for dynamic imports with specific files export def import-path [base: string, file: string]: nothing -> string { $base | path join $file -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/init.nu b/nulib/lib_provisioning/utils/init.nu index b939b72..91e07f4 100644 --- a/nulib/lib_provisioning/utils/init.nu +++ b/nulib/lib_provisioning/utils/init.nu @@ -13,9 +13,10 @@ export def show_titles []: nothing -> nothing { export def use_titles [ ]: nothing -> bool { if ($env.PROVISIONING_NO_TITLES? | default false) { return false } if ($env.PROVISIONING_NO_TERMINAL? | default false) { return false } - if ($env.PROVISIONING_ARGS? | str contains "-h" ) { return false } - if ($env.PROVISIONING_ARGS? | str contains "--notitles" ) { return false } - if ($env.PROVISIONING_ARGS? | str contains "query") and ($env.PROVISIONING_ARGS? | str contains "-o" ) { return false } + let args = ($env.PROVISIONING_ARGS? | default "") + if ($args | is-not-empty) and ($args | str contains "-h" ) { return false } + if ($args | is-not-empty) and ($args | str contains "--notitles" ) { return false } + if ($args | is-not-empty) and ($args | str contains "query") and ($args | str contains "-o" ) { return false } true } export def provisioning_init [ @@ -52,4 +53,4 @@ export def provisioning_init [ } exit 0 } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/logging.nu b/nulib/lib_provisioning/utils/logging.nu index df77561..954b8d5 100644 --- a/nulib/lib_provisioning/utils/logging.nu +++ b/nulib/lib_provisioning/utils/logging.nu @@ -92,4 +92,4 @@ export def log-subsection [ ] { let context_str = if ($context | is-not-empty) { $" [($context)]" } else { "" } print $" πŸ“Œ ($context_str) ($title)" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/mod.nu b/nulib/lib_provisioning/utils/mod.nu index 46a9fa9..3c311a9 100644 --- a/nulib/lib_provisioning/utils/mod.nu +++ b/nulib/lib_provisioning/utils/mod.nu @@ -1,23 +1,23 @@ # Exclude minor or specific parts for global 'export use' -export use interface.nu * -export use clean.nu * -export use error.nu * -export use help.nu * +export use interface.nu * +export use clean.nu * +export use error.nu * +export use help.nu * export use init.nu * export use generate.nu * -export use undefined.nu * +export use undefined.nu * - export use qr.nu * - export use ssh.nu * + export use qr.nu * + export use ssh.nu * - export use settings.nu * - export use templates.nu * + export use settings.nu * + export use templates.nu * # export use test.nu - export use format.nu * - export use files.nu * + export use format.nu * + export use files.nu * export use on_select.nu * export use imports.nu * diff --git a/nulib/lib_provisioning/utils/on_select.nu b/nulib/lib_provisioning/utils/on_select.nu index 2388dc1..2743bd6 100644 --- a/nulib/lib_provisioning/utils/on_select.nu +++ b/nulib/lib_provisioning/utils/on_select.nu @@ -1,65 +1,65 @@ export def run_on_selection [ - select: string + select: string name: string item_path: string main_path: string - root_path: string + root_path: string ]: nothing -> nothing { - if not ($item_path | path exists) { return } - match $select { + if not ($item_path | path exists) { return } + match $select { "edit" | "editor" | "ed" | "e" => { let cmd = ($env | get EDITOR? | default "vi") let full_cmd = $"($cmd) ($main_path)" - ^($cmd) $main_path + ^($cmd) $main_path show_clip_to $full_cmd true }, "view" | "vw" | "v" => { let cmd = ($env | get PROVISIONING_FILEVIEWER? | default (if (^bash -c "type -P bat" | is-not-empty) { "bat" } else { "cat" })) let full_cmd = $"($cmd) ($main_path)" - ^($cmd) $main_path + ^($cmd) $main_path show_clip_to $full_cmd true }, - "list" | "ls" | "l" => { + "list" | "ls" | "l" => { let full_cmd = $"ls -l ($item_path)" - print (ls $item_path | each {|it| { + print (ls $item_path | each {|it| { name: ($it.name | str replace $root_path ""), type: $it.type, size: $it.size, modified: $it.modified - }}) + }}) show_clip_to $full_cmd true }, - "tree" | "tr" | "t" => { + "tree" | "tr" | "t" => { let full_cmd = $"tree -L 3 ($item_path)" ^tree -L 3 $item_path show_clip_to $full_cmd true }, - "code" | "c" => { + "code" | "c" => { let full_cmd = $"code ($item_path)" ^code $item_path show_clip_to $full_cmd true }, - "shell" | "sh" | "s" => { - let full_cmd = $"($env.SHELL) -c " + $"cd ($item_path) ; ($env.SHELL)" + "shell" | "sh" | "s" => { + let full_cmd = $"($env.SHELL) -c " + $"cd ($item_path) ; ($env.SHELL)" print $"(_ansi default_dimmed)Use [ctrl-d] or 'exit' to end with(_ansi reset) ($env.SHELL)" - ^($env.SHELL) -c $"cd ($item_path) ; ($env.SHELL)" + ^($env.SHELL) -c $"cd ($item_path) ; ($env.SHELL)" show_titles - _print "Command " + _print "Command " (show_clip_to $full_cmd false) }, - "nu"| "n" => { - let full_cmd = $"($env.NU) -i -e " + $"cd ($item_path)" + "nu"| "n" => { + let full_cmd = $"($env.NU) -i -e " + $"cd ($item_path)" _print $"(_ansi default_dimmed)Use [ctrl-d] or 'exit' to end with(_ansi reset) nushell\n" - ^($env.NU) -i -e $"cd ($item_path)" + ^($env.NU) -i -e $"cd ($item_path)" show_titles _print "Command " (show_clip_to $full_cmd false) }, - "" => { + "" => { _print $"($name): ($item_path)" show_clip_to $item_path false }, - _ => { + _ => { _print $"($select) ($name): ($item_path)" show_clip_to $item_path false } } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/qr.nu b/nulib/lib_provisioning/utils/qr.nu index d51b7fb..ba3dd18 100644 --- a/nulib/lib_provisioning/utils/qr.nu +++ b/nulib/lib_provisioning/utils/qr.nu @@ -4,4 +4,4 @@ export def "make_qr" [ url?: string ] { show_qr ($url | default (get-provisioning-url)) -} +} diff --git a/nulib/lib_provisioning/utils/settings.nu b/nulib/lib_provisioning/utils/settings.nu index c921bb6..9102b9e 100644 --- a/nulib/lib_provisioning/utils/settings.nu +++ b/nulib/lib_provisioning/utils/settings.nu @@ -131,22 +131,40 @@ export def get_infra [ (get-workspace-path $effective_ws) } } -export def parse_kcl_file [ +# Local implementation to avoid circular imports with plugins_defs.nu +def _process_decl_file_local [ + decl_file: string + format: string +]: nothing -> string { + # Use external Nickel CLI (no plugin dependency) + let result = (^nickel export $decl_file --format $format | complete) + if $result.exit_code == 0 { + $result.stdout + } else { + error make { msg: $result.stderr } + } +} + +export def parse_nickel_file [ src: string target: string append: bool msg: string err_exit?: bool = false ]: nothing -> bool { - # Try nu_plugin_kcl first if available + # Try to process Nickel file let format = if (get-work-format) == "json" { "json" } else { "yaml" } - let result = (process_kcl_file $src $format) + let result = (do -i { + _process_decl_file_local $src $format + }) + if ($result | is-empty) { - let text = $"kcl ($src) failed code ($result.exit_code)" - (throw-error $msg $text "parse_kcl_file" --span (metadata $result).span) - if $err_exit { exit $result.exit_code } + let text = $"nickel ($src) failed" + (throw-error $msg $text "parse_nickel_file" --span (metadata $src).span) + if $err_exit { exit 1 } return false } + if $append { $result | save --append $target } else { @@ -176,19 +194,19 @@ export def load_defaults [ } let full_path = if ($item_path | path exists) { ($item_path) - } else if ($"($item_path).k" | path exists) { - $"($item_path).k" - } else if ($src_path | path dirname | path join $"($item_path).k" | path exists) { - $src_path | path dirname | path join $"($item_path).k" + } else if ($"($item_path).ncl" | path exists) { + $"($item_path).ncl" + } else if ($src_path | path dirname | path join $"($item_path).ncl" | path exists) { + $src_path | path dirname | path join $"($item_path).ncl" } else { "" } if $full_path == "" { return true } if (is_sops_file $full_path) { decode_sops_file $full_path $target_path true - (parse_kcl_file $target_path $target_path false $"πŸ›‘ load default settings failed ($target_path) ") + (parse_nickel_file $target_path $target_path false $"πŸ›‘ load default settings failed ($target_path) ") } else { - (parse_kcl_file $full_path $target_path false $"πŸ›‘ load default settings failed ($full_path)") + (parse_nickel_file $full_path $target_path false $"πŸ›‘ load default settings failed ($full_path)") } } export def get_provider_env [ @@ -199,7 +217,7 @@ export def get_provider_env [ $server.prov_settings } else { let file_path = ($settings.src_path | path join $server.prov_settings) - if ($file_path | str ends-with '.k' ) { $file_path } else { $"($file_path).k" } + if ($file_path | str ends-with '.ncl' ) { $file_path } else { $"($file_path).ncl" } } if not ($prov_env_path| path exists ) { if (is-debug-enabled) { _print $"πŸ›‘ load (_ansi cyan_bold)provider_env(_ansi reset) from ($server.prov_settings) failed at ($prov_env_path)" } @@ -210,13 +228,13 @@ export def get_provider_env [ let created_taskservs_dirpath = if ($str_created_taskservs_dirpath | str starts-with "/" ) { $str_created_taskservs_dirpath } else { $settings.src_path | path join $str_created_taskservs_dirpath } if not ( $created_taskservs_dirpath | path exists) { ^mkdir -p $created_taskservs_dirpath } let source_settings_path = ($created_taskservs_dirpath | path join $"($prov_env_path | path basename)") - let target_settings_path = ($created_taskservs_dirpath| path join $"($prov_env_path | path basename | str replace '.k' '').((get-work-format))") + let target_settings_path = ($created_taskservs_dirpath| path join $"($prov_env_path | path basename | str replace '.ncl' '').((get-work-format))") let res = if (is_sops_file $prov_env_path) { decode_sops_file $prov_env_path $source_settings_path true - (parse_kcl_file $source_settings_path $target_settings_path false $"πŸ›‘ load prov settings failed ($target_settings_path)") + (parse_nickel_file $source_settings_path $target_settings_path false $"πŸ›‘ load prov settings failed ($target_settings_path)") } else { cp $prov_env_path $source_settings_path - (parse_kcl_file $source_settings_path $target_settings_path false $"πŸ›‘ load prov settings failed ($prov_env_path)") + (parse_nickel_file $source_settings_path $target_settings_path false $"πŸ›‘ load prov settings failed ($prov_env_path)") } if not (is-debug-enabled) { rm -f $source_settings_path } if $res and ($target_settings_path | path exists) { @@ -345,10 +363,10 @@ def load-servers-from-definitions [ mut loaded_servers = [] for it in $servers_paths { - let file_path = if ($it | str ends-with ".k") { + let file_path = if ($it | str ends-with ".ncl") { $it } else { - $"($it).k" + $"($it).ncl" } let server_path = if ($file_path | str starts-with "/") { $file_path @@ -365,7 +383,7 @@ def load-servers-from-definitions [ } let target_settings_path = $"($wk_settings_path)/($it | str replace --all "/" "_").((get-work-format))" - if not (parse_kcl_file ($server_path) $target_settings_path false "πŸ›‘ load settings failed ") { + if not (parse_nickel_file ($server_path) $target_settings_path false "πŸ›‘ load settings failed ") { continue } if not ($target_settings_path | path exists) { @@ -477,7 +495,7 @@ export def load [ include_notuse?: bool = false --no_error ]: nothing -> record { - let source = if $in_src == null or ($in_src | str ends-with '.k' ) { $in_src } else { $"($in_src).k" } + let source = if $in_src == null or ($in_src | str ends-with '.ncl' ) { $in_src } else { $"($in_src).ncl" } let source_path = if $source != null and ($source | path type) == "dir" { $"($source)/((get-default-settings))" } else { $source } let src_path = if $source_path != null and ($source_path | path exists) { $"./($source_path)" @@ -503,21 +521,21 @@ export def load [ $env.PWD | path join $src_dir } let wk_settings_path = mktemp -d - if not (parse_kcl_file $"($src_path)" $"($wk_settings_path)/settings.((get-work-format))" false "πŸ›‘ load settings failed ") { + if not (parse_nickel_file $"($src_path)" $"($wk_settings_path)/settings.((get-work-format))" false "πŸ›‘ load settings failed ") { if $no_error { return {} } else { return } } if (is-debug-enabled) { _print $"DEBUG source path: ($src_path)" } let settings_file = $"($wk_settings_path)/settings.((get-work-format))" if not ($settings_file | path exists) { if $no_error { return {} } else { - (throw-error "πŸ›‘ settings file not created" $"parse_kcl_file succeeded but file not found: ($settings_file)" "settings->load") + (throw-error "πŸ›‘ settings file not created" $"parse_nickel_file succeeded but file not found: ($settings_file)" "settings->load") return } } let settings_data = open $settings_file if (is-debug-enabled) { _print $"DEBUG work path: ($wk_settings_path)" } - # Extract servers from top-level if present (KCL output has servers at top level) + # Extract servers from top-level if present (Nickel output has servers at top level) mut raw_servers = ($settings_data | get servers? | default []) let servers_paths = ($settings_data.settings | get servers_paths? | default []) @@ -603,8 +621,8 @@ export def save_settings_file [ ]: nothing -> nothing { let it_path = if ($target_file | path exists) { $target_file - } else if ($settings.src_path | path join $"($target_file).k" | path exists) { - ($settings.src_path | path join $"($target_file).k") + } else if ($settings.src_path | path join $"($target_file).ncl" | path exists) { + ($settings.src_path | path join $"($target_file).ncl") } else if ($settings.src_path | path join $"($target_file).((get-work-format))" | path exists) { ($settings.src_path | path join $"($target_file).((get-work-format))") } else { @@ -664,4 +682,4 @@ export def settings_with_env [ } } ($settings | merge { data: ($settings.data | merge { servers: $servers_with_ips}) }) -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/simple_validation.nu b/nulib/lib_provisioning/utils/simple_validation.nu index bc0a5d3..ccdb0c9 100644 --- a/nulib/lib_provisioning/utils/simple_validation.nu +++ b/nulib/lib_provisioning/utils/simple_validation.nu @@ -53,4 +53,4 @@ export def safe-run [ } else { $result.stdout } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/ssh.nu b/nulib/lib_provisioning/utils/ssh.nu index f464ad5..024cf61 100644 --- a/nulib/lib_provisioning/utils/ssh.nu +++ b/nulib/lib_provisioning/utils/ssh.nu @@ -6,30 +6,30 @@ export def ssh_cmd [ with_bash: bool cmd: string live_ip: string -] { +] { let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } if $ip == "" { return false } if not (check_connection $server $ip "ssh_cmd") { return false } - let remote_cmd = if $with_bash { - let ops = if (is-debug-enabled) { "-x" } else { "" } + let remote_cmd = if $with_bash { + let ops = if (is-debug-enabled) { "-x" } else { "" } $"bash ($ops) ($cmd)" } else { $cmd } let ssh_loglevel = if (is-debug-enabled) { _print $"Run ($remote_cmd) in ($server.installer_user)@($ip)" "-o LogLevel=info" - } else { + } else { "-o LogLevel=quiet" } let ssh_op_0 = if ($env.SSH_OPS | length) > 0 { $env.SSH_OPS | get 0 } else { "" } let ssh_op_1 = if ($env.SSH_OPS | length) > 1 { $env.SSH_OPS | get 1 } else { "" } let res = (^ssh "-o" $ssh_op_0 "-o" $ssh_op_1 "-o" IdentitiesOnly=yes $ssh_loglevel "-i" ($server.ssh_key_path | str replace ".pub" "") - $"($server.installer_user)@($ip)" ($remote_cmd) | complete) + $"($server.installer_user)@($ip)" ($remote_cmd) | complete) if $res.exit_code != 0 { _print $"❗ run ($remote_cmd) in ($server.hostname) errors ($res.stdout ) " return false @@ -43,10 +43,10 @@ export def scp_to [ source: list target: string live_ip: string -] { +] { let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } @@ -55,16 +55,16 @@ export def scp_to [ let source_files = ($source | str join " ") let ssh_op_0 = if ($env.SSH_OPS | length) > 0 { $env.SSH_OPS | get 0 } else { "" } let ssh_op_1 = if ($env.SSH_OPS | length) > 1 { $env.SSH_OPS | get 1 } else { "" } - let ssh_loglevel = if (is-debug-enabled) { + let ssh_loglevel = if (is-debug-enabled) { _print $"Sending ($source | str join ' ') to ($server.installer_user)@($ip)/tmp/($target)" - _print $"scp -o ($ssh_op_0) -o ($ssh_op_1) -o IdentitiesOnly=yes -i ($server.ssh_key_path | str replace ".pub" "") ($source_files) ($server.installer_user)@($ip):($target)" + _print $"scp -o ($ssh_op_0) -o ($ssh_op_1) -o IdentitiesOnly=yes -i ($server.ssh_key_path | str replace ".pub" "") ($source_files) ($server.installer_user)@($ip):($target)" "-o LogLevel=info" - } else { + } else { "-o LogLevel=quiet" } let res = (^scp "-o" $ssh_op_0 "-o" $ssh_op_1 "-o" IdentitiesOnly=yes $ssh_loglevel "-i" ($server.ssh_key_path | str replace ".pub" "") - $source_files $"($server.installer_user)@($ip):($target)" | complete) + $source_files $"($server.installer_user)@($ip):($target)" | complete) if $res.exit_code != 0 { _print $"❗ copy ($target | str join ' ') to ($server.hostname) errors ($res.stdout ) " return false @@ -78,10 +78,10 @@ export def scp_from [ source: string target: string live_ip: string -] { +] { let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } @@ -89,15 +89,15 @@ export def scp_from [ if not (check_connection $server $ip "scp_from") { return false } let ssh_op_0 = if ($env.SSH_OPS | length) > 0 { $env.SSH_OPS | get 0 } else { "" } let ssh_op_1 = if ($env.SSH_OPS | length) > 1 { $env.SSH_OPS | get 1 } else { "" } - let ssh_loglevel = if (is-debug-enabled) { + let ssh_loglevel = if (is-debug-enabled) { _print $"Getting ($target | str join ' ') from ($server.installer_user)@($ip)/tmp/($target)" "-o LogLevel=info" - } else { + } else { "-o LogLevel=quiet" } let res = (^scp "-o" $ssh_op_0 "-o" $ssh_op_1 "-o" IdentitiesOnly=yes $ssh_loglevel "-i" ($server.ssh_key_path | str replace ".pub" "") - $"($server.installer_user)@($ip):($source)" $target | complete) + $"($server.installer_user)@($ip):($source)" $target | complete) if $res.exit_code != 0 { _print $"❗ copy ($source) from ($server.hostname) to ($target) errors ($res.stdout ) " return false @@ -113,21 +113,21 @@ export def ssh_cp_run [ with_bash: bool live_ip: string ssh_remove: bool -] { +] { let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } - if $ip == "" { + if $ip == "" { _print $"❗ ssh_cp_run (_ansi red_bold)No IP(_ansi reset) to (_ansi green_bold)($server.hostname)(_ansi reset)" return false } if not (scp_to $settings $server $source $target $ip) { return false } if not (ssh_cmd $settings $server $with_bash $target $ip) { return false } if $env.PROVISIONING_SSH_DEBUG? != null and $env.PROVISIONING_SSH_DEBUG { return true } - if $ssh_remove { + if $ssh_remove { return (ssh_cmd $settings $server false $"rm -f ($target)" $ip) } true @@ -139,10 +139,10 @@ export def check_connection [ ] { if not (port_scan $ip $server.liveness_port 1) { _print ( - $"\nπŸ›‘ (_ansi red)Error connection(_ansi reset) ($origin) (_ansi blue)($server.hostname)(_ansi reset) " + + $"\nπŸ›‘ (_ansi red)Error connection(_ansi reset) ($origin) (_ansi blue)($server.hostname)(_ansi reset) " + $"(_ansi blue_bold)($ip)(_ansi reset) at ($server.liveness_port) (_ansi red_bold)failed(_ansi reset) " ) return false - } + } true -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/templates.nu b/nulib/lib_provisioning/utils/templates.nu index 9243355..fc1e36d 100644 --- a/nulib/lib_provisioning/utils/templates.nu +++ b/nulib/lib_provisioning/utils/templates.nu @@ -82,8 +82,19 @@ export def run_from_template [ return false } - # Call tera-render with JSON file path as context (second parameter) - let result = (tera-render $template_path $vars_path) + # Ensure tera plugin is loaded in this context + (plugin use tera) + + # Call tera-render with context data + if (is-debug-enabled) { + _print $"DEBUG: tera-render ($template_path) with context from ($vars_path)" + _print $"DEBUG: template exists: ($template_path | path exists)" + _print $"DEBUG: vars exists: ($vars_path | path exists)" + } + + # Load variables as a record and pass via pipeline + let vars_data = (open $vars_path --raw | from yaml) + let result = ($vars_data | tera-render $template_path) if ($result | describe) == "nothing" or ($result | str length) == 0 { let text = $"(_ansi yellow)template(_ansi reset): ($template_path)\n(_ansi yellow)vars(_ansi reset): ($vars_path)\n(_ansi red)Failed(_ansi reset)" diff --git a/nulib/lib_provisioning/utils/test.nu b/nulib/lib_provisioning/utils/test.nu index cbcf608..fc52ad1 100644 --- a/nulib/lib_provisioning/utils/test.nu +++ b/nulib/lib_provisioning/utils/test.nu @@ -1,9 +1,9 @@ export def on_test [] { - use nupm/ + use nupm/ cd $"($env.PROVISIONING)/core/nulib" nupm test test_addition cd $env.PWD nupm test basecamp_addition -} +} diff --git a/nulib/lib_provisioning/utils/ui.nu b/nulib/lib_provisioning/utils/ui.nu index 34ed501..effd3b4 100644 --- a/nulib/lib_provisioning/utils/ui.nu +++ b/nulib/lib_provisioning/utils/ui.nu @@ -2,10 +2,9 @@ # Exclude minor or specific parts for global 'export use' -export use clean.nu * -export use error.nu * -export use help.nu * - -export use interface.nu * -export use undefined.nu * +export use clean.nu * +export use error.nu * +export use help.nu * +export use interface.nu * +export use undefined.nu * diff --git a/nulib/lib_provisioning/utils/undefined.nu b/nulib/lib_provisioning/utils/undefined.nu index acfdacb..3034b37 100644 --- a/nulib/lib_provisioning/utils/undefined.nu +++ b/nulib/lib_provisioning/utils/undefined.nu @@ -4,24 +4,24 @@ export def option_undefined [ root: string src: string info?: string -] { - _print $"πŸ›‘ invalid_option ($src) ($info)" +] { + _print $"πŸ›‘ invalid_option ($src) ($info)" _print $"\nUse (_ansi blue_bold)((get-provisioning-name)) ($root) ($src) help(_ansi reset) for help on commands and options" } - + export def invalid_task [ src: string task: string --end -] { - let show_src = {|color| +] { + let show_src = {|color| if $src == "" { "" } else { $" (_ansi $color)($src)(_ansi reset)"} } - if $task != "" { - _print $"πŸ›‘ invalid (_ansi blue)((get-provisioning-name))(_ansi reset)(do $show_src "yellow") task or option: (_ansi red)($task)(_ansi reset)" - } else { - _print $"(_ansi blue)((get-provisioning-name))(_ansi reset)(do $show_src "yellow") no task or option found !" - } + if $task != "" { + _print $"πŸ›‘ invalid (_ansi blue)((get-provisioning-name))(_ansi reset)(do $show_src "yellow") task or option: (_ansi red)($task)(_ansi reset)" + } else { + _print $"(_ansi blue)((get-provisioning-name))(_ansi reset)(do $show_src "yellow") no task or option found !" + } _print $"Use (_ansi blue_bold)((get-provisioning-name))(_ansi reset)(do $show_src "blue_bold") (_ansi blue_bold)help(_ansi reset) for help on commands and options" - if $end and not (is-debug-enabled) { end_run "" } -} \ No newline at end of file + if $end and not (is-debug-enabled) { end_run "" } +} diff --git a/nulib/lib_provisioning/utils/validation.nu b/nulib/lib_provisioning/utils/validation.nu index 09981b2..37c356a 100644 --- a/nulib/lib_provisioning/utils/validation.nu +++ b/nulib/lib_provisioning/utils/validation.nu @@ -28,7 +28,7 @@ export def validate-path [ } return false } - + if $must_exist and not ($path | path exists) { print $"πŸ›‘ Path '($path)' does not exist" if ($context | is-not-empty) { @@ -37,7 +37,7 @@ export def validate-path [ print "πŸ’‘ Check if the path exists and you have proper permissions" return false } - + true } @@ -81,14 +81,14 @@ export def validate-settings [ settings: record required_fields: list ] { - let missing_fields = ($required_fields | where {|field| + let missing_fields = ($required_fields | where {|field| ($settings | try { get $field } catch { null } | is-empty) }) - + if ($missing_fields | length) > 0 { print "πŸ›‘ Missing required settings fields:" $missing_fields | each {|field| print $" - ($field)"} return false } true -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/validation_helpers.nu b/nulib/lib_provisioning/utils/validation_helpers.nu index 97839af..4e270be 100644 --- a/nulib/lib_provisioning/utils/validation_helpers.nu +++ b/nulib/lib_provisioning/utils/validation_helpers.nu @@ -28,7 +28,7 @@ export def validate-path [ } return false } - + if $must_exist and not ($path | path exists) { print $"πŸ›‘ Path '($path)' does not exist" if ($context | is-not-empty) { @@ -37,7 +37,7 @@ export def validate-path [ print "πŸ’‘ Check if the path exists and you have proper permissions" return false } - + true } @@ -69,12 +69,12 @@ export def validate-ip [ } return false } - + let valid_parts = ($ip_parts | each {|part| let num = ($part | into int) $num >= 0 and $num <= 255 }) - + if not ($valid_parts | all {|valid| $valid}) { print $"πŸ›‘ Invalid IP address values: ($ip)" if ($context | is-not-empty) { @@ -82,7 +82,7 @@ export def validate-ip [ } return false } - + true } @@ -105,10 +105,10 @@ export def validate-settings [ required_fields: list context?: string ]: bool { - let missing_fields = ($required_fields | where {|field| + let missing_fields = ($required_fields | where {|field| ($settings | try { get $field } catch { null } | is-empty) }) - + if ($missing_fields | length) > 0 { print "πŸ›‘ Missing required settings fields:" $missing_fields | each {|field| print $" - ($field)"} @@ -118,4 +118,4 @@ export def validate-settings [ return false } true -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_core.nu b/nulib/lib_provisioning/utils/version_core.nu index 81f2cbb..8a868fa 100644 --- a/nulib/lib_provisioning/utils/version_core.nu +++ b/nulib/lib_provisioning/utils/version_core.nu @@ -285,4 +285,4 @@ export def check-version [ fixed: $is_fixed status: $status } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_formatter.nu b/nulib/lib_provisioning/utils/version_formatter.nu index 8ca9e7a..da21dba 100644 --- a/nulib/lib_provisioning/utils/version_formatter.nu +++ b/nulib/lib_provisioning/utils/version_formatter.nu @@ -91,4 +91,4 @@ export def format-results [ for status in ($by_status | transpose key value) { print $" (format-status $status.key --icons=$icons): ($status.value | length)" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_loader.nu b/nulib/lib_provisioning/utils/version_loader.nu index 7fc1e49..fee7968 100644 --- a/nulib/lib_provisioning/utils/version_loader.nu +++ b/nulib/lib_provisioning/utils/version_loader.nu @@ -14,14 +14,14 @@ export def discover-configurations [ } else { $base_path } mut configurations = [] - # Load from known version files directly - try KCL first, then YAML - let version_files_kcl = [ - ($base | path join "core" | path join "versions.k") + # Load from known version files directly - try Nickel first, then YAML + let version_files_nickel = [ + ($base | path join "core" | path join "versions.ncl") ] - for file in $version_files_kcl { + for file in $version_files_nickel { if ($file | path exists) { - let configs = (load-kcl-version-file $file) + let configs = (load-nickel-version-file $file) if ($configs | is-not-empty) { $configurations = ($configurations | append $configs) } @@ -60,10 +60,10 @@ export def discover-configurations [ for provider_item in (ls $active_providers_path) { let provider_dir = ($active_providers_path | path join $provider_item.name) - # Try KCL version file first (single source of truth) - let kcl_version_file = ($provider_dir | path join "kcl" | path join "version.k") - if ($kcl_version_file | path exists) { - let configs = (load-kcl-version-file $kcl_version_file) + # Try Nickel version file first (single source of truth) + let nickel_version_file = ($provider_dir | path join "nickel" | path join "version.ncl") + if ($nickel_version_file | path exists) { + let configs = (load-nickel-version-file $nickel_version_file) if ($configs | is-not-empty) { $configurations = ($configurations | append $configs) } @@ -137,9 +137,9 @@ export def load-configuration-file [ } } "k" => { - # Parse KCL files for version information + # Parse Nickel files for version information let content = (open $file_path) - let version_data = (extract-kcl-versions $content) + let version_data = (extract-nickel-versions $content) for item in $version_data { let config = (create-configuration $item.name $item $context $file_path) $configs = ($configs | append $config) @@ -169,36 +169,36 @@ export def load-configuration-file [ $configs } -# Load KCL version file by compiling it to JSON -export def load-kcl-version-file [ +# Load Nickel version file by compiling it to JSON +export def load-nickel-version-file [ file_path: string ]: nothing -> list { if not ($file_path | path exists) { return [] } # Determine parent context - could be provider or core - # provider: extensions/providers/{name}/kcl/version.k -> extensions/providers/{name} - # core: core/versions.k -> core (no kcl dir) - let parent_dir = if ($file_path | str contains "/kcl/version.k") { - $file_path | path dirname | path dirname # kcl/version.k -> provider_dir + # provider: extensions/providers/{name}/nickel/version.ncl -> extensions/providers/{name} + # core: core/versions.ncl -> core (no nickel dir) + let parent_dir = if ($file_path | str contains "/nickel/version.ncl") { + $file_path | path dirname | path dirname # nickel/version.ncl -> provider_dir } else { - $file_path | path dirname # versions.k -> core + $file_path | path dirname # versions.ncl -> core } let context = (extract-context $parent_dir) mut configs = [] - # Compile KCL to JSON - let kcl_result = (^kcl run $file_path --format json | complete) + # Compile Nickel to JSON + let decl_result = (^nickel export $file_path --format json | complete) - # If KCL compilation succeeded, parse the output - if $kcl_result.exit_code != 0 { return $configs } + # If Nickel compilation succeeded, parse the output + if $decl_result.exit_code != 0 { return $configs } # Safely parse JSON with fallback let json_data = ( - $kcl_result.stdout | from json | default {} + $decl_result.stdout | from json | default {} ) - # Handle different KCL output formats: + # Handle different Nickel output formats: # 1. Provider files: Single object with {name, version, dependencies} # 2. Core files: Object {core_versions: [{}]} or plain array [{}] let is_array = ($json_data | describe | str contains "^list") @@ -210,7 +210,7 @@ export def load-kcl-version-file [ # It's an array (plain array format) $json_data } else if ($json_data | get name? | default null) != null { - # It's a single object (provider kcl/version.k) + # It's a single object (provider nickel/version.ncl) [$json_data] } else { [] @@ -386,8 +386,8 @@ export def create-configuration [ } } -# Extract version info from KCL content -export def extract-kcl-versions [ +# Extract version info from Nickel content +export def extract-nickel-versions [ content: string ]: nothing -> list { mut versions = [] @@ -428,4 +428,4 @@ export def extract-kcl-versions [ } $versions -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_manager.nu b/nulib/lib_provisioning/utils/version_manager.nu index c2ea8aa..c85f53e 100644 --- a/nulib/lib_provisioning/utils/version_manager.nu +++ b/nulib/lib_provisioning/utils/version_manager.nu @@ -205,8 +205,8 @@ export def update-configuration-file [ print $"⚠️ TOML update not implemented for ($file_path)" } "k" => { - # KCL update would need KCL parser/writer - print $"⚠️ KCL update not implemented for ($file_path)" + # Nickel update would need Nickel parser/writer + print $"⚠️ Nickel update not implemented for ($file_path)" } _ => { print $"⚠️ Unknown file type: ($ext)" @@ -238,4 +238,4 @@ export def set-fixed [ } else { print $"πŸ”“ Unpinned ($component_id)" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_registry.nu b/nulib/lib_provisioning/utils/version_registry.nu index f95c360..0bc00df 100644 --- a/nulib/lib_provisioning/utils/version_registry.nu +++ b/nulib/lib_provisioning/utils/version_registry.nu @@ -158,7 +158,7 @@ export def compare-registry-with-taskservs [ let taskserv_versions = ($taskservs | each { |ts| { id: $ts.id version: $ts.version - file: $ts.kcl_file + file: $ts.nickel_file matches_registry: ($ts.version == $registry_version) }}) @@ -232,4 +232,4 @@ export def set-registry-fixed [ } else { _print $"πŸ”“ Unpinned ($component_id) in registry" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/utils/version_taskserv.nu b/nulib/lib_provisioning/utils/version_taskserv.nu index 9ca34e1..330027c 100644 --- a/nulib/lib_provisioning/utils/version_taskserv.nu +++ b/nulib/lib_provisioning/utils/version_taskserv.nu @@ -1,14 +1,14 @@ #!/usr/bin/env nu # Taskserv version extraction and management utilities -# Handles KCL taskserv files and version configuration +# Handles Nickel taskserv files and version configuration use ../config/accessor.nu * use version_core.nu * use version_loader.nu * use interface.nu * -# Extract version field from KCL taskserv files -export def extract-kcl-version [ +# Extract version field from Nickel taskserv files +export def extract-nickel-version [ file_path: string ]: nothing -> string { if not ($file_path | path exists) { return "" } @@ -59,7 +59,7 @@ export def extract-kcl-version [ } } -# Discover all taskserv KCL files and their versions +# Discover all taskserv Nickel files and their versions export def discover-taskserv-configurations [ --base-path: string = "" ]: nothing -> list { @@ -74,32 +74,32 @@ export def discover-taskserv-configurations [ return [] } - # Find all .k files recursively in the taskservs directory - let all_k_files = (glob $"($taskservs_path)/**/*.k") + # Find all .ncl files recursively in the taskservs directory + let all_k_files = (glob $"($taskservs_path)/**/*.ncl") - let kcl_configs = ($all_k_files | each { |kcl_file| - let version = (extract-kcl-version $kcl_file) + let nickel_configs = ($all_k_files | each { |decl_file| + let version = (extract-nickel-version $decl_file) if ($version | is-not-empty) { - let relative_path = ($kcl_file | str replace $"($taskservs_path)/" "") + let relative_path = ($decl_file | str replace $"($taskservs_path)/" "") let path_parts = ($relative_path | split row "/" | where { |p| $p != "" }) # Determine ID from the path structure let id = if ($path_parts | length) >= 2 { - # If it's a server-specific file like "wuji-strg-1/kubernetes.k" - let filename = ($kcl_file | path basename | str replace ".k" "") + # If it's a server-specific file like "wuji-strg-1/kubernetes.ncl" + let filename = ($decl_file | path basename | str replace ".ncl" "") $"($path_parts.0)::($filename)" } else { - # If it's a general file like "proxy.k" - ($kcl_file | path basename | str replace ".k" "") + # If it's a general file like "proxy.ncl" + ($decl_file | path basename | str replace ".ncl" "") } { id: $id type: "taskserv" - kcl_file: $kcl_file + nickel_file: $decl_file version: $version metadata: { - source_file: $kcl_file + source_file: $decl_file category: "taskserv" path_structure: $path_parts } @@ -109,11 +109,11 @@ export def discover-taskserv-configurations [ } } | where { |item| $item != null }) - $kcl_configs + $nickel_configs } -# Update version in KCL file -export def update-kcl-version [ +# Update version in Nickel file +export def update-nickel-version [ file_path: string new_version: string ]: nothing -> nothing { @@ -163,13 +163,13 @@ export def check-taskserv-versions [ id: $config.id type: $config.type configured: $config.version - kcl_file: $config.kcl_file + nickel_file: $config.nickel_file status: "configured" } } } -# Update taskserv version in KCL file +# Update taskserv version in Nickel file export def update-taskserv-version [ taskserv_id: string new_version: string @@ -184,11 +184,11 @@ export def update-taskserv-version [ } if $dry_run { - _print $"πŸ” Would update ($taskserv_id) from ($config.version) to ($new_version) in ($config.kcl_file)" + _print $"πŸ” Would update ($taskserv_id) from ($config.version) to ($new_version) in ($config.nickel_file)" return } - update-kcl-version $config.kcl_file $new_version + update-nickel-version $config.nickel_file $new_version } # Bulk update multiple taskservs @@ -264,7 +264,7 @@ export def taskserv-sync-versions [ _print $"πŸ” Would update ($taskserv.id): ($taskserv.version) -> ($comp.registry_version)" } else { _print $"πŸ”„ Updating ($taskserv.id): ($taskserv.version) -> ($comp.registry_version)" - update-kcl-version $taskserv.file $comp.registry_version + update-nickel-version $taskserv.file $comp.registry_version } } } diff --git a/nulib/lib_provisioning/vm/backend_libvirt.nu b/nulib/lib_provisioning/vm/backend_libvirt.nu index b35dc17..43db39a 100644 --- a/nulib/lib_provisioning/vm/backend_libvirt.nu +++ b/nulib/lib_provisioning/vm/backend_libvirt.nu @@ -371,4 +371,4 @@ export def "libvirt-create-disk" [ size_gb: $size_gb format: "qcow2" } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/cleanup_scheduler.nu b/nulib/lib_provisioning/vm/cleanup_scheduler.nu index f4a2afc..ee37366 100644 --- a/nulib/lib_provisioning/vm/cleanup_scheduler.nu +++ b/nulib/lib_provisioning/vm/cleanup_scheduler.nu @@ -147,8 +147,12 @@ def run-scheduler-loop [interval_minutes: int] { print "VM Cleanup Scheduler starting..." print $"Check interval: ($interval_minutes) minutes" + print "Press Ctrl+C to stop scheduler" - loop { + mut iteration = 0 + let max_iterations = 1_000_000 # Safety limit: ~2 years at 1 min intervals + + while { $iteration < $max_iterations } { # Run cleanup let result = (cleanup-expired-vms) @@ -159,7 +163,11 @@ def run-scheduler-loop [interval_minutes: int] { # Wait for next check print $"[$(date now)] Next check in ($interval_minutes) minutes" sleep ($interval_minutes)m + + $iteration += 1 } + + print "Scheduler reached iteration limit - exiting" } def create-scheduler-script [interval: int, script_path: string] { @@ -168,9 +176,13 @@ def create-scheduler-script [interval: int, script_path: string] { let script_content = $' use lib_provisioning/vm/cleanup_scheduler.nu * -loop \{ +mut iteration = 0 +let max_iterations = 1_000_000 + +while \{ $iteration < $max_iterations \} \{ cleanup-expired-vms sleep ($interval)m + $iteration += 1 \} ' diff --git a/nulib/lib_provisioning/vm/golden_image_builder.nu b/nulib/lib_provisioning/vm/golden_image_builder.nu index 75e424a..c091461 100644 --- a/nulib/lib_provisioning/vm/golden_image_builder.nu +++ b/nulib/lib_provisioning/vm/golden_image_builder.nu @@ -643,4 +643,4 @@ apt-get upgrade -y apt-get clean rm -rf /var/lib/apt/lists/* " -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/lifecycle.nu b/nulib/lib_provisioning/vm/lifecycle.nu index d703228..1d1d0dc 100644 --- a/nulib/lib_provisioning/vm/lifecycle.nu +++ b/nulib/lib_provisioning/vm/lifecycle.nu @@ -7,7 +7,7 @@ use ./backend_libvirt.nu * use ./persistence.nu * export def "vm-create" [ - vm_config: record # VM configuration (from KCL) + vm_config: record # VM configuration (from Nickel) --backend: string = "libvirt" # Backend to use ]: record { """ diff --git a/nulib/lib_provisioning/vm/multi_tier_deployment.nu b/nulib/lib_provisioning/vm/multi_tier_deployment.nu index f9df40f..2a45fc4 100644 --- a/nulib/lib_provisioning/vm/multi_tier_deployment.nu +++ b/nulib/lib_provisioning/vm/multi_tier_deployment.nu @@ -414,4 +414,4 @@ def create-deployment-networks [name: string, tiers: list]: list } } | compact -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/nested_provisioning.nu b/nulib/lib_provisioning/vm/nested_provisioning.nu index 1963ae6..ae752db 100644 --- a/nulib/lib_provisioning/vm/nested_provisioning.nu +++ b/nulib/lib_provisioning/vm/nested_provisioning.nu @@ -389,4 +389,4 @@ def get-nesting-depth [vm: string]: int { } else { 0 } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/network_management.nu b/nulib/lib_provisioning/vm/network_management.nu index b09c9db..844d284 100644 --- a/nulib/lib_provisioning/vm/network_management.nu +++ b/nulib/lib_provisioning/vm/network_management.nu @@ -361,4 +361,4 @@ def get-network-connections [name: string]: list { bash -c $"cut -d'|' -f1 ($connections_file)" | lines -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/persistence.nu b/nulib/lib_provisioning/vm/persistence.nu index a9a2401..71b8403 100644 --- a/nulib/lib_provisioning/vm/persistence.nu +++ b/nulib/lib_provisioning/vm/persistence.nu @@ -180,4 +180,4 @@ export def "cleanup-temporary-vms" [ skipped: ($to_cleanup | length) results: $results } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/preparer.nu b/nulib/lib_provisioning/vm/preparer.nu index 6dd8a4a..abb0493 100644 --- a/nulib/lib_provisioning/vm/preparer.nu +++ b/nulib/lib_provisioning/vm/preparer.nu @@ -213,4 +213,4 @@ export def "ensure-vm-support" [host: string]: record { message: $"VM support installed and verified" primary_hypervisor: $status2.primary_backend } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/ssh_utils.nu b/nulib/lib_provisioning/vm/ssh_utils.nu index e5bcdb5..e534fd6 100644 --- a/nulib/lib_provisioning/vm/ssh_utils.nu +++ b/nulib/lib_provisioning/vm/ssh_utils.nu @@ -153,15 +153,15 @@ def get-vm-ip [vm_name: string]: string { def wait-for-ssh [ip: string, --timeout: int = 300]: bool { """Wait for SSH to become available""" - let start = (date now | date to-record | debug) - let max_wait = $timeout + let start_time = (date now) + let timeout_duration = ($timeout)sec + mut attempts = 0 + let max_attempts = ($timeout / 2) + 1 # Safety limit based on sleep 2sec - loop { - let elapsed = ( - (date now | date to-record | debug) - $start - ) + while { $attempts < $max_attempts } { + let elapsed = ((date now) - $start_time) - if $elapsed >= $max_wait { + if $elapsed >= $timeout_duration { return false } @@ -179,7 +179,10 @@ def wait-for-ssh [ip: string, --timeout: int = 300]: bool { # Wait before retry sleep 2sec + $attempts += 1 } + + false } export def "vm-provision" [ diff --git a/nulib/lib_provisioning/vm/state_recovery.nu b/nulib/lib_provisioning/vm/state_recovery.nu index f06262f..3ce3aa4 100644 --- a/nulib/lib_provisioning/vm/state_recovery.nu +++ b/nulib/lib_provisioning/vm/state_recovery.nu @@ -263,15 +263,15 @@ export def "wait-for-vm-ssh" [ ]: record { """Wait for VM SSH to become available""" - let start_time = (date now | date to-record) - let timeout_seconds = $timeout + let start_time = (date now) + let timeout_duration = ($timeout)sec + mut attempts = 0 + let max_attempts = ($timeout / 2) + 1 # Safety limit based on sleep 2s - loop { - let elapsed = ( - ((date now | date to-record) - $start_time) / 1_000_000_000 - ) + while { $attempts < $max_attempts } { + let elapsed = ((date now) - $start_time) - if $elapsed >= $timeout_seconds { + if $elapsed >= $timeout_duration { return { success: false error: $"SSH timeout after ($timeout_seconds) seconds" @@ -294,6 +294,12 @@ export def "wait-for-vm-ssh" [ } sleep 2s + $attempts += 1 + } + + { + success: false + error: $"SSH timeout after ($timeout) seconds" } } diff --git a/nulib/lib_provisioning/vm/vm_persistence.nu b/nulib/lib_provisioning/vm/vm_persistence.nu index 10405ee..124b0d8 100644 --- a/nulib/lib_provisioning/vm/vm_persistence.nu +++ b/nulib/lib_provisioning/vm/vm_persistence.nu @@ -426,4 +426,4 @@ export def "get-vm-persistence-stats" []: record { expired_vms: ($expired | length) auto_cleanup_enabled: ($temporary | where auto_cleanup == true | length) } -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/vm/volume_management.nu b/nulib/lib_provisioning/vm/volume_management.nu index 53c117d..549b6f1 100644 --- a/nulib/lib_provisioning/vm/volume_management.nu +++ b/nulib/lib_provisioning/vm/volume_management.nu @@ -352,4 +352,4 @@ export def "volume-stats" []: record { def get-volumes-directory []: string { """Get volumes directory path""" "{{paths.workspace}}/vms/volumes" -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/webhook/ai_webhook.nu b/nulib/lib_provisioning/webhook/ai_webhook.nu index 47c7ce7..7351e96 100644 --- a/nulib/lib_provisioning/webhook/ai_webhook.nu +++ b/nulib/lib_provisioning/webhook/ai_webhook.nu @@ -14,7 +14,7 @@ export def ai_webhook_handler [ if $debug { print $"Debug: Received webhook payload: ($payload | to json)" } - + # Validate AI is enabled for webhooks let ai_config = (get_ai_config) if not $ai_config.enabled or not $ai_config.enable_webhook_ai { @@ -24,7 +24,7 @@ export def ai_webhook_handler [ response: "πŸ€– AI is currently disabled for webhook integrations" } } - + # Extract message and metadata based on platform let parsed = (parse_webhook_payload $payload $platform) @@ -119,7 +119,7 @@ def format_webhook_response [response: string, platform: string, context: record } } ] - + if ($context.thread_ts? != null) { { text: $response @@ -193,12 +193,12 @@ export def slack_webhook [payload: record, --debug] { challenge: $payload.challenge } } - + # Skip bot messages to prevent loops if ($payload.event?.bot_id? != null) or ($payload.bot_id? != null) { return { success: true, message: "Ignored bot message" } } - + ai_webhook_handler $payload --platform "slack" --debug $debug } @@ -208,7 +208,7 @@ export def discord_webhook [payload: record, --debug] { if ($payload.author?.bot? == true) { return { success: true, message: "Ignored bot message" } } - + ai_webhook_handler $payload --platform "discord" --debug $debug } @@ -218,7 +218,7 @@ export def teams_webhook [payload: record, --debug] { if ($payload.from?.name? | str contains "bot") { return { success: true, message: "Ignored bot message" } } - + ai_webhook_handler $payload --platform "teams" --debug $debug } @@ -236,21 +236,21 @@ export def start_webhook_server [ if not (is_ai_enabled) { error make {msg: "AI is not enabled - cannot start webhook server"} } - + let ai_config = (get_ai_config) if not $ai_config.enable_webhook_ai { error make {msg: "AI webhook processing is disabled"} } - + print $"πŸ€– Starting AI webhook server on ($host):($port)" print "Available endpoints:" print " POST /webhook/slack - Slack integration" - print " POST /webhook/discord - Discord integration" + print " POST /webhook/discord - Discord integration" print " POST /webhook/teams - Microsoft Teams integration" print " POST /webhook/generic - Generic webhook" print " GET /health - Health check" print "" - + # Note: This is a conceptual implementation # In practice, you'd use a proper web server print "⚠️ This is a conceptual webhook server." @@ -264,7 +264,7 @@ export def start_webhook_server [ export def webhook_health_check [] { let ai_config = (get_ai_config) let ai_test = (test_ai_connection) - + { status: "healthy" ai_enabled: $ai_config.enabled @@ -291,9 +291,9 @@ export def test_webhook [ timestamp: (date now | format date "%Y-%m-%d %H:%M:%S") test: true } - + let result = (ai_webhook_handler $payload --platform $platform --debug $debug) - + print $"Platform: ($platform)" print $"User: ($user)" print $"Channel: ($channel)" @@ -301,4 +301,4 @@ export def test_webhook [ print "" print "AI Response:" print $result.response -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/workspace/commands.nu b/nulib/lib_provisioning/workspace/commands.nu index c733c64..86e74b7 100644 --- a/nulib/lib_provisioning/workspace/commands.nu +++ b/nulib/lib_provisioning/workspace/commands.nu @@ -44,13 +44,13 @@ export def "workspace activate" [ return } - # Validate provisioning.k or provisioning.yaml exists - let provisioning_kcl = ($config_path | path join "provisioning.k") + # Validate provisioning.ncl or provisioning.yaml exists + let provisioning_nickel = ($config_path | path join "provisioning.ncl") let provisioning_yaml = ($config_path | path join "provisioning.yaml") - if not (($provisioning_kcl | path exists) or ($provisioning_yaml | path exists)) { + if not (($provisioning_nickel | path exists) or ($provisioning_yaml | path exists)) { print $"(ansi red)βœ—(ansi reset) Missing workspace configuration" - print $"(ansi yellow)πŸ’‘(ansi reset) Missing: ($provisioning_kcl) or ($provisioning_yaml)" + print $"(ansi yellow)πŸ’‘(ansi reset) Missing: ($provisioning_nickel) or ($provisioning_yaml)" print $"(ansi yellow)πŸ’‘(ansi reset) Run migration: provisioning workspace migrate ($workspace_name)" return } @@ -62,7 +62,7 @@ export def "workspace activate" [ if ($parsed.infra | is-not-empty) { # Validate infra exists let infra_path = ([$workspace_path "infra" $parsed.infra] | path join) - let settings_file = ([$infra_path "settings.k"] | path join) + let settings_file = ([$infra_path "settings.ncl"] | path join) if not ($settings_file | path exists) { print $"(ansi red)βœ—(ansi reset) Infrastructure '($parsed.infra)' not found in workspace '($workspace_name)'" diff --git a/nulib/lib_provisioning/workspace/config_commands.nu b/nulib/lib_provisioning/workspace/config_commands.nu index bcc73ea..1374dcb 100644 --- a/nulib/lib_provisioning/workspace/config_commands.nu +++ b/nulib/lib_provisioning/workspace/config_commands.nu @@ -52,10 +52,10 @@ def get-workspace-context [ } # Path exists but is not registered - check if it looks like a workspace - # Try both .k and .yaml config files - let config_file_kcl = ($input_as_path | path join "config" | path join "provisioning.k") + # Try both .ncl and .yaml config files + let config_file_nickel = ($input_as_path | path join "config" | path join "provisioning.ncl") let config_file_yaml = ($input_as_path | path join "config" | path join "provisioning.yaml") - if (($config_file_kcl | path exists) or ($config_file_yaml | path exists)) { + if (($config_file_nickel | path exists) or ($config_file_yaml | path exists)) { # It's a valid workspace directory, return it return { name: ($input_as_path | path basename) @@ -81,26 +81,26 @@ def get-workspace-context [ # Show complete workspace configuration export def "workspace-config-show" [ workspace_name?: string - --format: string = "yaml" # yaml, json, toml, kcl + --format: string = "yaml" # yaml, json, toml, nickel ] { let workspace = (get-workspace-context $workspace_name) - # Load complete config - try KCL first, fallback to YAML + # Load complete config - try Nickel first, fallback to YAML let config_dir = ($workspace.path | path join "config") - let kcl_file = ($config_dir | path join "provisioning.k") + let decl_file = ($config_dir | path join "provisioning.ncl") let yaml_file = ($config_dir | path join "provisioning.yaml") - # Try KCL first, but fallback to YAML if compilation fails (silently) - let config_file = if ($kcl_file | path exists) { - # Try KCL compilation (silently - we have YAML fallback) - let result = (^kcl eval $kcl_file 2>/dev/null | complete) + # Try Nickel first, but fallback to YAML if compilation fails (silently) + let config_file = if ($decl_file | path exists) { + # Try Nickel compilation (silently - we have YAML fallback) + let result = (^nickel export $decl_file --format json 2>/dev/null | complete) if ($result.stdout | is-not-empty) { - $kcl_file + $decl_file } else if ($yaml_file | path exists) { # Silently fallback to YAML $yaml_file } else { - $kcl_file + $decl_file } } else if ($yaml_file | path exists) { $yaml_file @@ -109,37 +109,37 @@ export def "workspace-config-show" [ } if ($config_file | is-empty) { - print "❌ No workspace configuration found (neither .k nor .yaml)" + print "❌ No workspace configuration found (neither .ncl nor .yaml)" exit 1 } # Load the config file - let config = if ($config_file | str ends-with ".k") { - # Load KCL config (outputs YAML by default) - # Check if kcl.mod exists in the same directory - if so, use 'kcl run' from that directory + let config = if ($config_file | str ends-with ".ncl") { + # Load Nickel config (outputs YAML by default) + # Check if nickel.mod exists in the same directory - if so, use 'nickel export' from that directory let file_dir = ($config_file | path dirname) let file_name = ($config_file | path basename) - let kcl_mod_exists = (($file_dir | path join "kcl.mod") | path exists) + let decl_mod_exists = (($file_dir | path join "nickel.mod") | path exists) - let result = if $kcl_mod_exists { - # Use 'kcl run' for package-based configs (SST pattern with kcl.mod) - # Must run from the config directory so relative paths in kcl.mod resolve correctly - (^sh -c $"cd '($file_dir)' && kcl run ($file_name)" | complete) + let result = if $decl_mod_exists { + # Use 'nickel export' for package-based configs (SST pattern with nickel.mod) + # Must run from the config directory so relative paths in nickel.mod resolve correctly + (^sh -c $"cd '($file_dir)' && nickel export ($file_name) --format json" | complete) } else { - # Use 'kcl eval' for standalone configs - (^kcl eval $config_file | complete) + # Use 'nickel export' for standalone configs + (^nickel export $config_file --format json | complete) } - let kcl_output = $result.stdout - if ($kcl_output | is-empty) { - print "❌ Failed to load KCL config: empty output" + let decl_output = $result.stdout + if ($decl_output | is-empty) { + print "❌ Failed to load Nickel config: empty output" if ($result.stderr | is-not-empty) { print $"Error: ($result.stderr)" } exit 1 } - # Parse YAML output and extract workspace_config if present - let parsed = ($kcl_output | from yaml) + # Parse JSON output and extract workspace_config if present + let parsed = ($decl_output | from json) if (($parsed | columns) | any { |col| $col == "workspace_config" }) { $parsed.workspace_config } else { @@ -151,7 +151,7 @@ export def "workspace-config-show" [ } # Determine config format type for display - let config_type = if ($config_file | str ends-with ".k") { "KCL" } else { "YAML" } + let config_type = if ($config_file | str ends-with ".ncl") { "Nickel" } else { "YAML" } # Output with format specified match $format { @@ -170,12 +170,12 @@ export def "workspace-config-show" [ print "" ($config | to toml) } - "kcl" => { - # Show raw KCL if available - if ($config_file | str ends-with ".k") { + "nickel" => { + # Show raw Nickel if available + if ($config_file | str ends-with ".ncl") { open $config_file } else { - print "ℹ️ Configuration is stored in YAML format, not KCL" + print "ℹ️ Configuration is stored in YAML format, not Nickel" print " Use --format=yaml to view the config" ($config | to json) } @@ -195,22 +195,22 @@ export def "workspace-config-validate" [ mut all_valid = true - # Check main config - try KCL first, fallback to YAML + # Check main config - try Nickel first, fallback to YAML let config_dir = ($workspace.path | path join "config") - let kcl_file = ($config_dir | path join "provisioning.k") + let decl_file = ($config_dir | path join "provisioning.ncl") let yaml_file = ($config_dir | path join "provisioning.yaml") - # Try KCL first, but fallback to YAML if compilation fails (silently) - let config_file = if ($kcl_file | path exists) { - # Try KCL compilation (silently - we have YAML fallback) - let result = (^kcl eval $kcl_file 2>/dev/null | complete) + # Try Nickel first, but fallback to YAML if compilation fails (silently) + let config_file = if ($decl_file | path exists) { + # Try Nickel compilation (silently - we have YAML fallback) + let result = (^nickel export $decl_file --format json 2>/dev/null | complete) if ($result.stdout | is-not-empty) { - $kcl_file + $decl_file } else if ($yaml_file | path exists) { # Silently fallback to YAML $yaml_file } else { - $kcl_file + $decl_file } } else if ($yaml_file | path exists) { $yaml_file @@ -220,36 +220,36 @@ export def "workspace-config-validate" [ if ($config_file | is-empty) { print "βœ“ Main config: (not found)" - print " ❌ No KCL (.k) or YAML (.yaml) config file found" + print " ❌ No Nickel (.ncl) or YAML (.yaml) config file found" $all_valid = false } else { - let config_type = if ($config_file | str ends-with ".k") { "KCL" } else { "YAML" } + let config_type = if ($config_file | str ends-with ".ncl") { "Nickel" } else { "YAML" } print $"βœ“ Main config: ($config_file) [($config_type)]" - let config = if ($config_file | str ends-with ".k") { - # Load KCL config (silently, with fallback handled above) - # Check if kcl.mod exists in the same directory - if so, use 'kcl run' from that directory + let config = if ($config_file | str ends-with ".ncl") { + # Load Nickel config (silently, with fallback handled above) + # Check if nickel.mod exists in the same directory - if so, use 'nickel export' from that directory let file_dir = ($config_file | path dirname) let file_name = ($config_file | path basename) - let kcl_mod_exists = (($file_dir | path join "kcl.mod") | path exists) + let decl_mod_exists = (($file_dir | path join "nickel.mod") | path exists) - let result = if $kcl_mod_exists { - # Use 'kcl run' for package-based configs (SST pattern with kcl.mod) - # Must run from the config directory so relative paths in kcl.mod resolve correctly - (^sh -c $"cd '($file_dir)' && kcl run ($file_name)" 2>/dev/null | complete) + let result = if $decl_mod_exists { + # Use 'nickel export' for package-based configs (SST pattern with nickel.mod) + # Must run from the config directory so relative paths in nickel.mod resolve correctly + (^sh -c $"cd '($file_dir)' && nickel export ($file_name) --format json" 2>/dev/null | complete) } else { - # Use 'kcl eval' for standalone configs - (^kcl eval $config_file 2>/dev/null | complete) + # Use 'nickel export' for standalone configs + (^nickel export $config_file --format json 2>/dev/null | complete) } - let kcl_output = $result.stdout - if ($kcl_output | is-empty) { - print $" ❌ KCL compilation failed, but YAML fallback not available" + let decl_output = $result.stdout + if ($decl_output | is-empty) { + print $" ❌ Nickel compilation failed, but YAML fallback not available" $all_valid = false {} } else { - # Parse YAML output and extract workspace_config if present - let parsed = ($kcl_output | from yaml) + # Parse JSON output and extract workspace_config if present + let parsed = ($decl_output | from json) if (($parsed | columns) | any { |col| $col == "workspace_config" }) { $parsed.workspace_config } else { @@ -262,8 +262,8 @@ export def "workspace-config-validate" [ } if ($config | is-not-empty) { - if ($config_file | str ends-with ".k") { - print " βœ… Valid KCL (schema validated)" + if ($config_file | str ends-with ".ncl") { + print " βœ… Valid Nickel (schema validated)" } else { print " βœ… Valid YAML" } @@ -567,4 +567,4 @@ export def "workspace-config-list" [ } $configs | table -} \ No newline at end of file +} diff --git a/nulib/lib_provisioning/workspace/detection.nu b/nulib/lib_provisioning/workspace/detection.nu index 94b5735..9f0850a 100644 --- a/nulib/lib_provisioning/workspace/detection.nu +++ b/nulib/lib_provisioning/workspace/detection.nu @@ -54,8 +54,8 @@ export def get-effective-workspace [] { export def detect-infra-from-pwd [] { let pwd = $env.PWD - # Check if we're directly in an infra directory by looking for settings.k - let settings_file = ([$pwd "settings.k"] | path join) + # Check if we're directly in an infra directory by looking for settings.ncl + let settings_file = ([$pwd "settings.ncl"] | path join) if ($settings_file | path exists) { return ($pwd | path basename) } diff --git a/nulib/lib_provisioning/workspace/enforcement.nu b/nulib/lib_provisioning/workspace/enforcement.nu index 3bd478e..c67892f 100644 --- a/nulib/lib_provisioning/workspace/enforcement.nu +++ b/nulib/lib_provisioning/workspace/enforcement.nu @@ -22,6 +22,17 @@ export def get-workspace-exempt-commands []: nothing -> list { "cache" "status" "health" + "setup" # ✨ System setup commands (workspace-agnostic) + "st" # Alias for setup + "config" # Alias for setup + "providers" # ✨ FIXED: provider list doesn't need workspace + "plugin" + "plugins" + "taskserv" # ✨ FIXED: taskserv list doesn't need workspace (list is read-only) + "task" + "server" # ✨ FIXED: server list is read-only + "cluster" # ✨ FIXED: cluster list is read-only + "infra" # ✨ FIXED: infra list is read-only "-v" "--version" "-V" diff --git a/nulib/lib_provisioning/workspace/generate_docs.nu b/nulib/lib_provisioning/workspace/generate_docs.nu new file mode 100644 index 0000000..e0e50cf --- /dev/null +++ b/nulib/lib_provisioning/workspace/generate_docs.nu @@ -0,0 +1,220 @@ +# Workspace Documentation Generator +# Generates deployment, configuration, and troubleshooting guides from Jinja2 templates +# Uses workspace metadata to populate guide variables + +def extract-workspace-metadata [workspace_path: string] { + { + workspace_path: $workspace_path, + config_path: $"($workspace_path)/config/config.ncl", + } +} + +def extract-workspace-name [metadata: record] { + cd $metadata.workspace_path + nickel export config/config.ncl | from json | get workspace.name +} + +def extract-provider-config [metadata: record] { + cd $metadata.workspace_path + let config = (nickel export config/config.ncl | from json) + let providers = $config.providers + + let provider_names = ($providers | columns) + let provider_list = ( + $provider_names + | each { |name| { name: $name, enabled: (($providers | get $name).enabled) } } + ) + + let first_enabled_provider = ( + $provider_list + | where enabled == true + | first + | get name + ) + + { + name: $first_enabled_provider, + enabled: true + } +} + +def extract-infrastructures [workspace_path: string] { + let infra_dir = $"($workspace_path)/infra" + + if ($infra_dir | path exists) { + ls $infra_dir + | where type == dir + | get name + | each { |path| $path | path basename } + } else { + [] + } +} + +def extract-servers [workspace_path: string, infra: string] { + let servers_file = $"($workspace_path)/infra/($infra)/servers.ncl" + + if ($servers_file | path exists) { + cd $workspace_path + let exported = (nickel export $"infra/($infra)/servers.ncl" | from json) + $exported.servers + } else { + [] + } +} + +def extract-taskservs [workspace_path: string, infra: string] { + let taskservs_dir = $"($workspace_path)/infra/($infra)/taskservs" + + if ($taskservs_dir | path exists) { + ls $taskservs_dir + | where name ends-with .ncl + | get name + | each { |path| $path | path basename | str replace --regex '\.ncl$' '' } + } else { + [] + } +} + +def generate-guide [template_path: string, output_path: string, variables: record] { + let output_dir = ($output_path | path dirname) + + if not ($output_dir | path exists) { + mkdir $output_dir + } + + let rendered = (tera-render $template_path $variables) + $rendered | save --force $output_path +} + +export def generate-all-guides [workspace_path: string, template_dir: string, output_dir: string] { + let metadata = (extract-workspace-metadata $workspace_path) + let workspace_name = (extract-workspace-name $metadata) + let provider_info = (extract-provider-config $metadata) + let all_infra = (extract-infrastructures $workspace_path) + # Filter out library/technical directories + let infrastructures = ($all_infra | where $it != "lib") + + let default_infra = if ($infrastructures | is-empty) { + "default" + } else { + $infrastructures | first + } + + let extracted_servers = (extract-servers $workspace_path $default_infra) + let taskservs = (extract-taskservs $workspace_path $default_infra) + + # Map server fields to template-friendly names + let servers = ( + $extracted_servers + | each { |srv| + let stg = if (($srv.storages | length) > 0) { + ($srv.storages | get 0).total + } else { + 0 + } + { name: $srv.hostname, plan: $srv.plan, storage: $stg, provider: $srv.provider, zone: $srv.zone } + } + ) + + let variables = { + workspace_name: $workspace_name, + workspace_path: $workspace_path, + workspace_description: "Workspace infrastructure deployment", + primary_provider: $provider_info.name, + primary_zone: "es-mad1", + alternate_zone: "nl-ams1", + default_infra: $default_infra, + providers: [$provider_info.name], + infrastructures: $infrastructures, + servers: $servers, + taskservs: $taskservs, + pricing_estimate: "€30-40/month", + provider_url: "https://hub.upcloud.com", + provider_api_url: "https://upcloud.com/api/", + provider_api_host: "api.upcloud.com", + provider_status_url: "https://status.upcloud.com", + provider_env_vars: { + "UPCLOUD_USER": "username", + "UPCLOUD_PASSWORD": "password", + }, + provider_defaults: { + "api_timeout": "30", + }, + provider_zone_defaults: { + "zone": "es-mad1", + "plan": "2xCPU-4GB", + }, + infrastructure_purposes: { + "wuji": "Kubernetes cluster for production workloads", + "sgoyol": "Development and testing environment", + }, + server_plans: [ + "1xCPU-1GB", + "1xCPU-2GB", + "2xCPU-4GB", + "2xCPU-8GB", + "4xCPU-8GB", + "4xCPU-16GB", + ], + available_zones: [ + "us-east-1", + "us-west-1", + "nl-ams1", + "es-mad1", + "fi-hel1", + ], + provider_config_example: { + "username": "your-username", + "password": "your-password", + "default-zone": "es-mad1", + }, + } + + print $"Generating guides for workspace: ($workspace_name)" + + let guides = [ + { + template: "deployment-guide.md.j2", + output: "deployment-guide.md", + }, + { + template: "configuration-guide.md.j2", + output: "configuration-guide.md", + }, + { + template: "troubleshooting.md.j2", + output: "troubleshooting.md", + }, + { + template: "README.md.j2", + output: "README.md", + }, + ] + + $guides + | each { |guide| + let template_path = $"($template_dir)/($guide.template)" + let output_path = $"($output_dir)/($guide.output)" + + print $" Generating ($guide.output)..." + + generate-guide $template_path $output_path $variables + } + + print "Documentation generation complete!" +} + +def main [workspace_path: string] { + # Get absolute paths - resolve from project root + let current_dir = (pwd) + let abs_workspace_path = (($workspace_path | path expand) | if (($in | path type) == relative) { ($"($current_dir)/($in)") } else { $in }) + let template_dir = ($"($current_dir)/provisioning/templates/docs" | path expand) + let output_dir = ($"($abs_workspace_path)/docs" | path expand) + + if not ($template_dir | path exists) { + print $"Template directory not found at ($template_dir)" + } else { + generate-all-guides $abs_workspace_path $template_dir $output_dir + } +} diff --git a/nulib/lib_provisioning/workspace/helpers.nu b/nulib/lib_provisioning/workspace/helpers.nu index e5ec64f..c64eb21 100644 --- a/nulib/lib_provisioning/workspace/helpers.nu +++ b/nulib/lib_provisioning/workspace/helpers.nu @@ -170,7 +170,7 @@ export def get-infra-options [workspace_name: string] { for entry in ($entries | lines) { let entry_path = ([$infra_base $entry] | path join) if ($entry_path | path exists) { - let settings = ([$entry_path "settings.k"] | path join) + let settings = ([$entry_path "settings.ncl"] | path join) if ($settings | path exists) { $infras = ($infras | append $entry) } diff --git a/nulib/lib_provisioning/workspace/init.nu b/nulib/lib_provisioning/workspace/init.nu index c3f56c0..c383b1d 100644 --- a/nulib/lib_provisioning/workspace/init.nu +++ b/nulib/lib_provisioning/workspace/init.nu @@ -4,11 +4,15 @@ # name = "workspace init" # group = "workspace" # tags = ["workspace", "initialize", "interactive"] -# version = "2.0.0" -# requires = ["forminquire.nu:1.0.0", "nushell:0.109.0"] -# note = "Migrated to FormInquire with fallback to prompt-based input" +# version = "3.0.0" +# requires = ["nushell:0.109.0"] +# note = "MIGRATION: ForminQuire (Jinja2 templates) archived. Use TypeDialog forms instead" +# migration = "See: provisioning/.coder/archive/forminquire/ (deprecated) β†’ provisioning/.typedialog/provisioning/form.toml (new)" -use ../../../forminquire/nulib/forminquire.nu * +# ARCHIVED: use ../../../forminquire/nulib/forminquire.nu * +# ForminQuire has been archived to: provisioning/.coder/archive/forminquire/ +# New solution: Use TypeDialog for interactive forms (typedialog, typedialog-tui, typedialog-web) +use ../utils/interface.nu * # Interactive workspace creation with activation prompt export def workspace-init-interactive [] { @@ -124,91 +128,100 @@ export def workspace-init [ } } - # 2. Copy KCL modules from provisioning and create workspace-specific config - _print "\nπŸ“ Setting up KCL modules and configuration..." + # 2. Create Nickel-based configuration + _print "\nπŸ“ Setting up Nickel configuration..." let created_timestamp = (date now | format date "%Y-%m-%dT%H:%M:%SZ") - let templates_dir = "/Users/Akasha/project-provisioning/provisioning/config/templates" - let provisioning_kcl_dir = "/Users/Akasha/project-provisioning/provisioning/kcl" + let provisioning_root = "/Users/Akasha/project-provisioning/provisioning" - # 2a. Copy .kcl directory from provisioning/kcl (contains all KCL modules) - if ($provisioning_kcl_dir | path exists) { - let workspace_kcl_dir = $"($workspace_path)/.kcl" + # 2a. Create config/config.ncl (master workspace configuration) + let owner_name = $env.USER + let config_ncl_content = $"# Workspace Configuration - ($workspace_name) +# Master configuration file for infrastructure and providers +# Format: Nickel (IaC configuration language) - # Use cp -r to recursively copy entire directory - cp -r $provisioning_kcl_dir $workspace_kcl_dir - _print $" βœ… Copied: provisioning/kcl β†’ .kcl/" - } else { - _print $" ⚠️ Warning: Provisioning kcl directory not found at ($provisioning_kcl_dir)" - } +{ + workspace = { + name = \"($workspace_name)\", + path = \"($workspace_path)\", + description = \"Workspace: ($workspace_name)\", + metadata = { + owner = \"($owner_name)\", + created = \"($created_timestamp)\", + environment = \"development\", + }, + }, - # 2b. Create metadata.yaml in .provisioning (metadata only, no KCL files) - let metadata_template_path = $"($templates_dir)/metadata.yaml.template" - if ($metadata_template_path | path exists) { - let metadata_content = ( - open $metadata_template_path - | str replace --all "{{WORKSPACE_NAME}}" $workspace_name - | str replace --all "{{WORKSPACE_CREATED_AT}}" $created_timestamp - ) - $metadata_content | save -f $"($workspace_path)/.provisioning/metadata.yaml" - _print $" βœ… Created: .provisioning/metadata.yaml" - } else { - _print $" ⚠️ Warning: Metadata template not found at ($metadata_template_path)" - } + providers = { + local = { + name = \"local\", + enabled = true, + workspace = \"($workspace_name)\", + auth = { + interface = \"local\", + }, + paths = { + base = \".providers/local\", + cache = \".providers/local/cache\", + state = \".providers/local/state\", + }, + }, + }, +} +" + $config_ncl_content | save -f $"($workspace_path)/config/config.ncl" + _print $" βœ… Created: config/config.ncl" - # 2c. Create config/kcl.mod from template (workspace config package) - let config_kcl_mod_template_path = $"($templates_dir)/config-kcl.mod.template" - if ($config_kcl_mod_template_path | path exists) { - let config_kcl_mod_content = (open $config_kcl_mod_template_path) - $config_kcl_mod_content | save -f $"($workspace_path)/config/kcl.mod" - _print $" βœ… Created: config/kcl.mod" - } else { - _print $" ⚠️ Warning: Config kcl.mod template not found" - } + # 2b. Create metadata.yaml in .provisioning + let metadata_content = $"workspace_name: \"($workspace_name)\" +workspace_path: \"($workspace_path)\" +created_at: \"($created_timestamp)\" +version: \"1.0.0\" +" + $metadata_content | save -f $"($workspace_path)/.provisioning/metadata.yaml" + _print $" βœ… Created: .provisioning/metadata.yaml" - # 2d. Create config/provisioning.k from workspace config template (workspace-specific override) - let workspace_config_template_path = $"($templates_dir)/workspace-config.k.template" - let kcl_config_content = ( - open $workspace_config_template_path - | str replace --all "{{WORKSPACE_NAME}}" $workspace_name - | str replace --all "{{WORKSPACE_PATH}}" $workspace_path - | str replace --all "{{PROVISIONING_PATH}}" "/Users/Akasha/project-provisioning/provisioning" - | str replace --all "{{CREATED_TIMESTAMP}}" $created_timestamp - | str replace --all "{{INFRA_NAME}}" "default" - ) - $kcl_config_content | save -f $"($workspace_path)/config/provisioning.k" - _print $" βœ… Created: config/provisioning.k \(Workspace Override\)" + # 2c. Create infra/default directory and Nickel infrastructure files + mkdir $"($workspace_path)/infra/default" - # 2e. Create workspace root kcl.mod from template - let root_kcl_mod_template_path = $"($templates_dir)/kcl.mod.template" - if ($root_kcl_mod_template_path | path exists) { - let root_kcl_mod_content = ( - open $root_kcl_mod_template_path - | str replace --all "{{WORKSPACE_NAME}}" $workspace_name - ) - $root_kcl_mod_content | save -f $"($workspace_path)/kcl.mod" - _print $" βœ… Created: kcl.mod" - } else { - _print $" ⚠️ Warning: Root kcl.mod template not found" - } + let infra_main_ncl = $"# Default Infrastructure Configuration +# Entry point for infrastructure deployment - # 2f. Create platform target configuration for services - _print "\n🌐 Creating platform services configuration..." - let platform_config_dir = $"($workspace_path)/config/platform" - mkdir $platform_config_dir +{ + workspace_name = \"($workspace_name)\", + infrastructure = \"default\", - let platform_target_template_path = $"($templates_dir)/platform-target.yaml.template" - if ($platform_target_template_path | path exists) { - let platform_target_content = ( - open $platform_target_template_path - | str replace --all "{{WORKSPACE_NAME}}" $workspace_name - ) - $platform_target_content | save -f $"($platform_config_dir)/target.yaml" - _print $" βœ… Created: config/platform/target.yaml" - } else { - _print $" ⚠️ Warning: Platform target template not found" - } + servers = [ + { + hostname = \"($workspace_name)-server-0\", + provider = \"local\", + plan = \"1xCPU-2GB\", + zone = \"local\", + storages = [{total = 25}], + }, + ], +} +" + $infra_main_ncl | save -f $"($workspace_path)/infra/default/main.ncl" + _print $" βœ… Created: infra/default/main.ncl" - # 2g. Create .platform directory for runtime connection metadata + let infra_servers_ncl = $"# Server Definitions for Default Infrastructure + +{ + servers = [ + { + hostname = \"($workspace_name)-server-0\", + provider = \"local\", + plan = \"1xCPU-2GB\", + zone = \"local\", + storages = [{total = 25}], + }, + ], +} +" + $infra_servers_ncl | save -f $"($workspace_path)/infra/default/servers.ncl" + _print $" βœ… Created: infra/default/servers.ncl" + + # 2d. Create .platform directory for runtime connection metadata mkdir $"($workspace_path)/.platform" _print $" βœ… Created: .platform/" @@ -247,6 +260,19 @@ export def workspace-init [ # 7. Create .gitignore for workspace create-workspace-gitignore $workspace_path + # 8. Generate workspace documentation (deployment, configuration, troubleshooting guides) + _print "\nπŸ“š Generating documentation..." + use ./generate_docs.nu * + let template_dir = "/Users/Akasha/project-provisioning/provisioning/templates/docs" + let output_dir = $"($workspace_path)/docs" + + if ($template_dir | path exists) { + generate-all-guides $workspace_path $template_dir $output_dir + _print $" βœ… Generated workspace documentation in ($output_dir)" + } else { + _print $" ⚠️ Documentation templates not found at ($template_dir)" + } + _print $"\nβœ… Workspace '($workspace_name)' initialized successfully!" _print $"\nπŸ“‹ Workspace Summary:" _print $" Name: ($workspace_name)" @@ -256,6 +282,7 @@ export def workspace-init [ if ($platform_services | is-not-empty) { _print $" Platform: ($platform_services | str join ', ')" } + _print $" Docs: ($workspace_path)/docs" _print "" # Use intelligent hints system for next steps diff --git a/nulib/lib_provisioning/workspace/migrate_to_kcl.nu b/nulib/lib_provisioning/workspace/migrate_to_kcl.nu index 161522d..c21c0a2 100644 --- a/nulib/lib_provisioning/workspace/migrate_to_kcl.nu +++ b/nulib/lib_provisioning/workspace/migrate_to_kcl.nu @@ -1,10 +1,10 @@ -# Workspace Configuration Migration: YAML β†’ KCL -# Converts existing provisioning.yaml workspace configs to KCL format +# Workspace Configuration Migration: YAML β†’ Nickel +# Converts existing provisioning.yaml workspace configs to Nickel format use ../config/accessor.nu * # ============================================================================ -# Convert YAML Workspace Config to KCL +# Convert YAML Workspace Config to Nickel # ============================================================================ export def migrate-config [ @@ -12,7 +12,7 @@ export def migrate-config [ --all # Migrate all workspaces --backup # Create backups of original YAML files --check # Check mode (show what would be done) - --force # Force migration even if KCL file exists + --force # Force migration even if Nickel file exists --verbose # Verbose output ] { # Validate inputs @@ -88,12 +88,12 @@ def migrate_single_workspace [ --verbose: bool ] { let yaml_file = ($workspace_path | path join "config" | path join "provisioning.yaml") - let kcl_file = ($workspace_path | path join "config" | path join "provisioning.k") + let decl_file = ($workspace_path | path join "config" | path join "provisioning.ncl") if $verbose { print $"Processing workspace: ($workspace_name)" print $" YAML: ($yaml_file)" - print $" KCL: ($kcl_file)" + print $" Nickel: ($decl_file)" } # Check if YAML config exists @@ -109,16 +109,16 @@ def migrate_single_workspace [ } } - # Check if KCL file already exists - if ($kcl_file | path exists) and (not $force) { + # Check if Nickel file already exists + if ($decl_file | path exists) and (not $force) { if $verbose { - print $" ⚠️ KCL file already exists, skipping (use --force to overwrite)" + print $" ⚠️ Nickel file already exists, skipping (use --force to overwrite)" } return { workspace: $workspace_name success: false skipped: true - error: "KCL file already exists" + error: "Nickel file already exists" } } @@ -137,9 +137,9 @@ def migrate_single_workspace [ } } - # Convert YAML to KCL - let kcl_content = try { - yaml_to_kcl $yaml_config $workspace_name + # Convert YAML to Nickel + let nickel_content = try { + yaml_to_nickel $yaml_config $workspace_name } catch {|e| if $verbose { print $" ❌ Conversion failed: ($e)" @@ -154,10 +154,10 @@ def migrate_single_workspace [ if $check { if $verbose { - print $" [CHECK MODE] Would write ($kcl_file | str length) characters" + print $" [CHECK MODE] Would write ($decl_file | str length) characters" print "" - print "Generated KCL (first 500 chars):" - print ($kcl_content | str substring [0 500]) + print "Generated Nickel (first 500 chars):" + print ($nickel_content | str substring [0 500]) print "..." } return { @@ -183,22 +183,22 @@ def migrate_single_workspace [ } } - # Write KCL file + # Write Nickel file try { - $kcl_content | save $kcl_file + $nickel_content | save $decl_file if $verbose { - print $" βœ… Created ($kcl_file)" + print $" βœ… Created ($decl_file)" } - # Validate KCL + # Validate Nickel try { - let _ = (kcl eval $kcl_file) + let _ = (nickel export $decl_file --format json) if $verbose { - print $" βœ… KCL validation passed" + print $" βœ… Nickel validation passed" } } catch { if $verbose { - print $" ⚠️ KCL validation warning (may still be usable)" + print $" ⚠️ Nickel validation warning (may still be usable)" } } @@ -210,27 +210,27 @@ def migrate_single_workspace [ } } catch {|e| if $verbose { - print $" ❌ Failed to write KCL file: ($e)" + print $" ❌ Failed to write Nickel file: ($e)" } return { workspace: $workspace_name success: false skipped: false - error: $"Failed to write KCL file: ($e)" + error: $"Failed to write Nickel file: ($e)" } } } # ============================================================================ -# YAML to KCL Conversion +# YAML to Nickel Conversion # ============================================================================ -def yaml_to_kcl [ +def yaml_to_nickel [ yaml_config: record workspace_name: string ] { - # Start building KCL structure - let kcl_lines = [ + # Start building Nickel structure + let nickel_lines = [ '"""' 'Workspace Configuration' 'Auto-generated from provisioning.yaml' @@ -338,12 +338,12 @@ def yaml_to_kcl [ let cache_section = ' cache: { path: "" }' let infra_section = ' infra: {}' let tools_section = ' tools: {}' - let kcl_section = ' kcl: {}' + let nickel_section = ' nickel: {}' let ssh_section = ' ssh: {}' - # Assemble final KCL - let kcl_content = ([ - ...$kcl_lines + # Assemble final Nickel + let nickel_content = ([ + ...$nickel_lines '' $workspace_section '' @@ -383,11 +383,11 @@ def yaml_to_kcl [ '' $tools_section '' - $kcl_section + $nickel_section '' $ssh_section '}' ] | str join "\n") - $kcl_content + $nickel_content } diff --git a/nulib/lib_provisioning/workspace/notation.nu b/nulib/lib_provisioning/workspace/notation.nu index 72dc3ba..4deaac5 100644 --- a/nulib/lib_provisioning/workspace/notation.nu +++ b/nulib/lib_provisioning/workspace/notation.nu @@ -45,7 +45,7 @@ def infra-exists? [workspace_name: string, infra_name: string] { let workspace_path = ($workspace.path) let infra_path = ([$workspace_path "infra" $infra_name] | path join) - let settings_file = ([$infra_path "settings.k"] | path join) + let settings_file = ([$infra_path "settings.ncl"] | path join) ($settings_file | path exists) } diff --git a/nulib/lib_provisioning/workspace/sync.nu b/nulib/lib_provisioning/workspace/sync.nu index f3c43a1..29c54f2 100644 --- a/nulib/lib_provisioning/workspace/sync.nu +++ b/nulib/lib_provisioning/workspace/sync.nu @@ -64,7 +64,7 @@ export def "workspace update" [ {name: ".providers", source: ($prov_root | path join "provisioning/extensions/providers"), target: ($workspace_path | path join ".providers")} {name: ".clusters", source: ($prov_root | path join "provisioning/extensions/clusters"), target: ($workspace_path | path join ".clusters")} {name: ".taskservs", source: ($prov_root | path join "provisioning/extensions/taskservs"), target: ($workspace_path | path join ".taskservs")} - {name: ".kcl", source: ($prov_root | path join "provisioning/kcl"), target: ($workspace_path | path join ".kcl")} + {name: ".nickel", source: ($prov_root | path join "provisioning/nickel"), target: ($workspace_path | path join ".nickel")} ] # Show plan @@ -122,9 +122,9 @@ export def "workspace update" [ cp -r $update.source $update.target print $" (ansi green)βœ“(ansi reset) Updated: ($update.name)" - # Fix KCL module paths after copy (for providers) + # Fix Nickel module paths after copy (for providers) if ($update.name == ".providers") { - _fix-provider-kcl-paths $update.target $verbose + _fix-provider-nickel-paths $update.target $verbose } } @@ -132,19 +132,19 @@ export def "workspace update" [ print $"(ansi green)βœ“(ansi reset) Workspace update complete" } -# Helper: Fix kcl.mod paths in copied providers -def _fix-provider-kcl-paths [ +# Helper: Fix nickel.mod paths in copied providers +def _fix-provider-nickel-paths [ providers_path: string verbose: bool ]: nothing -> nothing { - # Find all kcl.mod files in provider subdirectories - let kcl_mods = (glob $"($providers_path)/**/kcl.mod") + # Find all nickel.mod files in provider subdirectories + let nickel_mods = (glob $"($providers_path)/**/nickel.mod") - if ($kcl_mods | is-empty) { + if ($nickel_mods | is-empty) { return } - for mod_file in $kcl_mods { + for mod_file in $nickel_mods { if not ($mod_file | path exists) { continue } @@ -153,19 +153,19 @@ def _fix-provider-kcl-paths [ let content = (open $mod_file) # Fix provider paths to correct relative paths - # Providers are in infra//.providers//kcl/kcl.mod - # Should reference: ../../../.kcl/packages/provisioning + # Providers are in infra//.providers//nickel/nickel.mod + # Should reference: ../../../.nickel/packages/provisioning let updated = ( $content - | str replace --all '{ path = "../../../../kcl' '{ path = "../../../.kcl/packages/provisioning' - | str replace --all '{ path = "../../../../.kcl' '{ path = "../../../.kcl/packages/provisioning' + | str replace --all '{ path = "../../../../nickel' '{ path = "../../../.nickel/packages/provisioning' + | str replace --all '{ path = "../../../../.nickel' '{ path = "../../../.nickel/packages/provisioning' ) # Only write if content changed if ($content != $updated) { $updated | save -f $mod_file if $verbose { - print $" Fixed KCL path in: ($mod_file)" + print $" Fixed Nickel path in: ($mod_file)" } } } @@ -205,7 +205,7 @@ export def "workspace check-updates" [ {name: ".providers", path: ($workspace_path | path join ".providers")} {name: ".clusters", path: ($workspace_path | path join ".clusters")} {name: ".taskservs", path: ($workspace_path | path join ".taskservs")} - {name: ".kcl", path: ($workspace_path | path join ".kcl")} + {name: ".nickel", path: ($workspace_path | path join ".nickel")} {name: "config", path: ($workspace_path | path join "config")} ] diff --git a/nulib/lib_provisioning/workspace/version.nu b/nulib/lib_provisioning/workspace/version.nu index 4d5a23f..0b4a346 100644 --- a/nulib/lib_provisioning/workspace/version.nu +++ b/nulib/lib_provisioning/workspace/version.nu @@ -316,16 +316,16 @@ export def validate-workspace-structure [ } } - # Check for required files (KCL or YAML configuration) - let config_kcl_path = ($workspace_path | path join "config" | path join "provisioning.k") + # Check for required files (Nickel or YAML configuration) + let config_schema_path = ($workspace_path | path join "config" | path join "provisioning.ncl") let config_yaml_path = ($workspace_path | path join "config" | path join "provisioning.yaml") - if (not ($config_kcl_path | path exists) and not ($config_yaml_path | path exists)) { + if (not ($config_schema_path | path exists) and not ($config_yaml_path | path exists)) { $issues = ($issues | append { type: "missing_file" - path: "config/provisioning.k or config/provisioning.yaml" + path: "config/provisioning.ncl or config/provisioning.yaml" severity: "error" - message: "Main configuration file missing (provisioning.k or provisioning.yaml required)" + message: "Main configuration file missing (provisioning.ncl or provisioning.yaml required)" }) } diff --git a/nulib/libremote.nu b/nulib/libremote.nu index 4c2e71b..25b78b9 100644 --- a/nulib/libremote.nu +++ b/nulib/libremote.nu @@ -1,9 +1,9 @@ export def _ansi [ arg: string ]: nothing -> string { - if (is-terminal --stdout) { + if (is-terminal --stdout) { $"(ansi $arg)" - } else { + } else { "" } } @@ -13,7 +13,7 @@ export def log_debug [ ]: nothing -> nothing { use std std log debug $msg -} +} export def format_out [ data: string @@ -24,7 +24,7 @@ export def format_out [ "json" => ($data | from json), _ => $data, } - match $mode { + match $mode { "table" => { ($msg | table -i false) }, @@ -39,8 +39,8 @@ export def _print [ ]: nothing -> nothing { if ($env.PROVISIONING_OUT | is-empty) { print (format_out $data $src $mode) - } else { - match $env.PROVISIONING_OUT { + } else { + match $env.PROVISIONING_OUT { "json" => { if $context != "result" { return } if $src == "json" { @@ -85,4 +85,4 @@ export def _print [ } } } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/ai.nu b/nulib/main_provisioning/ai.nu index e6559e6..8094470 100644 --- a/nulib/main_provisioning/ai.nu +++ b/nulib/main_provisioning/ai.nu @@ -65,7 +65,7 @@ def ai_template_command [ if ($prompt | is-empty) { error make {msg: "AI template generation requires --prompt"} } - + let result = (ai_generate_template $prompt $template_type) print $"# AI Generated ($template_type) Template" print $"# Prompt: ($prompt)" @@ -82,7 +82,7 @@ def ai_query_command [ if ($prompt | is-empty) { error make {msg: "AI query requires --prompt"} } - + let context_data = if ($context | is-empty) { {} } else { @@ -92,7 +92,7 @@ def ai_query_command [ {raw_context: $context} } } - + let result = (ai_process_query $prompt $context_data) print $result } @@ -105,10 +105,10 @@ def ai_webhook_command [ if ($prompt | is-empty) { error make {msg: "AI webhook processing requires --prompt"} } - + let user_id = if ($args | length) > 0 { $args.0 } else { "cli" } let channel = if ($args | length) > 1 { $args.1 } else { "direct" } - + let result = (ai_process_webhook $prompt $user_id $channel) print $result } @@ -116,7 +116,7 @@ def ai_webhook_command [ # Test AI connectivity and configuration def ai_test_command [] { print "Testing AI configuration..." - + let validation = (validate_ai_config) if not $validation.valid { print "❌ AI configuration issues found:" @@ -125,9 +125,9 @@ def ai_test_command [] { } return } - + print "βœ… AI configuration is valid" - + let test_result = (test_ai_connection) if $test_result.success { print $"βœ… ($test_result.message)" @@ -142,7 +142,7 @@ def ai_test_command [] { # Show AI configuration def ai_config_command [] { let config = (get_ai_config) - + print "πŸ€– AI Configuration:" print $" Enabled: ($config.enabled)" print $" Provider: ($config.provider)" @@ -155,7 +155,7 @@ def ai_config_command [] { print $" Template AI: ($config.enable_template_ai)" print $" Query AI: ($config.enable_query_ai)" print $" Webhook AI: ($config.enable_webhook_ai)" - + if $config.enabled and ($config.api_key? == null) { print "" print "⚠️ API key not configured" @@ -168,7 +168,7 @@ def ai_config_command [] { # Enable AI functionality def ai_enable_command [] { - print "AI functionality can be enabled by setting ai.enabled = true in your KCL settings" + print "AI functionality can be enabled by setting ai.enabled = true in your Nickel settings" print "Example configuration:" print "" print "ai: AIProvider {" @@ -186,7 +186,7 @@ def ai_enable_command [] { # Disable AI functionality def ai_disable_command [] { - print "AI functionality can be disabled by setting ai.enabled = false in your KCL settings" + print "AI functionality can be disabled by setting ai.enabled = false in your Nickel settings" print "This will disable all AI features while preserving configuration." } @@ -244,9 +244,9 @@ export def ai_generate [ if ($prompt | is-empty) { error make {msg: "AI generation requires --prompt"} } - + let result = (ai_generate_template $prompt $template_type) - + if ($output | is-empty) { print $result } else { @@ -267,9 +267,9 @@ export def ai_query_infra [ provider: ($provider | default "") output_format: $output_format } - + let result = (ai_process_query $query $context) - + match $output_format { "json" => { {query: $query, response: $result} | to json } "yaml" => { {query: $query, response: $result} | to yaml } @@ -428,4 +428,4 @@ def enhanced_ai_help_command [] { print " β€’ Predictive analytics" print " β€’ Interactive chat mode" print " β€’ Batch query processing" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/api.nu b/nulib/main_provisioning/api.nu index 36e9074..0b87e7e 100644 --- a/nulib/main_provisioning/api.nu +++ b/nulib/main_provisioning/api.nu @@ -315,4 +315,4 @@ ENDPOINTS: For more information, visit: https://docs.provisioning.dev/api " -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/batch.nu b/nulib/main_provisioning/batch.nu index 076302c..d564bf3 100644 --- a/nulib/main_provisioning/batch.nu +++ b/nulib/main_provisioning/batch.nu @@ -16,4 +16,4 @@ export def "main batch" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "batch" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/configuration.nu b/nulib/main_provisioning/commands/configuration.nu index 85ca067..282950c 100644 --- a/nulib/main_provisioning/commands/configuration.nu +++ b/nulib/main_provisioning/commands/configuration.nu @@ -18,6 +18,11 @@ export def handle_config_command [ "init" => { handle_init $ops $flags } "validate" | "val" => { handle_validate $ops $flags } "config-template" => { handle_config_template $ops $flags } + "export" => { handle_config_export $ops $flags } + "workspace" | "ws" => { handle_config_workspace $ops $flags } + "platform" | "plat" => { handle_config_platform $ops $flags } + "providers" | "prov" => { handle_config_providers $ops $flags } + "services" | "svc" => { handle_config_services $ops $flags } _ => { print $"❌ Unknown configuration command: ($command)" print "" @@ -28,13 +33,21 @@ export def handle_config_command [ print " init - Initialize infrastructure configuration" print " validate - Validate configuration" print " config-template - Generate config template" + print " export - Export Nickel config to TOML" + print " workspace - Configure workspace settings" + print " platform - Configure platform services" + print " providers - List/manage providers" + print " services - List/manage platform services" print "" - print "Environment subcommands:" - print " env list - List all environments" - print " env current - Show current environment" - print " env switch - Switch to environment" - print " env show [env] - Show environment details" - print " env validate [env] - Validate environment" + print "Configuration subcommands:" + print " config export - Export all configs" + print " config export - Export specific service" + print " config validate - Validate Nickel config" + print " config workspace info - Show workspace info" + print " config platform orchestrator - Configure orchestrator" + print " config platform kms - Configure KMS" + print " config providers list - List all providers" + print " config services list - List all services" print "" print "Use 'provisioning help configuration' for more details" exit 1 @@ -337,4 +350,283 @@ def handle_config_template [ops: string, flags: record] { print "❌ Unknown config-template command. Use 'provisioning config-template help' for available options." } } -} \ No newline at end of file +} + +# Config export handler - Exports Nickel config to TOML for services +def handle_config_export [ops: string, flags: record] { + use ../../lib_provisioning/config/export.nu * + + let service = if ($ops | is-empty) { "" } else { $ops | split row " " | first } + + print "πŸ“¦ Exporting Configuration" + print "==========================" + print "" + + if ($service | is-empty) { + # Export all configs + print "πŸ”„ Exporting all configuration sections..." + print "" + export-all-configs + print "βœ… Configuration export complete" + print "" + print "Generated files:" + print " β€’ workspace_librecloud/config/generated/workspace.toml" + print " β€’ workspace_librecloud/config/generated/providers/*.toml" + print " β€’ workspace_librecloud/config/generated/platform/*.toml" + } else { + # Export specific service + print $"πŸ”„ Exporting platform service: ($service)..." + export-platform-config $service + print $"βœ… Exported: workspace_librecloud/config/generated/platform/($service).toml" + } +} + +# Config workspace handler - Configure workspace settings +def handle_config_workspace [ops: string, flags: record] { + let subcmd = if ($ops | is-empty) { "" } else { $ops | split row " " | first } + + match $subcmd { + "info" => { + use ../../lib_provisioning/config/export.nu * + + print "πŸ“‹ Workspace Information" + print "=======================" + print "" + + show-config + } + "validate" => { + use ../../lib_provisioning/config/export.nu * + + print "βœ“ Validating workspace configuration..." + let result = validate-config + if $result.valid { + print "βœ… Configuration is valid" + } else { + print $"❌ Configuration validation failed: ($result.error)" + exit 1 + } + } + "help" | "h" => { + print "πŸ“‹ Workspace Configuration Commands" + print "====================================" + print "" + print "Commands:" + print " config workspace info - Show workspace information" + print " config workspace validate - Validate workspace configuration" + print "" + print "Examples:" + print " provisioning config workspace info" + print " provisioning config workspace validate" + } + _ => { + print "❌ Unknown workspace command. Use 'provisioning config workspace help' for available options." + } + } +} + +# Config platform handler - Configure platform services +def handle_config_platform [ops: string, flags: record] { + let service = if ($ops | is-empty) { "" } else { $ops | split row " " | first } + + match $service { + "orchestrator" => { + print "βš™οΈ Configuring Orchestrator Service" + print "====================================" + print "" + print "To configure the orchestrator interactively:" + print "" + print "Option 1: Use TypeDialog (interactive form)" + print " provisioning-dialog ~/.typedialog/provisioning/platform/orchestrator/form.toml" + print "" + print "Option 2: Edit configuration directly" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: platform.orchestrator" + print "" + print "Option 3: Export existing configuration" + print " provisioning config export orchestrator" + print "" + print "Then verify:" + print " provisioning config validate" + } + "kms" => { + print "πŸ” Configuring KMS Service" + print "==========================" + print "" + print "Edit KMS configuration:" + print " workspace_librecloud/config/config.ncl" + print " Section: platform.kms" + print "" + print "Available KMS backends:" + print " β€’ rustyvault - RustyVault KMS" + print " β€’ age - Age encryption" + print " β€’ aws - AWS KMS" + print " β€’ vault - HashiCorp Vault" + print " β€’ cosmian - Cosmian KMS" + } + "control-center" => { + print "πŸŽ›οΈ Configuring Control Center Service" + print "======================================" + print "" + print "To configure the control center interactively:" + print "" + print "Option 1: Use TypeDialog (interactive form)" + print " typedialog form .typedialog/provisioning/platform/control-center/form.toml" + print "" + print "Option 2: Edit configuration directly" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: platform.control_center" + print "" + print "Then verify:" + print " provisioning config validate" + print "" + print "Control Center manages:" + print " β€’ Admin interface and web UI" + print " β€’ User authentication (JWT)" + print " β€’ Rate limiting and CORS" + print " β€’ Session management" + } + "mcp-server" => { + print "πŸ”Œ Configuring MCP Server Service" + print "==================================" + print "" + print "To configure the MCP server interactively:" + print "" + print "Option 1: Use TypeDialog (interactive form)" + print " typedialog form .typedialog/provisioning/platform/mcp-server/form.toml" + print "" + print "Option 2: Edit configuration directly" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: platform.mcp_server" + print "" + print "Then verify:" + print " provisioning config validate" + print "" + print "MCP Server provides:" + print " β€’ Model Context Protocol integration" + print " β€’ Tool and prompt management" + print " β€’ Resource caching" + print " β€’ AI assistant integration" + } + "installer" => { + print "πŸš€ Configuring Installer Service" + print "=================================" + print "" + print "To configure the installer interactively:" + print "" + print "Option 1: Use TypeDialog (interactive form)" + print " typedialog form .typedialog/provisioning/platform/installer/form.toml" + print "" + print "Option 2: Edit configuration directly" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: platform.installer" + print "" + print "Then verify:" + print " provisioning config validate" + print "" + print "Installer configures:" + print " β€’ Deployment mode (solo, multiuser, cicd, enterprise)" + print " β€’ Container platform (docker, podman, kubernetes)" + print " β€’ Service selection and enablement" + print " β€’ Resource allocation" + print " β€’ High availability settings" + } + "help" | "h" | "" => { + print "πŸ“‹ Platform Service Configuration Commands" + print "==========================================" + print "" + print "Commands:" + print " config platform orchestrator - Configure orchestrator service" + print " config platform control-center - Configure control center UI" + print " config platform mcp-server - Configure MCP server" + print " config platform installer - Configure installer" + print " config platform kms - Configure KMS service" + print "" + print "For more details:" + print " provisioning config platform " + print "" + print "Interactive Configuration (Recommended):" + print " typedialog form .typedialog/provisioning/platform//form.toml" + } + _ => { + print $"❌ Unknown platform service: ($service)" + print "" + print "Available services: orchestrator, control-center, mcp-server, vault-service, extension-registry, rag, ai-service, provisioning-daemon" + print "" + print "Use 'provisioning config platform help' for more information" + } + } +} + +# Config providers handler - List and manage providers +def handle_config_providers [ops: string, flags: record] { + use ../../lib_provisioning/config/export.nu * + + let subcmd = if ($ops | is-empty) { "" } else { $ops | split row " " | first } + + match $subcmd { + "list" => { + print "☁️ Configured Cloud Providers" + print "==============================" + print "" + + list-providers + } + "help" | "h" | "" => { + print "πŸ“‹ Provider Configuration Commands" + print "==================================" + print "" + print "Commands:" + print " config providers list - List all configured providers" + print "" + print "To configure providers:" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: providers" + print "" + print "Available providers:" + print " β€’ upcloud - UpCloud provider (European cloud)" + print " β€’ aws - Amazon Web Services" + print " β€’ local - Local/testing provider" + } + _ => { + print $"❌ Unknown providers command: ($subcmd)" + } + } +} + +# Config services handler - List and manage platform services +def handle_config_services [ops: string, flags: record] { + use ../../lib_provisioning/config/export.nu * + + let subcmd = if ($ops | is-empty) { "" } else { $ops | split row " " | first } + + match $subcmd { + "list" => { + print "πŸ”§ Configured Platform Services" + print "===============================" + print "" + + list-platform-services + } + "help" | "h" | "" => { + print "πŸ“‹ Platform Services Commands" + print "============================" + print "" + print "Commands:" + print " config services list - List all configured services" + print "" + print "To configure services:" + print " Edit: workspace_librecloud/config/config.ncl" + print " Section: platform" + print "" + print "Available services:" + print " β€’ orchestrator - Infrastructure orchestrator" + print " β€’ kms - Key management system" + print " β€’ control-center - Admin control panel" + print " β€’ plugins - Native performance plugins" + } + _ => { + print $"❌ Unknown services command: ($subcmd)" + } + } +} diff --git a/nulib/main_provisioning/commands/development.nu b/nulib/main_provisioning/commands/development.nu index 0349b1f..4ab5d88 100644 --- a/nulib/main_provisioning/commands/development.nu +++ b/nulib/main_provisioning/commands/development.nu @@ -70,4 +70,4 @@ def handle_version [ops: string, flags: record] { def handle_pack [ops: string, flags: record] { let args = build_module_args $flags $ops run_module $args "pack" --exec -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/generation.nu b/nulib/main_provisioning/commands/generation.nu index 78b948b..085198d 100644 --- a/nulib/main_provisioning/commands/generation.nu +++ b/nulib/main_provisioning/commands/generation.nu @@ -143,4 +143,4 @@ export def handle_generation_command [ exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/guides.nu b/nulib/main_provisioning/commands/guides.nu index afd8be5..eca2a41 100644 --- a/nulib/main_provisioning/commands/guides.nu +++ b/nulib/main_provisioning/commands/guides.nu @@ -37,7 +37,7 @@ def display_cheatsheet_summary [] { print $"(_ansi yellow_bold)Orchestration:(_ansi reset)" print $" provisioning wf list # List workflows" print $" provisioning wf monitor # Monitor workflow" - print $" provisioning bat submit # Submit batch workflow" + print $" provisioning bat submit # Submit batch workflow" print $" provisioning orch status # Orchestrator status" print "" print $"(_ansi yellow_bold)Platform:(_ansi reset)" diff --git a/nulib/main_provisioning/commands/infrastructure.nu b/nulib/main_provisioning/commands/infrastructure.nu index 3c70e7c..b9e53c5 100644 --- a/nulib/main_provisioning/commands/infrastructure.nu +++ b/nulib/main_provisioning/commands/infrastructure.nu @@ -5,20 +5,81 @@ use ../flags.nu * use ../../lib_provisioning * use ../../lib_provisioning/plugins/auth.nu * +# Pre-load server module to preserve plugin context (tera, auth, kms, etc.) +# This is needed so template rendering and other plugin operations work +# in the same Nushell process +use ../../servers/create.nu * + # Helper to run module commands +# Modules are pre-loaded above to preserve plugin context def run_module [ args: string module: string - option?: string + subcommand?: string # Optional explicit subcommand (for create operations) --exec ] { - let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } - - # Always add --notitles when dispatching to submodules to prevent double title display - if $exec { - exec $"($env.PROVISIONING_NAME)" $use_debug -mod $module ($option | default "") $args --notitles + # Convert args string to list by splitting on spaces + let args_list = if ($args | is-not-empty) { + $args | split row " " | where {|x| ($x | str trim | is-not-empty) } } else { - ^$"($env.PROVISIONING_NAME)" $use_debug -mod $module ($option | default "") $args --notitles + [] + } + + # Call the appropriate module's main function + # Server module is pre-loaded above, so plugins (tera, auth, kms, etc.) are in scope + match $module { + "server" => { + # For server: call the "main create" function directly from the already-loaded servers/create.nu + # This preserves the tera plugin context in the same process + # If subcommand is explicitly provided (from handle_server), use it + # Otherwise, extract from args + let actual_subcommand = if ($subcommand | is-not-empty) { + $subcommand + } else { + let op_list = ($args | split row " " | where { |x| ($x | is-not-empty) }) + if ($op_list | length) > 0 { $op_list | first } else { "help" } + } + + # For now, only handle "create" directly. For others, use -mod + match $actual_subcommand { + "create" | "c" => { + # The servers/create.nu is pre-loaded at the top of this file + # Call "main create" function directly with the arguments + # This preserves the tera plugin context in the same process + let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } + let cmd_args = [-mod, "server", "create", ...$args_list] + exec $"($env.PROVISIONING_NAME)" $use_debug ...$cmd_args + } + _ => { + # For other operations (delete, list, ssh, etc.), use -mod + let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } + let cmd_args = [-mod, "server", ...$args_list] + exec $"($env.PROVISIONING_NAME)" $use_debug ...$cmd_args + } + } + } + "taskserv" | "task" => { + # Taskserv uses exec mode + let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } + let cmd_args = [-mod, $module, ...$args_list, --notitles] + exec $"($env.PROVISIONING_NAME)" $use_debug ...$cmd_args + } + "cluster" => { + # Cluster uses exec mode + let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } + let cmd_args = [-mod, $module, ...$args_list, --notitles] + exec $"($env.PROVISIONING_NAME)" $use_debug ...$cmd_args + } + "infra" => { + # Infra uses exec mode since it's a legacy module + let use_debug = if ($env.PROVISIONING_DEBUG? | default false) { "-x" } else { "" } + let cmd_args = [-mod, $module, ...$args_list, --notitles] + exec $"($env.PROVISIONING_NAME)" $use_debug ...$cmd_args + } + _ => { + print $"❌ Unknown module: ($module)" + exit 1 + } } } @@ -31,6 +92,65 @@ export def handle_infrastructure_command [ set_debug_env $flags match $command { + "create" | "c" => { + # Handle: provisioning create server/taskserv/cluster ... + let create_ops_list = if ($ops | is-not-empty) { + $ops | split row " " | where {|x| ($x | is-not-empty) } + } else { [] } + + let resource_type = if (($create_ops_list | length) > 0) { + $create_ops_list | first + } else { "" } + + let resource_name_and_args = if (($create_ops_list | length) > 1) { + $create_ops_list | skip 1 | str join " " + } else { "" } + + match $resource_type { + "server" | "s" => { handle_server $"create $resource_name_and_args" $flags } + "taskserv" | "task" | "t" => { handle_taskserv $"create $resource_name_and_args" $flags } + "cluster" | "cl" => { handle_cluster $"create $resource_name_and_args" $flags } + _ => { + if ($resource_type | is-empty) { + print "❌ Resource type required for create command" + } else { + print $"❌ Unknown resource type for create: ($resource_type)" + } + print "" + print "Usage: provisioning create " + print "" + print "Resources:" + print " server (s) - Create a server" + print " taskserv (t) - Create a task service" + print " cluster (cl) - Create a cluster" + exit 1 + } + } + } + "delete" | "d" => { + # Handle: provisioning delete server/taskserv/cluster ... + let delete_ops_list = if ($ops | is-not-empty) { + $ops | split row " " | where {|x| ($x | is-not-empty) } + } else { [] } + + let resource_type = if (($delete_ops_list | length) > 0) { + $delete_ops_list | first + } else { "" } + + let resource_name_and_args = if (($delete_ops_list | length) > 1) { + $delete_ops_list | skip 1 | str join " " + } else { "" } + + match $resource_type { + "server" | "s" => { handle_server $"delete $resource_name_and_args" $flags } + "taskserv" | "task" | "t" => { handle_taskserv $"delete $resource_name_and_args" $flags } + "cluster" | "cl" => { handle_cluster $"delete $resource_name_and_args" $flags } + _ => { + print $"❌ Unknown resource type for delete: ($resource_type)" + exit 1 + } + } + } "server" => { handle_server $ops $flags } "taskserv" | "task" => { handle_taskserv $ops $flags } "cluster" => { handle_cluster $ops $flags } @@ -95,7 +215,7 @@ def handle_server [ops: string, flags: record] { } # Authentication check for server operations (metadata-driven) - let operation_parts = ($ops | split row " ") + let operation_parts = ($ops | split row " " | where {|x| ($x | is-not-empty)}) let action = if ($operation_parts | is-empty) { "" } else { $operation_parts | first } # Determine operation type @@ -112,8 +232,17 @@ def handle_server [ops: string, flags: record] { check-operation-auth $operation_name $operation_type $flags } - let args = build_module_args $flags $ops - run_module $args "server" --exec + # Extract the remaining arguments after the action verb (create/delete/list/etc) + let action_and_args = if ($operation_parts | length) > 1 { + $operation_parts | skip 1 | str join " " + } else { + "" + } + + let args = build_module_args $flags $action_and_args + # Pass the action as explicit subcommand so run_module knows which operation is being performed + # For create operations, this preserves plugin context by calling "main create" directly + run_module $args "server" $action --exec } # Task service command handler @@ -273,4 +402,4 @@ export def handle_create_server_task [ops: string, flags: record] { # Create taskservs let taskserv_args = build_module_args $flags $"- ($ops)" run_module $taskserv_args "taskserv" "create" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/integrations.nu b/nulib/main_provisioning/commands/integrations.nu index e3b1e41..516acc3 100644 --- a/nulib/main_provisioning/commands/integrations.nu +++ b/nulib/main_provisioning/commands/integrations.nu @@ -474,7 +474,7 @@ def cmd-orch [ "validate" => { let workflow = ($args | get 0?) if ($workflow == null) { - print "Usage: provisioning orch validate [--strict]" + print "Usage: provisioning orch validate [--strict]" exit 1 } let strict = ("--strict" in $args) or ("-s" in $args) @@ -497,7 +497,7 @@ def cmd-orch [ "submit" => { let workflow = ($args | get 0?) if ($workflow == null) { - print "Usage: provisioning orch submit [--priority <0-100>]" + print "Usage: provisioning orch submit [--priority <0-100>]" exit 1 } let priority = (parse-flag $args "--priority" "-p" | default "50" | into int) @@ -1081,8 +1081,8 @@ def help-orch [] { print "Actions:" print " status Check orchestrator status" print " tasks List tasks in queue" - print " validate Validate KCL workflow" - print " submit Submit workflow for execution" + print " validate Validate Nickel workflow" + print " submit Submit workflow for execution" print " monitor Monitor task progress" print "" print "Options:" @@ -1098,8 +1098,8 @@ def help-orch [] { print "Examples:" print " provisioning orch status" print " provisioning orch tasks --status pending --limit 10" - print " provisioning orch validate workflow.k --strict" - print " provisioning orch submit workflow.k --priority 80" + print " provisioning orch validate workflow.ncl --strict" + print " provisioning orch submit workflow.ncl --priority 80" } def help-runtime [] { diff --git a/nulib/main_provisioning/commands/orchestration.nu b/nulib/main_provisioning/commands/orchestration.nu index 1b67019..5f2e77b 100644 --- a/nulib/main_provisioning/commands/orchestration.nu +++ b/nulib/main_provisioning/commands/orchestration.nu @@ -116,4 +116,4 @@ def handle_orchestrator [ops: string, flags: record] { # Orchestrator has simpler argument requirements run_module $ops "orchestrator" --exec -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/secretumvault.nu b/nulib/main_provisioning/commands/secretumvault.nu new file mode 100644 index 0000000..0983cb5 --- /dev/null +++ b/nulib/main_provisioning/commands/secretumvault.nu @@ -0,0 +1,458 @@ +# SecretumVault Command Handlers +# Handles: kms encrypt, kms decrypt, kms generate-key, kms health, kms version, kms rotate-key + +use ../flags.nu * +use ../../lib_provisioning/plugins/secretumvault.nu * + +# Main SecretumVault command dispatcher +export def handle_secretumvault_command [ + command: string + ops: string + flags: record +] { + match $command { + "secretumvault" | "sv" | "vault" => { + let subcommand = if ($ops | is-not-empty) { + ($ops | split row " " | get 0) + } else { + "help" + } + + let remaining_ops = if ($ops | is-not-empty) { + ($ops | split row " " | skip 1 | str join " ") + } else { + "" + } + + match $subcommand { + "encrypt" => { handle_sv_encrypt $remaining_ops $flags } + "decrypt" => { handle_sv_decrypt $remaining_ops $flags } + "generate-key" | "generate" | "gen-key" => { handle_sv_generate_key $remaining_ops $flags } + "encrypt-file" | "enc-file" => { handle_sv_encrypt_file $remaining_ops $flags } + "decrypt-file" | "dec-file" => { handle_sv_decrypt_file $remaining_ops $flags } + "rotate-key" | "rotate" => { handle_sv_rotate_key $remaining_ops $flags } + "health" | "check" => { handle_sv_health $flags } + "version" | "ver" => { handle_sv_version $flags } + "status" | "info" => { handle_sv_status $flags } + "help" => { show_sv_help } + _ => { + print $"❌ Unknown SecretumVault subcommand: ($subcommand)" + print "" + print "Available SecretumVault subcommands:" + print " encrypt - Encrypt data" + print " decrypt - Decrypt data" + print " generate-key - Generate new encryption key" + print " encrypt-file - Encrypt configuration file" + print " decrypt-file - Decrypt configuration file" + print " rotate-key - Rotate encryption key" + print " health - Check service health" + print " version - Get version information" + print " status - Show plugin status and configuration" + print " help - Show this help message" + print "" + print "Use 'provisioning secretumvault help' for more details" + exit 1 + } + } + } + _ => { + print $"❌ Unknown SecretumVault command: ($command)" + print "Use 'provisioning secretumvault help' for available commands" + exit 1 + } + } +} + +# Encrypt plaintext data +def handle_sv_encrypt [ops: string, flags: record] { + let plaintext = if ($flags.data? | is-not-empty) { + $flags.data + } else if ($flags.plaintext? | is-not-empty) { + $flags.plaintext + } else if ($ops | is-not-empty) { + $ops + } else { + print $"❌ Error: plaintext data required" + print "Usage: provisioning secretumvault encrypt [--key-id <key>]" + exit 1 + } + + let key_id = if ($flags.key_id? | is-not-empty) { $flags.key_id } else { "" } + + print $"Encrypting data..." + + let result = (do -i { + if ($key_id | is-empty) { + plugin-secretumvault-encrypt $plaintext + } else { + plugin-secretumvault-encrypt $plaintext --key-id $key_id + } + }) + + if $result != null { + print $"βœ“ Encryption successful\n" + + if ($result | type) == "record" { + print $"Key ID: ($result.key_id? | default 'N/A')" + print $"Algorithm: ($result.algorithm? | default 'AES-256-GCM')" + print $"Ciphertext:" + print $result.ciphertext + } else { + print $result + } + } else { + print $"❌ Encryption failed" + exit 1 + } +} + +# Decrypt ciphertext data +def handle_sv_decrypt [ops: string, flags: record] { + let ciphertext = if ($flags.ciphertext? | is-not-empty) { + $flags.ciphertext + } else if ($ops | is-not-empty) { + $ops + } else { + print $"❌ Error: ciphertext required" + print "Usage: provisioning secretumvault decrypt <ciphertext> [--key-id <key>]" + exit 1 + } + + let key_id = if ($flags.key_id? | is-not-empty) { $flags.key_id } else { "" } + + print $"Decrypting data..." + + let result = (do -i { + if ($key_id | is-empty) { + plugin-secretumvault-decrypt $ciphertext + } else { + plugin-secretumvault-decrypt $ciphertext --key-id $key_id + } + }) + + if $result != null { + print $"βœ“ Decryption successful\n" + + if ($result | type) == "record" { + if ($result.plaintext? | is-not-empty) { + print $"Plaintext:" + print $result.plaintext + print $"Key ID: ($result.key_id? | default 'N/A')" + } else { + print $result + } + } else { + print $result + } + } else { + print $"❌ Decryption failed" + exit 1 + } +} + +# Generate new data key +def handle_sv_generate_key [ops: string, flags: record] { + let bits_input = if ($flags.bits? | is-not-empty) { $flags.bits } else { "" } + let bits = if ($bits_input | is-empty) { + 256 + } else { + let conversion = (do { $bits_input | into int } | complete) + if $conversion.exit_code == 0 { $conversion.stdout } else { 256 } + } + + let key_id = if ($flags.key_id? | is-not-empty) { $flags.key_id } else { "" } + + print $"Generating data key ($bits) bits..." + + let result = (do -i { + if ($key_id | is-empty) { + plugin-secretumvault-generate-key --bits $bits + } else { + plugin-secretumvault-generate-key --bits $bits --key-id $key_id + } + }) + + if $result != null { + print $"βœ“ Key generation successful\n" + + if ($result | type) == "record" { + print $"Key Size: ($result.bits? | default $bits) bits" + print $"Algorithm: ($result.algorithm? | default 'AES')" + print $"Key ID: ($result.key_id? | default 'N/A')" + print $"Plaintext Key:" + print $result.plaintext + print "" + print $"Encrypted Key:" + print $result.ciphertext + } else { + print $result + } + } else { + print $"❌ Key generation failed" + exit 1 + } +} + +# Encrypt configuration file +def handle_sv_encrypt_file [ops: string, flags: record] { + let file = if ($flags.file? | is-not-empty) { + $flags.file + } else if ($ops | is-not-empty) { + ($ops | split row " " | get 0) + } else { + print $"❌ Error: file path required" + print "Usage: provisioning secretumvault encrypt-file <file> [--output <path>] [--key-id <key>]" + exit 1 + } + + if not ($file | path exists) { + print $"❌ Error: file not found: ($file)" + exit 1 + } + + let output = if ($flags.output? | is-not-empty) { + $flags.output + } else { + $"($file).enc" + } + + let key_id = if ($flags.key_id? | is-not-empty) { $flags.key_id } else { "" } + + print $"Encrypting file: ($file)" + + let result = (do -i { + if ($key_id | is-empty) { + encrypt-config-file $file --output $output + } else { + encrypt-config-file $file --output $output --key-id $key_id + } + }) + + if $result != null { + if ($result.success? | default false) { + print $"βœ“ File encrypted successfully\n" + print $"Input: ($result.input_file? | default $file)" + print $"Output: ($result.output_file? | default $output)" + print $"Key ID: ($result.key_id? | default 'N/A')" + } else { + print $"❌ File encryption failed" + exit 1 + } + } else { + print $"❌ File encryption failed" + exit 1 + } +} + +# Decrypt configuration file +def handle_sv_decrypt_file [ops: string, flags: record] { + let file = if ($flags.file? | is-not-empty) { + $flags.file + } else if ($ops | is-not-empty) { + ($ops | split row " " | get 0) + } else { + print $"❌ Error: file path required" + print "Usage: provisioning secretumvault decrypt-file <file> [--output <path>] [--key-id <key>]" + exit 1 + } + + if not ($file | path exists) { + print $"❌ Error: file not found: ($file)" + exit 1 + } + + let output = if ($flags.output? | is-not-empty) { + $flags.output + } else { + let base_name = ($file | str replace '.enc' '') + $"($base_name).dec" + } + + let key_id = if ($flags.key_id? | is-not-empty) { $flags.key_id } else { "" } + + print $"Decrypting file: ($file)" + + let result = (do -i { + if ($key_id | is-empty) { + decrypt-config-file $file --output $output + } else { + decrypt-config-file $file --output $output --key-id $key_id + } + }) + + if $result != null { + if ($result.success? | default false) { + print $"βœ“ File decrypted successfully\n" + print $"Input: ($result.input_file? | default $file)" + print $"Output: ($result.output_file? | default $output)" + print $"Key ID: ($result.key_id? | default 'N/A')" + } else { + print $"❌ File decryption failed" + exit 1 + } + } else { + print $"❌ File decryption failed" + exit 1 + } +} + +# Rotate encryption key +def handle_sv_rotate_key [ops: string, flags: record] { + let key_id = if ($flags.key_id? | is-not-empty) { + $flags.key_id + } else if ($ops | is-not-empty) { + $ops + } else { + "" + } + + print $"Rotating encryption key..." + + let result = (do -i { + if ($key_id | is-empty) { + plugin-secretumvault-rotate-key + } else { + plugin-secretumvault-rotate-key --key-id $key_id + } + }) + + if $result != null { + print $"βœ“ Key rotation successful\n" + + if ($result | type) == "record" { + print $"Status: ($result.status? | default 'Success')" + print $"Message: ($result.message? | default 'Key rotated successfully')" + } else { + print $result + } + } else { + print $"❌ Key rotation failed" + exit 1 + } +} + +# Check service health +def handle_sv_health [flags: record] { + print $"Checking SecretumVault health..." + + let result = (do -i { + plugin-secretumvault-health + }) + + if $result != null { + print $"βœ“ Health check complete\n" + + if ($result | type) == "record" { + let healthy = ($result.healthy? | default false) + let health_status = if $healthy { $"βœ“ Healthy" } else { $"βœ— Unhealthy" } + + print $"Status: $health_status" + print $"Status Code: ($result.status_code? | default 'N/A')" + print $"Version: ($result.version? | default 'unknown')" + print $"Initialized: ($result.initialized? | default false)" + print $"Sealed: ($result.sealed? | default true)" + } else { + print $result + } + } else { + print $"❌ Health check failed" + exit 1 + } +} + +# Get version information +def handle_sv_version [flags: record] { + print $"Getting SecretumVault version..." + + let result = (do -i { + plugin-secretumvault-version + }) + + if $result != null { + print $"βœ“ Version information\n" + print $"SecretumVault Version: ($result)" + } else { + print $"❌ Version check failed" + exit 1 + } +} + +# Show plugin status and configuration +def handle_sv_status [flags: record] { + print $"SecretumVault Plugin Status\n" + + let info = (do -i { + plugin-secretumvault-info + }) + + if $info != null { + print $"Plugin Available: ($info.plugin_available)" + print $"Plugin Enabled: ($info.plugin_enabled)" + print $"Service URL: ($info.service_url)" + print $"Mount Point: ($info.mount_point)" + print $"Default Key: ($info.default_key)" + print $"Authenticated: ($info.authenticated)" + print $"Mode: ($info.mode)" + } else { + print $"❌ Status check failed" + exit 1 + } +} + +# Show SecretumVault help +def show_sv_help [] { + print "╔════════════════════════════════════════════════════════╗" + print "β•‘ πŸ” SECRETUMVAULT KMS OPERATIONS β•‘" + print "β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•" + print "" + print "DESCRIPTION" + print " SecretumVault is a post-quantum ready KMS system" + print " for secure encryption, decryption, and key management" + print "" + print "COMMANDS" + print " encrypt <plaintext> [--key-id <key>]" + print " Encrypt plaintext data" + print "" + print " decrypt <ciphertext> [--key-id <key>]" + print " Decrypt ciphertext" + print "" + print " generate-key [--bits <int>] [--key-id <key>]" + print " Generate new encryption key, 256 bits default" + print "" + print " encrypt-file <file> [--output <path>] [--key-id <key>]" + print " Encrypt configuration file to .enc format" + print "" + print " decrypt-file <file> [--output <path>] [--key-id <key>]" + print " Decrypt encrypted configuration file" + print "" + print " rotate-key [--key-id <key>]" + print " Rotate encryption key" + print "" + print " health" + print " Check SecretumVault service health" + print "" + print " version" + print " Display SecretumVault version" + print "" + print " status" + print " Show plugin configuration and mode" + print "" + print "ENVIRONMENT VARIABLES" + print " SECRETUMVAULT_URL Service URL, http://localhost:8200 default" + print " SECRETUMVAULT_TOKEN Authentication token, required" + print " SECRETUMVAULT_MOUNT_POINT Vault mount path, transit default" + print " SECRETUMVAULT_KEY_NAME Key name, provisioning-master default" + print " SECRETUMVAULT_TLS_VERIFY Enable TLS verification, false default" + print "" + print "EXAMPLES" + print " provisioning secretumvault encrypt 'my-secret' --key-id master-key" + print " provisioning secretumvault decrypt 'vault:v1:...' --key-id master-key" + print " provisioning secretumvault health" + print " provisioning secretumvault version" + print "" + print "SHORTCUTS" + print " sv β†’ secretumvault, vault β†’ secretumvault, enc β†’ encrypt" + print " dec β†’ decrypt, health β†’ check, version β†’ ver, status β†’ info" + print "" + print "For more info: docs/user/SECRETUMVAULT_KMS_GUIDE.md" +} diff --git a/nulib/main_provisioning/commands/setup.nu b/nulib/main_provisioning/commands/setup.nu index 8dec8e1..bcedbfb 100644 --- a/nulib/main_provisioning/commands/setup.nu +++ b/nulib/main_provisioning/commands/setup.nu @@ -63,6 +63,10 @@ export def cmd-setup [ setup-command-status --verbose=$verbose } + "versions" | "gen-versions" => { + setup-command-versions $args --verbose=$verbose + } + "help" | "h" | "" => { print-setup-help } @@ -426,6 +430,62 @@ def setup-command-status [ } } +# Generate versions file from versions.ncl +def setup-command-versions [ + args: list<string> + --verbose +] { + use ../../lib_provisioning/setup/utils.nu create_versions_file + + if ($args | any { |a| $a == "--help" or $a == "-h" }) { + print "" + print "Generate Versions File" + print "─────────────────────────────────────────────────────────────" + print "" + print "USAGE:" + print " provisioning setup versions [OPTIONS]" + print "" + print "OPTIONS:" + print " --output FILE Output filename (default: versions)" + print " --help, -h Show this help message" + print "" + print "DESCRIPTION:" + print " Generates a bash-compatible versions file from versions.ncl" + print " in KEY=VALUE format for use by shell scripts." + print "" + return + } + + let output_file = ( + if ($args | any { |a| $a == "--output" }) { + let idx = ($args | position { |a| $a == "--output" }) + if ($idx + 1) < ($args | length) { + $args | get ($idx + 1) + } else { + "versions" + } + } else { + "versions" + } + ) + + print-setup-info $"Generating versions file: ($output_file)" + + if (create_versions_file $output_file) { + print-setup-success $"Versions file generated successfully" + if $verbose { + let provisioning = ($env.PROVISIONING? | default ($env.PWD)) + let versions_file = ($provisioning | path join "core" | path join $output_file) + print $"Location: ($versions_file)" + print "" + print "Content:" + print (open $versions_file) + } + } else { + print-setup-error "Failed to generate versions file" + } +} + # ============================================================================ # HELP DISPLAY # ============================================================================ @@ -453,6 +513,7 @@ def print-setup-help [] { print " validate Validate current configuration" print " detect Detect system capabilities" print " migrate Migrate existing configurations" + print " versions Generate bash-compatible versions file" print " status Show setup status" print " help Show this help message" print "" @@ -473,6 +534,9 @@ def print-setup-help [] { print " # Migrate existing workspace" print " provisioning setup migrate --auto" print "" + print " # Generate bash versions file (for shell scripts)" + print " provisioning setup versions" + print "" print "OPTIONS:" print " --check, -c Dry-run without making changes" diff --git a/nulib/main_provisioning/commands/utilities.nu b/nulib/main_provisioning/commands/utilities.nu index eff0ae9..e2204b9 100644 --- a/nulib/main_provisioning/commands/utilities.nu +++ b/nulib/main_provisioning/commands/utilities.nu @@ -152,10 +152,10 @@ def handle_cache [ops: string, flags: record] { print "β–Έ Time-To-Live (TTL) Settings:" let ttl_final = ($config | get --optional ttl_final_config | default "300") - let ttl_kcl = ($config | get --optional ttl_kcl | default "1800") + let ttl_nickel = ($config | get --optional ttl_nickel | default "1800") let ttl_sops = ($config | get --optional ttl_sops | default "900") print (" Final Config: " + ($ttl_final | into string) + "s (5 minutes)") - print (" KCL Compilation: " + ($ttl_kcl | into string) + "s (30 minutes)") + print (" Nickel Compilation: " + ($ttl_nickel | into string) + "s (30 minutes)") print (" SOPS Decryption: " + ($ttl_sops | into string) + "s (15 minutes)") print " Provider Config: 600s (10 minutes)" print " Platform Config: 600s (10 minutes)" @@ -210,7 +210,7 @@ def handle_cache [ops: string, flags: record] { print "Available settings for get/set:" print " enabled - Cache enabled (true/false)" print " ttl_final_config - TTL for final config (seconds)" - print " ttl_kcl - TTL for KCL compilation (seconds)" + print " ttl_nickel - TTL for Nickel compilation (seconds)" print " ttl_sops - TTL for SOPS decryption (seconds)" print "" print "Examples:" @@ -254,7 +254,7 @@ Cache Management Commands: Available settings (for get/set): enabled - Cache enabled (true/false) ttl_final_config - TTL for final config (seconds) - ttl_kcl - TTL for KCL compilation (seconds) + ttl_nickel - TTL for Nickel compilation (seconds) ttl_sops - TTL for SOPS decryption (seconds) Examples: @@ -262,7 +262,7 @@ Examples: provisioning cache config get ttl_final_config provisioning cache config set ttl_final_config 600 provisioning cache config set enabled false - provisioning cache clear kcl + provisioning cache clear nickel provisioning cache list " } @@ -275,7 +275,7 @@ Examples: print " config show - Show cache configuration" print " config get <key> - Get specific cache setting" print " config set <k> <v> - Set cache setting" - print " clear [type] - Clear cache (all, kcl, sops, final)" + print " clear [type] - Clear cache (all, nickel, sops, final)" print " list [type] - List cached items" print " help - Show this help message" print "" @@ -283,7 +283,7 @@ Examples: print " provisioning cache status" print " provisioning cache config get ttl_final_config" print " provisioning cache config set ttl_final_config 600" - print " provisioning cache clear kcl" + print " provisioning cache clear nickel" exit 1 } } @@ -291,7 +291,7 @@ Examples: # Providers command handler - supports list, info, install, remove, installed, validate def handle_providers [ops: string, flags: record] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * # Parse subcommand and arguments let parts = if ($ops | is-not-empty) { @@ -322,12 +322,12 @@ def handle_providers [ops: string, flags: record] { # List all available providers def handle_providers_list [flags: record, args: list] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * _print $"(_ansi green)PROVIDERS(_ansi reset) list: \n" # Parse flags - let show_kcl = ($args | any { |x| $x == "--kcl" }) + let show_nickel = ($args | any { |x| $x == "--nickel" }) let format_idx = ($args | enumerate | where item == "--format" | get 0?.index | default (-1)) let format = if $format_idx >= 0 and ($args | length) > ($format_idx + 1) { $args | get ($format_idx + 1) @@ -336,11 +336,11 @@ def handle_providers_list [flags: record, args: list] { } let no_cache = ($args | any { |x| $x == "--no-cache" }) - # Get providers using cached KCL module loader + # Get providers using cached Nickel module loader let providers = if $no_cache { - (discover-kcl-modules "providers") + (discover-nickel-modules "providers") } else { - (discover-kcl-modules-cached "providers") + (discover-nickel-modules-cached "providers") } match $format { @@ -351,8 +351,8 @@ def handle_providers_list [flags: record, args: list] { _print ($providers | to yaml) "yaml" "result" "table" } _ => { - # Table format - show summary or full with --kcl - if $show_kcl { + # Table format - show summary or full with --nickel + if $show_nickel { _print ($providers | to json) "json" "result" "table" } else { # Show simplified table @@ -367,25 +367,25 @@ def handle_providers_list [flags: record, args: list] { # Show detailed provider information def handle_providers_info [args: list, flags: record] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * if ($args | is-empty) { print "❌ Provider name required" - print "Usage: provisioning providers info <provider> [--kcl] [--no-cache]" + print "Usage: provisioning providers info <provider> [--nickel] [--no-cache]" exit 1 } let provider_name = $args | get 0 - let show_kcl = ($args | any { |x| $x == "--kcl" }) + let show_nickel = ($args | any { |x| $x == "--nickel" }) let no_cache = ($args | any { |x| $x == "--no-cache" }) print $"(_ansi blue_bold)πŸ“‹ Provider Information: ($provider_name)(_ansi reset)" print "" let providers = if $no_cache { - (discover-kcl-modules "providers") + (discover-nickel-modules "providers") } else { - (discover-kcl-modules-cached "providers") + (discover-nickel-modules-cached "providers") } let provider_info = ($providers | where name == $provider_name) @@ -399,22 +399,22 @@ def handle_providers_info [args: list, flags: record] { print $" Name: ($info.name)" print $" Type: ($info.type)" print $" Path: ($info.path)" - print $" Has KCL: ($info.has_kcl)" + print $" Has Nickel: ($info.has_nickel)" - if $show_kcl and $info.has_kcl { + if $show_nickel and $info.has_nickel { print "" - print " (_ansi cyan_bold)KCL Module:(_ansi reset)" - print $" Module Name: ($info.kcl_module_name)" - print $" KCL Path: ($info.kcl_path)" + print " (_ansi cyan_bold)Nickel Module:(_ansi reset)" + print $" Module Name: ($info.module_name)" + print $" Nickel Path: ($info.schema_path)" print $" Version: ($info.version)" print $" Edition: ($info.edition)" - # Check for kcl.mod file - let kcl_mod = ($info.kcl_path | path join "kcl.mod") - if ($kcl_mod | path exists) { + # Check for nickel.mod file + let decl_mod = ($info.schema_path | path join "nickel.mod") + if ($decl_mod | path exists) { print "" - print $" (_ansi cyan_bold)kcl.mod content:(_ansi reset)" - open $kcl_mod | lines | each {|line| print $" ($line)"} + print $" (_ansi cyan_bold)nickel.mod content:(_ansi reset)" + open $decl_mod | lines | each {|line| print $" ($line)"} } } @@ -423,7 +423,7 @@ def handle_providers_info [args: list, flags: record] { # Install provider for infrastructure def handle_providers_install [args: list, flags: record] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * if ($args | length) < 2 { print "❌ Provider name and infrastructure required" @@ -457,12 +457,12 @@ def handle_providers_install [args: list, flags: record] { print $"(_ansi yellow_bold)πŸ’‘ Next steps:(_ansi reset)" print $" 1. Check the manifest: ($infra_path)/providers.manifest.yaml" print $" 2. Update server definitions to use ($provider_name)" - print $" 3. Run: kcl run defs/servers.k" + print $" 3. Run: nickel run defs/servers.ncl" } # Remove provider from infrastructure def handle_providers_remove [args: list, flags: record] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * if ($args | length) < 2 { print "❌ Provider name and infrastructure required" @@ -485,7 +485,7 @@ def handle_providers_remove [args: list, flags: record] { # Confirmation unless forced if not $force { print $"(_ansi yellow)⚠️ This will remove provider ($provider_name) from ($infra_name)(_ansi reset)" - print " KCL dependencies will be updated." + print " Nickel dependencies will be updated." let response = (input "Continue? (y/N): ") if ($response | str downcase) != "y" { @@ -558,7 +558,7 @@ def handle_providers_installed [args: list, flags: record] { # Validate provider installation def handle_providers_validate [args: list, flags: record] { - use ../../lib_provisioning/kcl_module_loader.nu * + use ../../lib_provisioning/module_loader.nu * if ($args | is-empty) { print "❌ Infrastructure name required" @@ -593,9 +593,9 @@ def handle_providers_validate [args: list, flags: record] { # Load providers once using cache let all_providers = if $no_cache { - (discover-kcl-modules "providers") + (discover-nickel-modules "providers") } else { - (discover-kcl-modules-cached "providers") + (discover-nickel-modules-cached "providers") } for provider in $providers { @@ -611,8 +611,8 @@ def handle_providers_validate [args: list, flags: record] { let provider_info = ($available | first) # Check if symlink exists - let modules_dir = ($infra_path | path join ".kcl-modules") - let link_path = ($modules_dir | path join $provider_info.kcl_module_name) + let modules_dir = ($infra_path | path join ".nickel-modules") + let link_path = ($modules_dir | path join $provider_info.module_name) if not ($link_path | path exists) { $validation_errors = ($validation_errors | append $"Symlink missing: ($link_path)") @@ -624,10 +624,10 @@ def handle_providers_validate [args: list, flags: record] { } } - # Check kcl.mod - let kcl_mod_path = ($infra_path | path join "kcl.mod") - if not ($kcl_mod_path | path exists) { - $validation_errors = ($validation_errors | append "kcl.mod not found") + # Check nickel.mod + let nickel_mod_path = ($infra_path | path join "nickel.mod") + if not ($nickel_mod_path | path exists) { + $validation_errors = ($validation_errors | append "nickel.mod not found") } print "" @@ -674,12 +674,12 @@ def show_providers_help [] { (_ansi cyan_bold)β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•(_ansi reset) (_ansi green_bold)[Available Providers](_ansi reset) - (_ansi blue)provisioning providers list [--kcl] [--format <fmt>](_ansi reset) + (_ansi blue)provisioning providers list [--nickel] [--format <fmt>](_ansi reset) List all available providers Formats: table (default value), json, yaml - (_ansi blue)provisioning providers info <provider> [--kcl](_ansi reset) - Show detailed provider information with optional KCL details + (_ansi blue)provisioning providers info <provider> [--nickel](_ansi reset) + Show detailed provider information with optional Nickel details (_ansi green_bold)[Provider Installation](_ansi reset) (_ansi blue)provisioning providers install <provider> <infra> [--version <v>](_ansi reset) @@ -702,8 +702,8 @@ def show_providers_help [] { # List all providers provisioning providers list - # Show KCL module details - provisioning providers info upcloud --kcl + # Show Nickel module details + provisioning providers info upcloud --nickel # Install provider provisioning providers install upcloud myinfra @@ -884,7 +884,7 @@ def handle_plugin_test [ops: string, flags: record] { } else { print $"(_ansi red)❌ Plugin name required(_ansi reset)" print $"Usage: provisioning plugin test <plugin_name>" - print $"Valid plugins: auth, kms, tera, kcl" + print $"Valid plugins: auth, kms, tera, nickel" exit 1 } @@ -959,7 +959,7 @@ def show_plugin_help [] { β€’ (_ansi cyan)auth(_ansi reset) - JWT authentication with MFA support β€’ (_ansi cyan)kms(_ansi reset) - Key Management Service integration β€’ (_ansi cyan)tera(_ansi reset) - Template rendering engine - β€’ (_ansi cyan)kcl(_ansi reset) - KCL configuration language + β€’ (_ansi cyan)nickel(_ansi reset) - Nickel configuration language (_ansi green_bold)EXAMPLES(_ansi reset) @@ -1109,4 +1109,4 @@ def show_guide_list [docs_dir: path] { (_ansi default_dimmed)πŸ’‘ All guides provide copy-paste ready commands Perfect for quick start and reference!(_ansi reset) " -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/commands/workspace.nu b/nulib/main_provisioning/commands/workspace.nu index 6ae962c..ab39ea6 100644 --- a/nulib/main_provisioning/commands/workspace.nu +++ b/nulib/main_provisioning/commands/workspace.nu @@ -314,4 +314,4 @@ def handle_template [ops: string, flags: record] { let args = build_module_args $flags $ops run_module $args "template" --exec -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/contexts.nu b/nulib/main_provisioning/contexts.nu index 16a9723..157ad80 100644 --- a/nulib/main_provisioning/contexts.nu +++ b/nulib/main_provisioning/contexts.nu @@ -3,48 +3,48 @@ use ops.nu provisioning_context_options use ../lib_provisioning/config/accessor.nu * use ../lib_provisioning/setup * -# Manage contexts settings +# Manage contexts settings export def "main context" [ task?: string # server (s) | task (t) | service (sv) name?: string # server (s) | task (t) | service (sv) --key (-k): string --value (-v): string - ...args # Args for create command + ...args # Args for create command --reset (-r) # Restore defaults - --serverpos (-p): int # Server position in settings - --wait (-w) # Wait servers to be created - --settings (-s): string # Settings path + --serverpos (-p): int # Server position in settings + --wait (-w) # Wait servers to be created + --settings (-s): string # Settings path --outfile (-o): string # Output file --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles ] { parse_help_command "context" --task {provisioning_context_options} --end - if $debug { $env.PROVISIONING_DEBUG = true } - let config_path = (setup_config_path) + if $debug { $env.PROVISIONING_DEBUG = true } + let config_path = (setup_config_path) let default_context_path = ($config_path | path join "default_context.yaml") let name_context_path = ($config_path | path join $"($name).yaml") let context_path = ($config_path | path join "context.yaml") - let set_as_default = { + let set_as_default = { rm -f $context_path ^ln -s $name_context_path $context_path _print ( - $"(_ansi blue_bold)($name)(_ansi reset) set as (_ansi green)default context(_ansi reset)" + + $"(_ansi blue_bold)($name)(_ansi reset) set as (_ansi green)default context(_ansi reset)" + $" in (_ansi default_dimmed)($config_path)(_ansi reset)" ) } - match $task { - "h" => { + match $task { + "h" => { ^$"((get-provisioning-name))" context --help _print (provisioning_context_options) } - "create" | "c" | "new" => { - if $name == null or $name == "" { - _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " + "create" | "c" | "new" => { + if $name == null or $name == "" { + _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " } if ($name_context_path |path exists) { _print $"(_ansi blue_bold)($name)(_ansi reset) already in (_ansi default_dimmed)($config_path)(_ansi reset)" @@ -55,28 +55,28 @@ export def "main context" [ } do $set_as_default }, - "default" | "d" => { - if $name == null or $name == "" { - _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " + "default" | "d" => { + if $name == null or $name == "" { + _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " exit 1 } if not ($name_context_path | path exists) { - _print $"πŸ›‘ No (_ansi red)($name)(_ansi reset) found in (_ansi default_dimmed)($config_path)(_ansi reset) " + _print $"πŸ›‘ No (_ansi red)($name)(_ansi reset) found in (_ansi default_dimmed)($config_path)(_ansi reset) " exit 1 } do $set_as_default }, - "remove" | "r" => { + "remove" | "r" => { if $name == null { - _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " + _print $"πŸ›‘ No (_ansi red)name(_ansi reset) value " exit 1 } - if $name == "" or not ( $name_context_path | path exists) { + if $name == "" or not ( $name_context_path | path exists) { _print $"πŸ›‘ context path (_ansi blue_bold)($name)(_ansi reset) not found " exit 1 } - let context = (setup_user_context $name) - let curr_infra = ($context | get infra? | default null) + let context = (setup_user_context $name) + let curr_infra = ($context | get infra? | default null) if $curr_infra == $name { _print ( $"(_ansi blue_bold)($name)(_ansi reset) removed as (_ansi green)default context(_ansi reset) " + @@ -86,33 +86,33 @@ export def "main context" [ rm -f $name_context_path $context_path _print $"(_ansi blue_bold)($name)(_ansi reset) context removed " }, - "edit" | "e" => { + "edit" | "e" => { let editor = ($env | get EDITOR? | default "vi") let config_path = (setup_user_context_path $name) ^$editor $config_path - }, - "view" | "v" => { + }, + "view" | "v" => { _print ((setup_user_context $name) | table -e) - }, - "set" | "s" => { - let context = (setup_user_context $name) - let curr_value = if ($key in ($context | columns)) { $context | get $key } else { null } + }, + "set" | "s" => { + let context = (setup_user_context $name) + let curr_value = if ($key in ($context | columns)) { $context | get $key } else { null } if $curr_value == null { - _print $"πŸ›‘ invalid ($key) in setup " + _print $"πŸ›‘ invalid ($key) in setup " exit 1 } if $curr_value == $value { - _print $"πŸ›‘ ($key) ($value) already set " + _print $"πŸ›‘ ($key) ($value) already set " exit 1 } - # if $context != null and ( $context.infra | path exists) { return $context.infra } - let new_context = ($context | update $key $value) + # if $context != null and ( $context.infra | path exists) { return $context.infra } + let new_context = ($context | update $key $value) setup_save_context $new_context }, - "i" | "install" => { + "i" | "install" => { install_config $reset --context }, - _ => { + _ => { invalid_task "context" ($task | default "") --end }, } @@ -204,4 +204,4 @@ export def "list-workspace-contexts" [] { export def "get-active-workspace-context" [] { let contexts = (list-workspace-contexts) $contexts | where active == true | first -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/control-center.nu b/nulib/main_provisioning/control-center.nu index c0897d2..fae6191 100644 --- a/nulib/main_provisioning/control-center.nu +++ b/nulib/main_provisioning/control-center.nu @@ -16,4 +16,4 @@ export def "main control-center" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "control-center" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/create.nu b/nulib/main_provisioning/create.nu index 661ddc7..8240b1a 100644 --- a/nulib/main_provisioning/create.nu +++ b/nulib/main_provisioning/create.nu @@ -23,7 +23,6 @@ export def "main create" [ --dry-run # Show what would be done without executing --verbose (-v) # Verbose output with enhanced logging ]: nothing -> nothing { - # Enhanced validation and logging if ($target | is-empty) { log-error "Target parameter is required" "create" @@ -163,4 +162,4 @@ export def show-creation-progress [ ]: nothing -> nothing { let percent = (($current * 100) / $total | into int) log-progress $operation $percent "progress" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/dashboard.nu b/nulib/main_provisioning/dashboard.nu index 884debb..7c1ef22 100644 --- a/nulib/main_provisioning/dashboard.nu +++ b/nulib/main_provisioning/dashboard.nu @@ -155,4 +155,4 @@ def show_dashboard_status []: nothing -> nothing { print "2. Start API: provisioning api start" } print "3. Create dashboard: provisioning dashboard demo" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/delete.nu b/nulib/main_provisioning/delete.nu index 87c4087..7cc0d32 100644 --- a/nulib/main_provisioning/delete.nu +++ b/nulib/main_provisioning/delete.nu @@ -2,8 +2,8 @@ use ../lib_provisioning/config/accessor.nu * def prompt_delete [ - target: string - target_name: string + target: string + target_name: string yes: bool name?: string ]: nothing -> string { @@ -15,16 +15,16 @@ def prompt_delete [ } if not $yes or not ((($env.PROVISIONING_ARGS? | default "")) | str contains "--yes") { _print ( $"To (_ansi red_bold)delete ($target_name) (_ansi reset) " + - $" (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " + $" (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " ) let user_input = (input --numchar 3) if $user_input != "yes" and $user_input != "YES" { exit 1 } $name - } else { + } else { $env.PROVISIONING_ARGS = ($env.PROVISIONING_ARGS? | find -v "yes") - ($name | default "" | str replace "yes" "") + ($name | default "" | str replace "yes" "") } } @@ -32,48 +32,48 @@ def prompt_delete [ export def "main delete" [ target?: string # server (s) | task (t) | service (sv) name?: string # target name in settings - ...args # Args for create command - --serverpos (-p): int # Server position in settings + ...args # Args for create command + --serverpos (-p): int # Server position in settings --keepstorage # Keep storage --yes (-y) # confirm delete - --wait (-w) # Wait servers to be created - --infra (-i): string # Infra path - --settings (-s): string # Settings path + --wait (-w) # Wait servers to be created + --infra (-i): string # Infra path + --settings (-s): string # Settings path --outfile (-o): string # Output file --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } parse_help_command "delete" --end - if $debug { $env.PROVISIONING_DEBUG = true } - let use_debug = if $debug or (is-debug-enabled) { "-x" } else { "" } - match $target { - "server"| "servers" | "s" => { - prompt_delete "server" "servers" $yes $name + if $debug { $env.PROVISIONING_DEBUG = true } + let use_debug = if $debug or (is-debug-enabled) { "-x" } else { "" } + match $target { + "server"| "servers" | "s" => { + prompt_delete "server" "servers" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "server" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles }, - "storage" => { - prompt_delete "server" "storage" $yes $name + "storage" => { + prompt_delete "server" "storage" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "server" $env.PROVISIONING_ARGS --yes --notitles }, - "taskserv" | "taskservs" | "t" => { + "taskserv" | "taskservs" | "t" => { prompt_delete "taskserv" "tasks/services" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "tasksrv" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles }, - "clusters"| "clusters" | "cl" => { + "clusters"| "clusters" | "cl" => { prompt_delete "cluster" "cluster" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "cluster" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles }, - _ => { + _ => { invalid_task "delete" ($target | default "") --end exit }, diff --git a/nulib/main_provisioning/dispatcher.nu b/nulib/main_provisioning/dispatcher.nu index 0f1f764..30dd00a 100644 --- a/nulib/main_provisioning/dispatcher.nu +++ b/nulib/main_provisioning/dispatcher.nu @@ -15,6 +15,7 @@ use commands/diagnostics.nu * use commands/integrations.nu * use commands/vm_domain.nu * use commands/platform.nu * +use commands/secretumvault.nu * use ../lib_provisioning * use ../lib_provisioning/workspace/enforcement.nu * use ../lib_provisioning/commands/traits.nu * @@ -107,7 +108,7 @@ export def get_command_registry []: nothing -> record { "plat": "platform platform" "platform": "platform platform" - # Configuration commands (env, allenv, show, init, validate) + # Configuration commands (env, allenv, show, init, validate, export, workspace, platform, services) "e": "config env" "env": "config env" "allenv": "config allenv" @@ -116,6 +117,27 @@ export def get_command_registry []: nothing -> record { "validate": "config validate" "val": "config validate" "config-template": "config config-template" + "export": "config export" + "config-export": "config export" + "config-validate": "config validate" + "ws-config": "config workspace" + "config-workspace": "config workspace" + "plat-config": "config platform" + "config-platform": "config platform" + "config-providers": "config providers" + "config-services": "config services" + + # Platform service configuration shortcuts + "config-orchestrator": "config platform orchestrator" + "orch-config": "config platform orchestrator" + "config-cc": "config platform control-center" + "cc-config": "config platform control-center" + "config-mcp": "config platform mcp-server" + "mcp-config": "config platform mcp-server" + "config-installer": "config platform installer" + "installer-config": "config platform installer" + "config-kms": "config platform kms" + "kms-config": "config platform kms" # Authentication commands (auth, login, logout, mfa) - mapped to integrations for plugin support "login": "integrations auth login" @@ -184,6 +206,9 @@ export def get_command_registry []: nothing -> record { "kms-status": "integrations kms status" "encrypt": "integrations kms encrypt" "decrypt": "integrations kms decrypt" + "sv": "secretumvault secretumvault" + "vault": "secretumvault secretumvault" + "secretumvault": "secretumvault secretumvault" "orch-status": "integrations orch status" "orch-tasks": "integrations orch tasks" @@ -245,8 +270,42 @@ export def dispatch_command [ args: list flags: record ] { - let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops_list = ($args | skip 1) + + # Find first non-flag argument as the task + # (flags have already been parsed by main function, but reorder_args may have moved them) + let matches = ($args | enumerate | where {|item| + not ($item.item | str starts-with "-") and ($item.item | is-not-empty) + }) + + let task_result = if ($matches | length) > 0 { + $matches | first + } else { + null + } + + let task = if ($task_result | is-not-empty) { + $task_result.item + } else { + "" + } + + # DEBUG + if ($env.PROVISIONING_DEBUG? | default false) { + print $"DEBUG dispatcher: task = '($task)'" >&2 + } + + let task_index = if ($task_result | is-not-empty) { + $task_result.index + } else { + 0 + } + + # Get remaining args after task + let ops_list = if $task_index < ($args | length) { + ($args | skip ($task_index + 1)) + } else { + [] + } let ops_str = ($ops_list | str join " ") # Handle empty command @@ -357,6 +416,7 @@ export def dispatch_command [ "generation" => { handle_generation_command $command $final_ops $updated_flags } "guides" => { handle_guide_command $command $final_ops $updated_flags } "authentication" => { handle_authentication_command $command $final_ops $updated_flags } + "secretumvault" => { handle_secretumvault_command $command $final_ops $updated_flags } "diagnostics" => { handle_diagnostics_command $command $final_ops $updated_flags } "integrations" => { handle_integrations_command $command $final_ops $updated_flags } "platform" => { handle_platform_command $command $final_ops $updated_flags } @@ -496,4 +556,4 @@ def handle_special_command [command: string, ops: string, flags: record] { exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/extensions.nu b/nulib/main_provisioning/extensions.nu index f7a8daf..3757175 100644 --- a/nulib/main_provisioning/extensions.nu +++ b/nulib/main_provisioning/extensions.nu @@ -91,4 +91,4 @@ export def "main profile create-examples" [ } create-example-profiles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/flags.nu b/nulib/main_provisioning/flags.nu index 10c1f2c..8f4c28d 100644 --- a/nulib/main_provisioning/flags.nu +++ b/nulib/main_provisioning/flags.nu @@ -226,4 +226,4 @@ export def extract-workspace-infra-from-flags [flags: record] { $flags.infra }) } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/generate.nu b/nulib/main_provisioning/generate.nu index 537eac4..5e117be 100644 --- a/nulib/main_provisioning/generate.nu +++ b/nulib/main_provisioning/generate.nu @@ -7,54 +7,54 @@ use ../lib_provisioning/config/accessor.nu * # Generate infrastructure configurations export def "main generate" [ #hostname?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # Infra path - --settings (-s): string # Settings path - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created - --wait (-w) # Wait servers to be created - --outfile: string # Optional output format: json | yaml | csv | text | md | nuon + ...args # Args for create command + --infra (-i): string # Infra path + --settings (-s): string # Settings path + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created + --wait (-w) # Wait servers to be created + --outfile: string # Optional output format: json | yaml | csv | text | md | nuon --find (-f): string # Optional generate find a value (empty if no value found) - --cols (-l): string # Optional generate columns list separated with comma + --cols (-l): string # Optional generate columns list separated with comma --template(-t): string # Template path or name in PROVISION_KLOUDS_PATH - --ips # Optional generate get IPS only for target "servers-info" + --ips # Optional generate get IPS only for target "servers-info" --prov: string # Optional provider name to filter generate --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } if $helpinfo { _print (provisioning_generate_options) - if not (is-debug-enabled) { end_run "" } + if not (is-debug-enabled) { end_run "" } exit } parse_help_command "generate" --end - if $debug { $env.PROVISIONING_DEBUG = true } + if $debug { $env.PROVISIONING_DEBUG = true } #use defs [ load_settings ] - let curr_settings = if $infra != null { - if $settings != null { + let curr_settings = if $infra != null { + if $settings != null { (load_settings --infra $infra --settings $settings) - } else { + } else { (load_settings --infra $infra) } } else { - if $settings != null { + if $settings != null { (load_settings --settings $settings) - } else { + } else { (load_settings false true) } } - #let cmd_template = if ($template | is-empty ) { + #let cmd_template = if ($template | is-empty ) { # ($args | try { get 0 } catch { "") } #} else { $template } #let str_out = if $outfile == null { "none" } else { $outfile } @@ -62,37 +62,37 @@ export def "main generate" [ let str_cols = if $cols == null { "" } else { $cols } let str_find = if $find == null { "" } else { $find } let str_template = if $template == null { "" } else { $template } - let cmd_target = if ($args | length) > 0 { ($args| get 0) } else { "" } + let cmd_target = if ($args | length) > 0 { ($args| get 0) } else { "" } $env.PROVISIONING_MODULE = "generate" let ops = $"(($env.PROVISIONING_ARGS? | default "")) " | str replace $env.PROVISIONING_MODULE "" | str replace $" ($cmd_target) " "" | str trim #generate_provision $args $curr_settings $str_template match $cmd_target { "new" | "n" => { - let args_list = if ($args | length) > 0 { - ($args| skip 1) + let args_list = if ($args | length) > 0 { + ($args| skip 1) } else { [] } generate_provision $args_list $curr_settings $str_template }, - "server" | "servers" => { + "server" | "servers" => { #use utils/format.nu datalist_to_format - _print (datalist_to_format $str_out + _print (datalist_to_format $str_out (mw_generate_servers $curr_settings $str_find $cols --prov $prov --serverpos $serverpos) ) }, - "server-status" | "servers-status" | "server-info" | "servers-info" => { + "server-status" | "servers-status" | "server-info" | "servers-info" => { let list_cols = if ($cmd_target | str contains "status") { if ($str_cols | str contains "state") { $str_cols } else { $str_cols + ",state" } } else { $str_cols } - # not use $str_cols to filter previous $ips selection + # not use $str_cols to filter previous $ips selection (out_data_generate_info $curr_settings (mw_servers_info $curr_settings $str_find --prov $prov --serverpos $serverpos) #(mw_servers_info $curr_settings $find $cols --prov $prov --serverpos $serverpos) $list_cols $str_out - $ips + $ips ) }, "servers-def" | "server-def" => { @@ -132,16 +132,16 @@ export def generate_new_infra [ let infra_path = if ($args | is-empty) { "" } else { $args | first } let infra_name = ($infra_path | path basename) let target_path = if ($infra_path | str contains "/") { - $infra_path + $infra_path } else if ((get-provisioning-infra-path) | path exists) and not ((get-provisioning-infra-path) | path join $infra_path | path exists) { ((get-provisioning-infra-path) | path join $infra_path) } else { $infra_path } - if ($target_path | path exists) { + if ($target_path | path exists) { _print $"πŸ›‘ Path (_ansi yellow_bold)($target_path)(_ansi reset) already exits" return - } + } ^mkdir -p $target_path _print $"(_ansi green)($infra_name)(_ansi reset) created in (_ansi green)($target_path | path dirname)(_ansi reset)" _print $"(_ansi green)($infra_name)(_ansi reset) ... " @@ -150,9 +150,9 @@ export def generate_new_infra [ } else if ($template | str contains "/") and ($template | path exists) { $template } else if ((get-provisioning-infra-path) | path join $template | path exists) { - ((get-provisioning-infra-path) | path join $template) + ((get-provisioning-infra-path) | path join $template) } - let new_created = if not ($target_path | path join "settings.k" | path exists) { + let new_created = if not ($target_path | path join "settings.ncl" | path exists) { ^cp -pr ...(glob ($template_path | path join "*")) ($target_path) _print $"copy (_ansi green)($template)(_ansi reset) to (_ansi green)($infra_name)(_ansi reset)" true @@ -166,7 +166,7 @@ export def generate_provision [ settings: record template: string ]: nothing -> nothing { - let generated_infra = if ($settings | is-empty) { + let generated_infra = if ($settings | is-empty) { if ($args | is-empty) { (throw-error $"πŸ›‘ ((get-provisioning-name)) generate " $"Invalid option (_ansi red)no settings and path found(_ansi reset)" $"((get-provisioning-name)) generate " --span (metadata $settings).span @@ -175,7 +175,7 @@ export def generate_provision [ generate_new_infra $args $template } } - if ($generated_infra | is-empty) { + if ($generated_infra | is-empty) { (throw-error $"πŸ›‘ ((get-provisioning-name)) generate " $"Invalid option (_ansi red)no settings and path found(_ansi reset)" $"((get-provisioning-name)) generate " --span (metadata $settings).span ) @@ -190,9 +190,9 @@ def out_data_generate_info [ ips: bool ]: nothing -> nothing { if ($data | is-empty) or (($data | first | default null) == null) { - if (is-debug-enabled) { print $"πŸ›‘ ((get-provisioning-name)) generate (_ansi red)no data found(_ansi reset)" } + if (is-debug-enabled) { print $"πŸ›‘ ((get-provisioning-name)) generate (_ansi red)no data found(_ansi reset)" } _print "" - return + return } let sel_data = if ($cols | is-not-empty) { let col_list = ($cols | split row ",") @@ -204,8 +204,8 @@ def out_data_generate_info [ #use utils/format.nu datalist_to_format print (datalist_to_format $outfile $sel_data) # let data_ips = (($data).ip_addresses? | flatten | find "public") - if $ips { - let ips_result = (mw_servers_ips $settings $data) + if $ips { + let ips_result = (mw_servers_ips $settings $data) print $ips_result } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/help_system.nu b/nulib/main_provisioning/help_system.nu index 96cfc2f..c0cff4c 100644 --- a/nulib/main_provisioning/help_system.nu +++ b/nulib/main_provisioning/help_system.nu @@ -216,7 +216,7 @@ def help-orchestration []: nothing -> string { $" (_ansi blue)workflow cleanup(_ansi reset) - Clean old workflows\n\n" + $"(_ansi green_bold)[Batch](_ansi reset) Multi-Provider Batch Operations\n" + - $" (_ansi blue)batch submit <file>(_ansi reset) - Submit KCL workflow [--wait]\n" + + $" (_ansi blue)batch submit <file>(_ansi reset) - Submit Nickel workflow [--wait]\n" + $" (_ansi blue)batch list(_ansi reset) - List batches [--status Running]\n" + $" (_ansi blue)batch status <id>(_ansi reset) - Get batch status\n" + $" (_ansi blue)batch monitor <id>(_ansi reset) - Real-time monitoring\n" + @@ -225,7 +225,7 @@ def help-orchestration []: nothing -> string { $" (_ansi blue)batch stats(_ansi reset) - Show statistics\n\n" + $"(_ansi default_dimmed)πŸ’‘ Batch workflows support mixed providers: UpCloud, AWS, and local\n" + - $" Example: provisioning batch submit deployment.k --wait(_ansi reset)\n" + $" Example: provisioning batch submit deployment.ncl --wait(_ansi reset)\n" ) } @@ -241,7 +241,7 @@ def help-development []: nothing -> string { $" (_ansi blue)module load <type> <ws> <mods>(_ansi reset) - Load modules into workspace\n" + $" (_ansi blue)module list <type> <ws>(_ansi reset)\t - List loaded modules\n" + $" (_ansi blue)module unload <type> <ws> <mod>(_ansi reset) - Unload module\n" + - $" (_ansi blue)module sync-kcl <infra>(_ansi reset)\t - Sync KCL dependencies\n\n" + + $" (_ansi blue)module sync-nickel <infra>(_ansi reset)\t - Sync Nickel dependencies\n\n" + $"(_ansi green_bold)[Architecture](_ansi reset) Layer System (_ansi cyan)STRATEGIC(_ansi reset)\n" + $" (_ansi blue)layer explain(_ansi reset) - Explain layer concept\n" + @@ -289,7 +289,7 @@ def help-workspace []: nothing -> string { $"(_ansi green_bold)[Synchronization](_ansi reset) Update Hidden Directories & Modules\n" + $" (_ansi blue)workspace check-updates [name](_ansi reset)\t - Check which directories need updating\n" + $" (_ansi blue)workspace update [name] [FLAGS](_ansi reset)\t - Update all hidden dirs and content\n" + - $" \t\t\tUpdates: .providers, .clusters, .taskservs, .kcl\n" + + $" \t\t\tUpdates: .providers, .clusters, .taskservs, .nickel\n" + $" (_ansi blue)workspace sync-modules [name] [FLAGS](_ansi reset)\t - Sync workspace modules\n\n" + $"(_ansi default_dimmed)Note: Optional workspace name [name] defaults to active workspace if not specified(_ansi reset)\n\n" + $"(_ansi green_bold)[Common Flags](_ansi reset)\n" + @@ -476,7 +476,7 @@ def help-setup []: nothing -> string { $" Settings are loaded in order \(highest priority wins\):\n\n" + $" 1. (_ansi blue)Runtime Arguments(_ansi reset) - CLI flags \(--flag value\)\n" + $" 2. (_ansi blue)Environment Variables(_ansi reset) - PROVISIONING_* variables\n" + - $" 3. (_ansi blue)Workspace Config(_ansi reset) - workspace/config/provisioning.k\n" + + $" 3. (_ansi blue)Workspace Config(_ansi reset) - workspace/config/provisioning.ncl\n" + $" 4. (_ansi blue)User Preferences(_ansi reset) - ~/.config/provisioning/user_config.yaml\n" + $" 5. (_ansi blue)System Defaults(_ansi reset) - Built-in configuration\n\n" + @@ -565,7 +565,7 @@ def help-concepts []: nothing -> string { $" (_ansi blue)Batch Workflows(_ansi reset)\n" + $" β€’ Multi-provider operations: UpCloud, AWS, and local\n" + $" β€’ Dependency resolution, rollback support\n" + - $" β€’ Defined in KCL workflow files\n\n" + + $" β€’ Defined in Nickel workflow files\n\n" + $"(_ansi green_bold)4. TYPICAL WORKFLOW(_ansi reset)\n\n" + $" 1. (_ansi cyan)Create workspace(_ansi reset): workspace init my-project\n" + @@ -784,7 +784,7 @@ def help-plugins []: nothing -> string { $" (_ansi blue_bold)nu_plugin_orchestrator(_ansi reset) (_ansi cyan)~30x faster(_ansi reset)\n" + " β€’ Direct file-based state access (no HTTP)\n" + - $" β€’ KCL workflow validation\n" + + $" β€’ Nickel workflow validation\n" + $" β€’ Commands: orch status, tasks, validate, submit, monitor\n" + $" β€’ Local task queue operations\n\n" + @@ -799,9 +799,9 @@ def help-plugins []: nothing -> string { $" β€’ Jinja2-compatible template rendering\n" + $" β€’ Used for config generation\n\n" + - $" (_ansi blue_bold)nu_plugin_kcl(_ansi reset)\n" + - $" β€’ KCL configuration language\n" + - $" β€’ Falls back to external KCL CLI\n\n" + + $" (_ansi blue_bold)nu_plugin_nickel(_ansi reset)\n" + + $" β€’ Nickel configuration language\n" + + $" β€’ Falls back to external Nickel CLI\n\n" + $"(_ansi green_bold)PERFORMANCE COMPARISON(_ansi reset)\n\n" + $" Operation Plugin HTTP Fallback\n" + @@ -866,12 +866,12 @@ def help-utilities []: nothing -> string { $" (_ansi blue)cache config show(_ansi reset) - Display all cache settings\n" + $" (_ansi blue)cache config get <setting>(_ansi reset) - Get specific cache setting [dot notation]\n" + $" (_ansi blue)cache config set <setting> <value>(_ansi reset) - Set cache setting\n" + - $" (_ansi blue)cache list [--type <type>](_ansi reset) - List cached items [all|kcl|sops|final]\n" + + $" (_ansi blue)cache list [--type <type>](_ansi reset) - List cached items [all|nickel|sops|final]\n" + $" (_ansi blue)cache clear [--type <type>](_ansi reset) - Clear cache [default: all]\n" + $" (_ansi blue)cache help(_ansi reset) - Show cache command help\n\n" + $"(_ansi cyan_bold) πŸ“Š Cache Features:(_ansi reset)\n" + - $" β€’ Intelligent TTL management \(KCL: 30m, SOPS: 15m, Final: 5m\)\n" + + $" β€’ Intelligent TTL management \(Nickel: 30m, SOPS: 15m, Final: 5m\)\n" + $" β€’ mtime-based validation for stale data detection\n" + $" β€’ SOPS cache with 0600 permissions\n" + $" β€’ Configurable cache size \(default: 100 MB\)\n" + @@ -889,8 +889,8 @@ def help-utilities []: nothing -> string { $" (_ansi blue)decrypt <file>(_ansi reset) - Decrypt file \(alias: kms decrypt\)\n\n" + $"(_ansi green_bold)[Provider Operations](_ansi reset) Cloud & Local Providers\n" + - $" (_ansi blue)providers list [--kcl] [--format <fmt>](_ansi reset) - List available providers\n" + - $" (_ansi blue)providers info <provider> [--kcl](_ansi reset) - Show detailed provider info\n" + + $" (_ansi blue)providers list [--nickel] [--format <fmt>](_ansi reset) - List available providers\n" + + $" (_ansi blue)providers info <provider> [--nickel](_ansi reset) - Show detailed provider info\n" + $" (_ansi blue)providers install <prov> <infra> [--version <v>](_ansi reset) - Install provider\n" + $" (_ansi blue)providers remove <provider> <infra> [--force](_ansi reset) - Remove provider\n" + $" (_ansi blue)providers installed <infra> [--format <fmt>](_ansi reset) - List installed\n" + @@ -918,16 +918,16 @@ def help-utilities []: nothing -> string { $" provisioning cache status\n\n" + $" # Get specific cache setting\n" + - $" provisioning cache config get ttl_kcl # Returns: 1800\n" + + $" provisioning cache config get ttl_nickel # Returns: 1800\n" + $" provisioning cache config get enabled # Returns: true\n\n" + $" # Configure cache\n" + - $" provisioning cache config set ttl_kcl 3000 # Change KCL TTL to 50min\n" + + $" provisioning cache config set ttl_nickel 3000 # Change Nickel TTL to 50min\n" + $" provisioning cache config set ttl_sops 600 # Change SOPS TTL to 10min\n\n" + $" # List cached items\n" + $" provisioning cache list # All cache items\n" + - $" provisioning cache list --type kcl # KCL compilation cache only\n\n" + + $" provisioning cache list --type nickel # Nickel compilation cache only\n\n" + $" # Clear cache\n" + $" provisioning cache clear # Clear all\n" + @@ -936,7 +936,7 @@ def help-utilities []: nothing -> string { $"(_ansi green_bold)CACHE SETTINGS REFERENCE(_ansi reset)\n\n" + $" enabled - Enable/disable cache \(true/false\)\n" + $" ttl_final_config - Final merged config TTL in seconds \(default: 300/5min\)\n" + - $" ttl_kcl - KCL compilation TTL \(default: 1800/30min\)\n" + + $" ttl_nickel - Nickel compilation TTL \(default: 1800/30min\)\n" + $" ttl_sops - SOPS decryption TTL \(default: 900/15min\)\n" + $" max_cache_size - Maximum cache size in bytes \(default: 104857600/100MB\)\n\n" + @@ -1016,7 +1016,7 @@ def help-tools []: nothing -> string { $" β€’ (_ansi cyan)aws(_ansi reset) - AWS CLI v2 \(Cloud provider tool\)\n" + $" β€’ (_ansi cyan)hcloud(_ansi reset) - Hetzner Cloud CLI \(Cloud provider tool\)\n" + $" β€’ (_ansi cyan)upctl(_ansi reset) - UpCloud CLI \(Cloud provider tool\)\n" + - $" β€’ (_ansi cyan)kcl(_ansi reset) - KCL configuration language\n" + + $" β€’ (_ansi cyan)nickel(_ansi reset) - Nickel configuration language\n" + $" β€’ (_ansi cyan)nu(_ansi reset) - Nushell scripting engine\n\n" + $"(_ansi green_bold)VERSION INFORMATION(_ansi reset)\n\n" + @@ -1051,8 +1051,8 @@ def help-diagnostics []: nothing -> string { $"(_ansi green_bold)[System Status](_ansi reset) Component Verification\n" + $" (_ansi blue)status(_ansi reset) - Show comprehensive system status\n" + " β€’ Nushell version check (requires 0.109.0+)\n" + - $" β€’ KCL CLI installation and version\n" + - " β€’ Nushell plugins (auth, KMS, tera, kcl, orchestrator)\n" + + $" β€’ Nickel CLI installation and version\n" + + " β€’ Nushell plugins (auth, KMS, tera, nickel, orchestrator)\n" + $" β€’ Active workspace configuration\n" + $" β€’ Cloud providers availability\n" + $" β€’ Orchestrator service status\n" + @@ -1070,7 +1070,7 @@ def help-diagnostics []: nothing -> string { " β€’ Workspace structure (infra/, config/, extensions/, runtime/)\n" + " β€’ Infrastructure state (servers, taskservs, clusters)\n" + $" β€’ Platform services connectivity\n" + - $" β€’ KCL schemas validity\n" + + $" β€’ Nickel schemas validity\n" + " β€’ Security configuration (KMS, auth, SOPS, Age)\n" + " β€’ Provider credentials (UpCloud, AWS)\n" + $" β€’ Fix recommendations with doc links\n\n" + @@ -1223,7 +1223,7 @@ def help-integrations []: nothing -> string { $" β€’ Architecture: docs/architecture/ECOSYSTEM_INTEGRATION.md\n" + $" β€’ Bridge crate: provisioning/platform/integrations/provisioning-bridge/\n" + $" β€’ Nushell modules: provisioning/core/nulib/lib_provisioning/integrations/\n" + - $" β€’ KCL schemas: provisioning/kcl/integrations/\n\n" + + $" β€’ Nickel schemas: provisioning/nickel/integrations/\n\n" + $"(_ansi default_dimmed)πŸ’‘ Tip: Use --check flag for dry-run mode\n" + $" Example: provisioning runtime exec 'docker ps' --check(_ansi reset)\n" diff --git a/nulib/main_provisioning/layer.nu b/nulib/main_provisioning/layer.nu index 6afa813..e71f546 100644 --- a/nulib/main_provisioning/layer.nu +++ b/nulib/main_provisioning/layer.nu @@ -16,4 +16,4 @@ export def "main layer" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "layer" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/mcp-server.nu b/nulib/main_provisioning/mcp-server.nu index 57df346..241b0b1 100644 --- a/nulib/main_provisioning/mcp-server.nu +++ b/nulib/main_provisioning/mcp-server.nu @@ -16,4 +16,4 @@ export def "main mcp-server" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "mcp-server" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/metadata_handler.nu b/nulib/main_provisioning/metadata_handler.nu index 0ae316d..b05df42 100644 --- a/nulib/main_provisioning/metadata_handler.nu +++ b/nulib/main_provisioning/metadata_handler.nu @@ -3,18 +3,21 @@ # name = "metadata handler" # group = "infrastructure" # tags = ["metadata", "forms", "validation", "interactive"] -# version = "1.0.0" -# requires = ["traits.nu", "forminquire.nu"] -# note = "Command metadata validation and interactive form handling in dispatcher" +# version = "2.0.0" +# requires = ["traits.nu"] +# note = "MIGRATION: ForminQuire (Jinja2 templates) archived. Use TypeDialog forms instead" +# migration = "See: provisioning/.coder/archive/forminquire/ (deprecated) β†’ provisioning/.typedialog/provisioning/form.toml (new)" # ============================================================================ # Metadata Handler for Dispatcher Integration -# Version: 1.0.0 +# Version: 2.0.0 # Purpose: Validate commands and execute interactive forms in dispatcher # ============================================================================ use ../lib_provisioning/commands/traits.nu * -use ../../forminquire/nulib/forminquire.nu * +# ARCHIVED: use ../../forminquire/nulib/forminquire.nu * +# ForminQuire has been archived to: provisioning/.coder/archive/forminquire/ +# New solution: Use TypeDialog for command metadata and form handling # Validate command exists and meets requirements def validate-command-execution [ diff --git a/nulib/main_provisioning/mod.nu b/nulib/main_provisioning/mod.nu index 8f6d4fb..ac92dfc 100644 --- a/nulib/main_provisioning/mod.nu +++ b/nulib/main_provisioning/mod.nu @@ -2,10 +2,12 @@ export use ops.nu * export use query.nu * -export use create.nu * -export use delete.nu * -export use status.nu * -export use update.nu * +# create.nu, delete.nu, status.nu, update.nu are handled by dispatcher +# Do not export them to avoid "main create", "main delete" etc conflicting with dispatch routing +# export use create.nu * +# export use delete.nu * +# export use status.nu * +# export use update.nu * export use generate.nu * # Modular command system (refactored) @@ -43,4 +45,4 @@ export use mcp-server.nu * #export use server/server_delete.nu * -#export module instances.nu \ No newline at end of file +#export module instances.nu diff --git a/nulib/main_provisioning/module.nu b/nulib/main_provisioning/module.nu index 8c9816e..a76eb31 100644 --- a/nulib/main_provisioning/module.nu +++ b/nulib/main_provisioning/module.nu @@ -17,4 +17,4 @@ use ../lib_provisioning/config/accessor.nu * # let debug_flag = if $debug { "--debug" } else { "" } # # ^($env.PROVISIONING_NAME) "module" $cmd_args $infra_flag $check_flag $out_flag $debug_flag -# } \ No newline at end of file +# } diff --git a/nulib/main_provisioning/ops.nu b/nulib/main_provisioning/ops.nu index 212b8a7..f8a5bd8 100644 --- a/nulib/main_provisioning/ops.nu +++ b/nulib/main_provisioning/ops.nu @@ -39,15 +39,15 @@ export def provisioning_options_legacy [ $"| ($target_items)\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) create - to create use one option: ($target_items)\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) delete - to delete use one option: ($target_items)\n" + - $"(_ansi blue)((get-provisioning-name))(_ansi reset) cst - to create (_ansi blue)Servers(_ansi reset) and (_ansi yellow)Tasks(_ansi reset). " + + $"(_ansi blue)((get-provisioning-name))(_ansi reset) cst - to create (_ansi blue)Servers(_ansi reset) and (_ansi yellow)Tasks(_ansi reset). " + $"Alias from (_ansi blue_bold)create-servers-tasks(_ansi reset)\n" + $"\n(_ansi blue)((get-provisioning-name))(_ansi reset) deploy-sel - to sel (_ansi blue)((get-provisioning-name))(_ansi reset) " + - $"(_ansi cyan_bold)deployments info(_ansi reset) --onsel [ (_ansi yellow_bold)e(_ansi reset)dit | " + + $"(_ansi cyan_bold)deployments info(_ansi reset) --onsel [ (_ansi yellow_bold)e(_ansi reset)dit | " + $"(_ansi yellow_bold)v(_ansi reset)iew | (_ansi yellow_bold)l(_ansi reset)ist | (_ansi yellow_bold)t(_ansi reset)ree " + - $"(_ansi yellow_bold)c(_ansi reset)ode | (_ansi yellow_bold)s(_ansi reset)hell | (_ansi yellow_bold)n(_ansi reset)u ]\n" + - $"\n(_ansi blue)((get-provisioning-name))(_ansi reset) deploy-rm - to remove (_ansi blue)((get-provisioning-name))(_ansi reset) " + + $"(_ansi yellow_bold)c(_ansi reset)ode | (_ansi yellow_bold)s(_ansi reset)hell | (_ansi yellow_bold)n(_ansi reset)u ]\n" + + $"\n(_ansi blue)((get-provisioning-name))(_ansi reset) deploy-rm - to remove (_ansi blue)((get-provisioning-name))(_ansi reset) " + $"(_ansi cyan_bold)deployments infos(_ansi reset)\n" + - $"(_ansi blue)((get-provisioning-name))(_ansi reset) destroy - to remove (_ansi blue)((get-provisioning-name))(_ansi reset) " + + $"(_ansi blue)((get-provisioning-name))(_ansi reset) destroy - to remove (_ansi blue)((get-provisioning-name))(_ansi reset) " + $"(_ansi cyan_bold)deployments infos(_ansi reset) and (_ansi green_bold)servers(_ansi reset) with confirmation or add '--yes'\n" + $"\n(_ansi green_bold)Targets(_ansi reset):\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) server - On Servers or instances \n" + @@ -72,9 +72,9 @@ export def provisioning_options_legacy [ $"(_ansi blue)((get-provisioning-name))(_ansi reset) new - To create a new (_ansi blue)((get-provisioning-name))(_ansi reset) Infrastructure \n" + $"\n(_ansi default_dimmed)To get help on Targets use:(_ansi reset) (_ansi blue)((get-provisioning-name))(_ansi reset) [target-name] help\n" + $"\n(_ansi default_dimmed)NOTICE: Most of Options and Targets have a shortcut by using a single dash and a letter(_ansi reset)\n" + - $"(_ansi default_dimmed)example(_ansi reset) -h (_ansi default_dimmed)for(_ansi reset)" + + $"(_ansi default_dimmed)example(_ansi reset) -h (_ansi default_dimmed)for(_ansi reset)" + $" --helpinfo (_ansi default_dimmed)or(_ansi reset) help" + - $" (_ansi default_dimmed)even it can simply be used as(_ansi reset) h \n" + $" (_ansi default_dimmed)even it can simply be used as(_ansi reset) h \n" ) } export def provisioning_context_options [ @@ -97,7 +97,7 @@ export def provisioning_setup_options [ $"(_ansi blue)((get-provisioning-name))(_ansi reset) versions - to generate (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)tools versions file (_ansi reset)\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) midddleware - to generate (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)providers middleware library(_ansi reset)\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) context - to create (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)context file(_ansi reset)\n" + - $"(_ansi blue)((get-provisioning-name))(_ansi reset) defaults - to create (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)defaults file(_ansi reset)" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) defaults - to create (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)defaults file(_ansi reset)" ) } export def provisioning_infra_options [ @@ -118,8 +118,8 @@ export def provisioning_tools_options [ $"(_ansi blue)((get-provisioning-name)) tools(_ansi reset) show providers - to show (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)providers (_ansi reset) info \n" + $"(_ansi blue)((get-provisioning-name)) tools(_ansi reset) show all - to show (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)tools and providers (_ansi reset) info \n" + $"(_ansi blue)((get-provisioning-name)) tools(_ansi reset) info - alias (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi cyan)tools show(_ansi reset) \n" + - $"\n(_ansi blue)((get-provisioning-name)) tools(_ansi reset) (_ansi cyan)[install | check | show](_ansi reset) commmands support to add specifict (_ansi green)'tool-name'(_ansi reset) at the end, " + - $"\n(_ansi blue)((get-provisioning-name)) tools(_ansi reset) (_ansi cyan)show or info(_ansi reset) commmands support to add specifict (_ansi green)'provider-name'(_ansi reset) at the end, " + + $"\n(_ansi blue)((get-provisioning-name)) tools(_ansi reset) (_ansi cyan)[install | check | show](_ansi reset) commmands support to add specifict (_ansi green)'tool-name'(_ansi reset) at the end, " + + $"\n(_ansi blue)((get-provisioning-name)) tools(_ansi reset) (_ansi cyan)show or info(_ansi reset) commmands support to add specifict (_ansi green)'provider-name'(_ansi reset) at the end, " + $"by default uses (_ansi green)'all'(_ansi reset)" + $"\n(_ansi blue)((get-provisioning-name)) tools(_ansi reset) (_ansi green)'tool-name'(_ansi reset) to check tool installation and version" ) @@ -129,7 +129,7 @@ export def provisioning_generate_options [ ( $"(_ansi green_bold)Generate options(_ansi reset):\n" + $"(_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)generate new [name-or-path](_ansi reset) - to create a new (_ansi blue)((get-provisioning-name))(_ansi reset) (_ansi yellow)directory(_ansi reset)" + - $"\nif '[name-or-path]' is not relative or full path it will be created in (_ansi blue)((get-provisioning-infra-path))(_ansi reset) " + + $"\nif '[name-or-path]' is not relative or full path it will be created in (_ansi blue)((get-provisioning-infra-path))(_ansi reset) " + $"\nadd (_ansi blue)--template [name](_ansi reset) to (_ansi cyan)copy(_ansi reset) from existing (_ansi green)template 'name'(_ansi reset) " + $"\ndefault (_ansi blue)template(_ansi reset) to use (_ansi cyan)((get-base-path) | path join (get-provisioning-generate-dirpath) | path join "default")(_ansi reset)" ) @@ -156,7 +156,7 @@ export def provisioning_validate_options [ print "Infrastructure Validation & Review Tool" print "========================================" print "" - print "Validates KCL/YAML configurations, checks best practices, and generates reports" + print "Validates Nickel/YAML configurations, checks best practices, and generates reports" print "" print "USAGE:" @@ -202,7 +202,7 @@ export def provisioning_validate_options [ print "VALIDATION RULES:" print " VAL001 YAML Syntax Validation (critical)" - print " VAL002 KCL Compilation Check (critical)" + print " VAL002 Nickel Compilation Check (critical)" print " VAL003 Unquoted Variable References (error, auto-fixable)" print " VAL004 Required Fields Validation (error)" print " VAL005 Resource Naming Conventions (warning, auto-fixable)" @@ -244,4 +244,4 @@ export def provisioning_validate_options [ print "" "" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/orchestrator.nu b/nulib/main_provisioning/orchestrator.nu index b66c87b..24ffe19 100644 --- a/nulib/main_provisioning/orchestrator.nu +++ b/nulib/main_provisioning/orchestrator.nu @@ -16,4 +16,4 @@ export def "main orchestrator" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "orchestrator" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/pack.nu b/nulib/main_provisioning/pack.nu index 3c21f7b..91a5ec1 100644 --- a/nulib/main_provisioning/pack.nu +++ b/nulib/main_provisioning/pack.nu @@ -26,4 +26,4 @@ export def "main pack" [ let keep_latest_flag = if ($keep_latest | is-not-empty) { $"--keep-latest ($keep_latest)" } else { "" } ^($env.PROVISIONING_NAME) "pack" $cmd_args $infra_flag $check_flag $out_flag $debug_flag $notitles_flag $dry_run_flag $force_flag $all_flag $keep_latest_flag -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/query.nu b/nulib/main_provisioning/query.nu index 59c83b6..40278bb 100644 --- a/nulib/main_provisioning/query.nu +++ b/nulib/main_provisioning/query.nu @@ -7,30 +7,30 @@ use ../lib_provisioning/config/accessor.nu * # Query infrastructure and services export def "main query" [ #hostname?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # Infra path - --settings (-s): string # Settings path - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created - --wait (-w) # Wait servers to be created - --outfile: string # Optional output format: json | yaml | csv | text | md | nuon + ...args # Args for create command + --infra (-i): string # Infra path + --settings (-s): string # Settings path + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created + --wait (-w) # Wait servers to be created + --outfile: string # Optional output format: json | yaml | csv | text | md | nuon --find (-f): string # Optional query find a value (empty if no value found) - --cols (-l): string # Optional query columns list separated with comma - --target(-t): string # Target element for query: servers-status | servers | servers-info | servers-def | defs - --ips # Optional query get IPS only for target "servers-info" + --cols (-l): string # Optional query columns list separated with comma + --target(-t): string # Target element for query: servers-status | servers | servers-info | servers-def | defs + --ips # Optional query get IPS only for target "servers-info" --prov: string # Optional provider name to filter query --ai_query: string # Natural language query using AI --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } # Handle AI query first if provided @@ -38,27 +38,27 @@ export def "main query" [ use ../lib_provisioning/ai/lib.nu * if (is_ai_enabled) and (get_ai_config).enable_query_ai { # Get current infrastructure context for AI - let curr_settings = if $infra != null { - if $settings != null { + let curr_settings = if $infra != null { + if $settings != null { (load_settings --infra $infra --settings $settings) - } else { + } else { (load_settings --infra $infra) } } else { - if $settings != null { + if $settings != null { (load_settings --settings $settings) - } else { + } else { (load_settings) } } - + let context = { infra: ($infra | default "") provider: ($prov | default "") available_targets: ["servers", "servers-status", "servers-info", "servers-def", "defs"] output_format: ($out | default "text") } - + let ai_response = (ai_process_query $ai_query $context) print $ai_response return @@ -69,18 +69,18 @@ export def "main query" [ } parse_help_command "query" --end - if $debug { $env.PROVISIONING_DEBUG = true } + if $debug { $env.PROVISIONING_DEBUG = true } #use defs [ load_settings ] - let curr_settings = if $infra != null { - if $settings != null { + let curr_settings = if $infra != null { + if $settings != null { (load_settings --infra $infra --settings $settings) - } else { + } else { (load_settings --infra $infra) } } else { - if $settings != null { + if $settings != null { (load_settings --settings $settings) - } else { + } else { (load_settings) } } @@ -91,28 +91,28 @@ export def "main query" [ let str_out = if $out == null { "" } else { $out } let str_cols = if $cols == null { "" } else { $cols } let str_find = if $find == null { "" } else { $find } - #use lib_provisioning * + #use lib_provisioning * match $cmd_target { - "server" | "servers" => { + "server" | "servers" => { #use utils/format.nu datalist_to_format - _print (datalist_to_format $str_out + _print (datalist_to_format $str_out (mw_query_servers $curr_settings $str_find $cols --prov $prov --serverpos $serverpos) ) }, - "server-status" | "servers-status" | "server-info" | "servers-info" => { + "server-status" | "servers-status" | "server-info" | "servers-info" => { let list_cols = if ($cmd_target | str contains "status") { if ($str_cols | str contains "state") { $str_cols } else { $str_cols + ",state" } } else { $str_cols } - # not use $str_cols to filter previous $ips selection + # not use $str_cols to filter previous $ips selection (out_data_query_info $curr_settings (mw_servers_info $curr_settings $str_find --prov $prov --serverpos $serverpos) #(mw_servers_info $curr_settings $find $cols --prov $prov --serverpos $serverpos) $list_cols $str_out - $ips + $ips ) }, "servers-def" | "server-def" => { @@ -152,9 +152,9 @@ def out_data_query_info [ ips: bool ]: nothing -> nothing { if ($data | is-empty) or (($data | first | default null) == null) { - if $env.PROVISIONING_DEBUG { print $"πŸ›‘ ((get-provisioning-name)) query (_ansi red)no data found(_ansi reset)" } + if $env.PROVISIONING_DEBUG { print $"πŸ›‘ ((get-provisioning-name)) query (_ansi red)no data found(_ansi reset)" } _print "" - return + return } let sel_data = if ($cols | is-not-empty) { let col_list = ($cols | split row ",") @@ -166,8 +166,8 @@ def out_data_query_info [ #use utils/format.nu datalist_to_format print (datalist_to_format $outfile $sel_data) # let data_ips = (($data).ip_addresses? | flatten | find "public") - if $ips { - let ips_result = (mw_servers_ips $settings $data) + if $ips { + let ips_result = (mw_servers_ips $settings $data) print $ips_result } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/secrets.nu b/nulib/main_provisioning/secrets.nu index 7604a0c..6cdac4e 100644 --- a/nulib/main_provisioning/secrets.nu +++ b/nulib/main_provisioning/secrets.nu @@ -6,43 +6,43 @@ export def "main secrets" [ sourcefile?: string # source file for secrets command targetfile?: string # target file for secrets command --provider (-p): string # secret provider: sops or kms - --encrypt (-e) # Encrypt file + --encrypt (-e) # Encrypt file --decrypt (-d) # Decrypt file - --gen (-g) # Generate encrypted files - --sed # Edit encrypted file + --gen (-g) # Generate encrypted files + --sed # Edit encrypted file --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debug for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debug for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } - + # Set secret provider if specified if ($provider | is-not-empty) { $env.PROVISIONING_SECRET_PROVIDER = $provider } - + parse_help_command "secrets" --end - if $debug { $env.PROVISIONING_DEBUG = true } - + if $debug { $env.PROVISIONING_DEBUG = true } + if $sourcefile == "sed" or $sourcefile == "ed" { on_secrets "sed" $targetfile end_run "secrets" return true } - + if $sed and $sourcefile != null and ($sourcefile | path exists) { on_secrets sed $sourcefile exit } - + if $encrypt { if $sourcefile == null or not ($sourcefile | path exists) { print $"πŸ›‘ Error on_secrets encrypt 'sourcefile' ($sourcefile) not found " @@ -50,34 +50,34 @@ export def "main secrets" [ } if ($targetfile | is-not-empty) { print $"on_secrets encrypt ($sourcefile) ($targetfile)" - on_secrets "encrypt" $sourcefile $targetfile - exit + on_secrets "encrypt" $sourcefile $targetfile + exit } else { print $"on_secrets encrypt ($sourcefile) " print (on_secrets "encrypt" $sourcefile) - exit + exit } } - + if $decrypt { if $sourcefile == null or not ($sourcefile | path exists) { print $"πŸ›‘ Error on_secrets decrypt 'sourcefile' ($sourcefile) not found " return false - } + } if ($targetfile | is-not-empty) { on_secrets decrypt $sourcefile $targetfile - exit - } else { + exit + } else { print (on_secrets decrypt $sourcefile) - exit + exit } } - + if $gen and $sourcefile != null { on_secrets generate $sourcefile $targetfile exit } - - option_undefined "secrets" "" + + option_undefined "secrets" "" end_run "secrets" -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/sops.nu b/nulib/main_provisioning/sops.nu index a6d3517..6465370 100644 --- a/nulib/main_provisioning/sops.nu +++ b/nulib/main_provisioning/sops.nu @@ -5,25 +5,25 @@ use ../lib_provisioning/config/accessor.nu * export def "main sops" [ sourcefile?: string # source file for sops command targetfile?: string # target file for sops command - --encrypt (-e) # SOPS encrypt file + --encrypt (-e) # SOPS encrypt file --decrypt (-d) # SOPS decrypt file - --gen (-g) # SOPS generate encrypted files - --sed # Edit sops encrypted file + --gen (-g) # SOPS generate encrypted files + --sed # Edit sops encrypted file --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } parse_help_command "sops" --end - if $debug { $env.PROVISIONING_DEBUG = true } + if $debug { $env.PROVISIONING_DEBUG = true } if $sourcefile == "sed" or $sourcefile == "ed" { on_sops "sed" $targetfile end_run "sops" @@ -40,32 +40,32 @@ export def "main sops" [ } if ($targetfile | is-not-empty) { print $"on_sops encrypt ($sourcefile) ($targetfile)" - on_sops "encrypt" $sourcefile $targetfile - exit + on_sops "encrypt" $sourcefile $targetfile + exit } else { print $"on_sops encrypt ($sourcefile) " print (on_sops "encrypt" $sourcefile) - exit + exit } } if $decrypt { if $sourcefile == null or not ($sourcefile | path exists) { print $"πŸ›‘ Error on_sops decrypt 'sourcefile' ($sourcefile) not found " return false - } + } if ($targetfile | is-not-empty) { on_sops decrypt $sourcefile $targetfile - exit - } else { + exit + } else { print (on_sops decrypt $sourcefile) - exit + exit } } if $gen and $sourcefile != null { on_sops generate $sourcefile $targetfile exit } - option_undefined "sops" "" + option_undefined "sops" "" #cleanup $settings.wk_path end_run "sops" } diff --git a/nulib/main_provisioning/taskserv.nu b/nulib/main_provisioning/taskserv.nu index 1db3080..330d0e2 100644 --- a/nulib/main_provisioning/taskserv.nu +++ b/nulib/main_provisioning/taskserv.nu @@ -88,10 +88,10 @@ def show_taskserv_versions [name?: string] { let group_name = ($item.name | path basename) let group_path = $item.name - # First check if group itself has kcl/kcl.mod (group-level taskserv) - let group_kcl_path = ($group_path | path join "kcl") - let group_kcl_mod = ($group_kcl_path | path join "kcl.mod") - if ($group_kcl_mod | path exists) { + # First check if group itself has nickel/nickel.mod (group-level taskserv) + let group_schema_path = ($group_path | path join "nickel") + let group_nickel_mod = ($group_schema_path | path join "nickel.mod") + if ($group_nickel_mod | path exists) { let metadata = { name: $group_name group: $group_name @@ -105,13 +105,13 @@ def show_taskserv_versions [name?: string] { for subitem in $subitems { let app_name = ($subitem.name | path basename) - # Skip 'kcl' and 'images' directories - if (not ($app_name == "kcl") and not ($app_name == "images")) { - let kcl_path = ($subitem.name | path join "kcl") - let kcl_mod_path = ($kcl_path | path join "kcl.mod") + # Skip 'nickel' and 'images' directories + if (not ($app_name == "nickel") and not ($app_name == "images")) { + let schema_path = ($subitem.name | path join "nickel") + let nickel_mod_path = ($schema_path | path join "nickel.mod") - # Check if this application has a kcl/kcl.mod file - if ($kcl_mod_path | path exists) { + # Check if this application has a nickel/nickel.mod file + if ($nickel_mod_path | path exists) { let metadata = { name: $app_name group: $group_name @@ -236,36 +236,36 @@ def check_taskserv_updates [ } # Get all taskservs (same logic as show_taskserv_versions) - let all_k_files = (glob $"($taskservs_path)/**/*.k") + let all_k_files = (glob $"($taskservs_path)/**/*.ncl") - let all_taskservs = ($all_k_files | each { |kcl_file| - # Skip __init__.k, schema files, and other utility files - if ($kcl_file | str ends-with "__init__.k") or ($kcl_file | str contains "/wrks/") or ($kcl_file | str ends-with "taskservs/version.k") { + let all_taskservs = ($all_k_files | each { |decl_file| + # Skip __init__.ncl, schema files, and other utility files + if ($decl_file | str ends-with "__init__.ncl") or ($decl_file | str contains "/wrks/") or ($decl_file | str ends-with "taskservs/version.ncl") { null } else { - let relative_path = ($kcl_file | str replace $"($taskservs_path)/" "") + let relative_path = ($decl_file | str replace $"($taskservs_path)/" "") let path_parts = ($relative_path | split row "/" | where { |p| $p != "" }) # Determine ID from the path structure let id = if ($path_parts | length) >= 3 { $path_parts.0 } else if ($path_parts | length) == 2 { - let filename = ($kcl_file | path basename | str replace ".k" "") + let filename = ($decl_file | path basename | str replace ".ncl" "") if $path_parts.0 == "no" { $"($path_parts.0)::($filename)" } else { $path_parts.0 } } else { - ($kcl_file | path basename | str replace ".k" "") + ($decl_file | path basename | str replace ".ncl" "") } - # Read version data from version.k file - let version_file = ($kcl_file | path dirname | path join "version.k") + # Read version data from version.ncl file + let version_file = ($decl_file | path dirname | path join "version.ncl") let version_info = if ($version_file | path exists) { - let kcl_result = (^kcl $version_file | complete) - if $kcl_result.exit_code == 0 and ($kcl_result.stdout | is-not-empty) { - let result = ($kcl_result.stdout | from yaml) + let decl_result = (^nickel $version_file | complete) + if $decl_result.exit_code == 0 and ($decl_result.stdout | is-not-empty) { + let result = ($decl_result.stdout | from yaml) { current: ($result | get version? | default {} | get current? | default "") source: ($result | get version? | default {} | get source? | default "") @@ -411,4 +411,4 @@ def check_taskserv_updates [ print "" print "πŸ’‘ To update a taskserv: provisioning taskserv update <name> <version>" } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/template.nu b/nulib/main_provisioning/template.nu index 7154af4..abaa6eb 100644 --- a/nulib/main_provisioning/template.nu +++ b/nulib/main_provisioning/template.nu @@ -16,4 +16,4 @@ export def "main template" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "template" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/tools.nu b/nulib/main_provisioning/tools.nu index b6e4d80..d54224e 100644 --- a/nulib/main_provisioning/tools.nu +++ b/nulib/main_provisioning/tools.nu @@ -1,6 +1,6 @@ -#!/usr/bin/env nu +#!/usr/bin/env nu # Info: Script to run Provisioning -# Author: JesusPerezLorenzo +# Author: JesusPerezLorenzo # Release: 1.0.4 # Date: 30-4-2024 @@ -36,13 +36,13 @@ export def "main tools" [ --yes (-y) # Auto-confirm prompts (skip interactive prompts) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } if (use_titles) { show_titles } if $helpinfo { _print (provisioning_tools_options) - # if not $env.PROVISIONING_DEBUG { end_run "" } + # if not $env.PROVISIONING_DEBUG { end_run "" } exit } let tools_task = if $task == null { "" } else { $task } @@ -59,8 +59,8 @@ export def "main tools" [ _print $"(_ansi blue_bold)((get-provisioning-name))(_ansi reset) tools (_ansi green_bold)($tools_args | str join ' ')(_ansi reset) " let target = ($args | get 0? | default "") let match = ($args | get 1? | default "") - match $target { - "a" | "all" => { + match $target { + "a" | "all" => { (show_tools_info $target) (show_provs_info $match) }, @@ -197,7 +197,7 @@ export def "main tools" [ _print $"(_ansi blue_bold)((get-provisioning-name))(_ansi reset) taskserv check (_ansi green_bold)($tools_args | str join ' ')(_ansi reset) " let taskservs_path = if ($args | length) > 0 { ($args | get 0) } else { "" } let configs = (discover-taskserv-configurations --base-path=$taskservs_path) - _print ($configs | select id version kcl_file | table) + _print ($configs | select id version decl_file | table) return }, "taskserv-update" | "tu" => { @@ -226,16 +226,16 @@ export def "main tools" [ ) }, } - if not $env.PROVISIONING_DEBUG { end_run "" } + if not $env.PROVISIONING_DEBUG { end_run "" } } export def show_tools_info [ match: string ]: nothing -> nothing { let tools_data = (open (get-provisioning-req-versions)) - if ($match | is-empty) { + if ($match | is-empty) { _print ($tools_data | table -e) - } else { + } else { let data_to_show = if ($match in ($tools_data | columns)) { $tools_data | get $match } else { null } _print ($data_to_show | table -e) } @@ -245,15 +245,15 @@ export def show_provs_info [ ]: nothing -> nothing { if not ((get-providers-path)| path exists) { _print $"❗Error providers path (_ansi red)((get-providers-path))(_ansi reset) not found" - return + return } - ^ls (get-providers-path) | each {|prv| + ^ls (get-providers-path) | each {|prv| if ($match | is-empty) or $match == ($prv | str trim) { let prv_path = ((get-providers-path) | path join ($prv | str trim) | path join "provisioning.yaml") - if ($prv_path | path exists) { + if ($prv_path | path exists) { _print $"(_ansi magenta_bold)($prv | str trim | str upcase)(_ansi reset)" - _print (open $prv_path | table -e) - } + _print (open $prv_path | table -e) + } } } } @@ -336,7 +336,7 @@ def provisioning_tools_options []: nothing -> string { $" β€’ (_ansi cyan)aws(_ansi reset) - AWS CLI v2\n" + $" β€’ (_ansi cyan)hcloud(_ansi reset) - Hetzner Cloud CLI\n" + $" β€’ (_ansi cyan)upctl(_ansi reset) - UpCloud CLI\n" + - $" β€’ (_ansi cyan)kcl(_ansi reset) - KCL configuration language\n" + + $" β€’ (_ansi cyan)nickel(_ansi reset) - Nickel configuration language\n" + $" β€’ (_ansi cyan)nu(_ansi reset) - Nushell scripting engine\n\n" + $"(_ansi green_bold)VERSION INFORMATION(_ansi reset)\n\n" + @@ -359,4 +359,4 @@ def provisioning_tools_options []: nothing -> string { $" Most tools are optional but recommended for specific cloud providers\n" + $" Pinning ensures version stability for production deployments(_ansi reset)\n" ) -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/update.nu b/nulib/main_provisioning/update.nu index 3a9bb76..1893c09 100644 --- a/nulib/main_provisioning/update.nu +++ b/nulib/main_provisioning/update.nu @@ -2,8 +2,8 @@ use ../lib_provisioning/config/accessor.nu * def prompt_update [ - target: string - target_name: string + target: string + target_name: string yes: bool name?: string ]: nothing -> string { @@ -15,61 +15,61 @@ def prompt_update [ } if not $yes or not ((($env.PROVISIONING_ARGS? | default "")) | str contains "--yes") { _print ( $"To (_ansi red_bold)update ($target_name) (_ansi reset) " + - $" (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " + $" (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " ) let user_input = (input --numchar 3) if $user_input != "yes" and $user_input != "YES" { exit 1 } $name - } else { + } else { $env.PROVISIONING_ARGS = ($env.PROVISIONING_ARGS? | find -v "yes") - ($name | default "" | str replace "yes" "") + ($name | default "" | str replace "yes" "") } } # Update infrastructure and services export def "main update" [ target?: string # server (s) | task (t) | service (sv) name?: string # target name in settings - ...args # Args for create command - --serverpos (-p): int # Server position in settings + ...args # Args for create command + --serverpos (-p): int # Server position in settings --keepstorage # Keep storage --yes (-y) # confirm update - --wait (-w) # Wait servers to be created - --infra (-i): string # Infra path - --settings (-s): string # Settings path + --wait (-w) # Wait servers to be created + --infra (-i): string # Infra path + --settings (-s): string # Settings path --outfile (-o): string # Output file --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } parse_help_command "update" --end - if $debug { $env.PROVISIONING_DEBUG = true } - let use_debug = if $debug or $env.PROVISIONING_DEBUG { "-x" } else { "" } - match $target { - "server"| "servers" | "s" => { + if $debug { $env.PROVISIONING_DEBUG = true } + let use_debug = if $debug or $env.PROVISIONING_DEBUG { "-x" } else { "" } + match $target { + "server"| "servers" | "s" => { let use_keepstorage = if $keepstorage { "--keepstorage "} else { "" } prompt_update "server" "servers" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "server" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles $use_keepstorage }, - "taskserv" | "taskservs" | "t" => { + "taskserv" | "taskservs" | "t" => { prompt_update "taskserv" "tasks/services" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "tasksrv" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles }, - "clusters"| "clusters" | "cl" => { + "clusters"| "clusters" | "cl" => { prompt_update "cluster" "cluster" $yes $name ^$"((get-provisioning-name))" $use_debug -mod "cluster" ($env.PROVISIONING_ARGS | str replace $target '') --yes --notitles }, - _ => { + _ => { invalid_task "update" ($target | default "") --end exit }, diff --git a/nulib/main_provisioning/validate.nu b/nulib/main_provisioning/validate.nu index 88bb5b0..f5e2979 100644 --- a/nulib/main_provisioning/validate.nu +++ b/nulib/main_provisioning/validate.nu @@ -160,7 +160,7 @@ export def "main validate rules" []: nothing -> nothing { let rules = [ {id: "VAL001", category: "syntax", severity: "critical", name: "YAML Syntax Validation", auto_fix: false} - {id: "VAL002", category: "compilation", severity: "critical", name: "KCL Compilation Check", auto_fix: false} + {id: "VAL002", category: "compilation", severity: "critical", name: "Nickel Compilation Check", auto_fix: false} {id: "VAL003", category: "syntax", severity: "error", name: "Unquoted Variable References", auto_fix: true} {id: "VAL004", category: "schema", severity: "error", name: "Required Fields Validation", auto_fix: false} {id: "VAL005", category: "best_practices", severity: "warning", name: "Resource Naming Conventions", auto_fix: true} @@ -275,7 +275,7 @@ def show_validation_help []: nothing -> nothing { def setup_validation_environment [verbose: bool]: nothing -> nothing { # Check required dependencies - let dependencies = ["kcl"] # Add other required tools + let dependencies = ["nickel"] # Add other required tools for dep in $dependencies { let check = (^bash -c $"type -P ($dep)" | complete) @@ -340,4 +340,4 @@ def show_validation_next_steps [result: record]: nothing -> nothing { print "" print "For detailed information, check the generated reports in the output directory." print "Use --help for more usage examples and CI/CD integration guidance." -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/version.nu b/nulib/main_provisioning/version.nu index eba5662..419f477 100644 --- a/nulib/main_provisioning/version.nu +++ b/nulib/main_provisioning/version.nu @@ -16,4 +16,4 @@ export def "main version" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "version" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/versions.nu b/nulib/main_provisioning/versions.nu index bbedd07..2d2c44b 100644 --- a/nulib/main_provisioning/versions.nu +++ b/nulib/main_provisioning/versions.nu @@ -67,4 +67,4 @@ export def "version sync" [ version update-all print "πŸ”„ Synced all versions" } -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/workflow.nu b/nulib/main_provisioning/workflow.nu index 4d3f48e..816a478 100644 --- a/nulib/main_provisioning/workflow.nu +++ b/nulib/main_provisioning/workflow.nu @@ -16,4 +16,4 @@ export def "main workflow" [ let debug_flag = if $debug { "--debug" } else { "" } ^($env.PROVISIONING_NAME) "workflow" $cmd_args $infra_flag $check_flag $out_flag $debug_flag --notitles -} \ No newline at end of file +} diff --git a/nulib/main_provisioning/workspace.nu b/nulib/main_provisioning/workspace.nu index b48e8af..0b6df3e 100644 --- a/nulib/main_provisioning/workspace.nu +++ b/nulib/main_provisioning/workspace.nu @@ -166,4 +166,4 @@ export def "main workspace" [ exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/module_registry.nu b/nulib/module_registry.nu new file mode 100644 index 0000000..fa1cba8 --- /dev/null +++ b/nulib/module_registry.nu @@ -0,0 +1,198 @@ +#!/usr/bin/env nu +# Module Registry - Command-to-Modules Mapping +# Fase 2: Lazy Loading Inteligente +# Maps commands to their required modules for dynamic loading +# This enables loading only necessary modules instead of all 362 +# Follows: @.claude/guidelines/nushell/NUSHELL_GUIDELINES.md + +# === INFRASTRUCTURE COMMANDS === +export const INFRASTRUCTURE_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/workspace/enforcement.nu" + "lib_provisioning/utils/interface.nu" + "servers/utils.nu" + "servers/ssh.nu" +] + +# === TASKSERV COMMANDS === +export const TASKSERV_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/utils/interface.nu" + "taskservs/utils.nu" + "lib_provisioning/defs/lists.nu" +] + +# === CLUSTER COMMANDS === +export const CLUSTER_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/utils/interface.nu" + "clusters/utils.nu" +] + +# === WORKSPACE COMMANDS === +# Note: Fast-path commands (list, active) use lib_minimal.nu +# Only switch/register/etc need full module loading +export const WORKSPACE_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/user/config.nu" + "lib_provisioning/workspace/commands.nu" + "lib_provisioning/workspace/enforcement.nu" + "lib_provisioning/utils/interface.nu" +] + +# === ORCHESTRATION COMMANDS === +export const ORCHESTRATION_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/platform/bootstrap.nu" + "lib_provisioning/utils/interface.nu" + "main_provisioning/orchestrator.nu" +] + +# === CONFIGURATION/VALIDATION COMMANDS === +export const CONFIG_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/config/validator.nu" + "lib_provisioning/utils/interface.nu" + "main_provisioning/validate.nu" +] + +# === DEVELOPMENT COMMANDS === +export const DEVELOPMENT_MODULES = [ + "lib_provisioning/config/loader.nu" + "lib_provisioning/defs/lists.nu" + "lib_provisioning/utils/interface.nu" + "main_provisioning/commands/development.nu" + "main_provisioning/version.nu" +] + +# === CORE COMMON MODULES (Always needed for any full command) === +export const CORE_MODULES = [ + "std log" + "lib_provisioning/utils/interface.nu" + "main_provisioning/flags.nu" +] + +# === COMMAND TO MODULES MAPPING === +# Maps first-level commands to required modules +# Rule 8: Pure function (read-only lookup) +# Rule 1: Explicit types +export def get-command-modules [command: string]: nothing -> list<string> { + let modules = match $command { + # Infrastructure - servers, clusters + "server" | "servers" | "s" => { + ($CORE_MODULES | append $INFRASTRUCTURE_MODULES) + } + + # Infrastructure - taskservs + "taskserv" | "taskservs" | "task" | "t" => { + ($CORE_MODULES | append $TASKSERV_MODULES) + } + + # Infrastructure - clusters + "cluster" | "clusters" | "cl" => { + ($CORE_MODULES | append $CLUSTER_MODULES) + } + + # Workspace management (switch, register, etc) + # Note: list/active use fast-path + "workspace" | "ws" => { + ($CORE_MODULES | append $WORKSPACE_MODULES) + } + + # Orchestration/workflow + "workflow" | "wf" | "orchestrator" | "orch" => { + ($CORE_MODULES | append $ORCHESTRATION_MODULES) + } + + # Configuration validation + "validate" | "config" => { + ($CORE_MODULES | append $CONFIG_MODULES) + } + + # Development commands + "module" | "version" | "layer" => { + ($CORE_MODULES | append $DEVELOPMENT_MODULES) + } + + # For all other commands, load common infrastructure + _ => { + $CORE_MODULES + } + } + + $modules | uniq +} + +# Get modules for command (used by main provisioning to decide what to load) +# Rule 2: Single purpose - just return modules list +# Note: Actual loading is done in main provisioning file with literal 'use' statements +export def get-modules-for-command [command: string]: nothing -> list<string> { + get-command-modules $command +} + +# Get module loading statistics +# Rule 8: Pure function, Rule 2: Single purpose +export def get-module-stats []: nothing -> record { + let infra_count = ($INFRASTRUCTURE_MODULES | length) + let taskserv_count = ($TASKSERV_MODULES | length) + let cluster_count = ($CLUSTER_MODULES | length) + let workspace_count = ($WORKSPACE_MODULES | length) + let orch_count = ($ORCHESTRATION_MODULES | length) + let config_count = ($CONFIG_MODULES | length) + let dev_count = ($DEVELOPMENT_MODULES | length) + let core_count = ($CORE_MODULES | length) + + let total_unique = ( + ( + $INFRASTRUCTURE_MODULES + | append $TASKSERV_MODULES + | append $CLUSTER_MODULES + | append $WORKSPACE_MODULES + | append $ORCHESTRATION_MODULES + | append $CONFIG_MODULES + | append $DEVELOPMENT_MODULES + | append $CORE_MODULES + ) | uniq | length + ) + + { + core: $core_count + infrastructure: $infra_count + taskserv: $taskserv_count + cluster: $cluster_count + workspace: $workspace_count + orchestration: $orch_count + config: $config_count + development: $dev_count + total_categories: 8 + total_unique_modules: $total_unique + estimated_reduction: "362 β†’ 45+ modules (8x reduction)" + } +} + +# Display module registry info +# Rule 2: Single purpose - just display +export def show-module-registry []: nothing -> string { + let stats = (get-module-stats) + + " + === Module Registry Statistics === + + Core Modules: " + ($stats.core | into string) + " + Infrastructure: " + ($stats.infrastructure | into string) + " modules + Taskserv: " + ($stats.taskserv | into string) + " modules + Cluster: " + ($stats.cluster | into string) + " modules + Workspace: " + ($stats.workspace | into string) + " modules + Orchestration: " + ($stats.orchestration | into string) + " modules + Configuration: " + ($stats.config | into string) + " modules + Development: " + ($stats.development | into string) + " modules + + Total Unique: " + ($stats.total_unique_modules | into string) + " modules + Estimated: " + $stats.estimated_reduction + " + + Comparison: + - Full Load: 362 modules (4000ms) + - Lazy Load: 45 modules (500ms) + - Fast-Path: 5 modules (50ms) + " +} diff --git a/nulib/observability/agents.nu b/nulib/observability/agents.nu index 70de83b..22215db 100644 --- a/nulib/observability/agents.nu +++ b/nulib/observability/agents.nu @@ -731,4 +731,4 @@ export def get_agent_status [agent_name?: string]: nothing -> any { # Return status of specific agent {} } -} \ No newline at end of file +} diff --git a/nulib/observability/collectors.nu b/nulib/observability/collectors.nu index ced8f6b..a05893d 100644 --- a/nulib/observability/collectors.nu +++ b/nulib/observability/collectors.nu @@ -652,4 +652,4 @@ export def query_observability_data [ } else { $combined_data } -} \ No newline at end of file +} diff --git a/nulib/providers/discover.nu b/nulib/providers/discover.nu index 46159e3..0a166b9 100644 --- a/nulib/providers/discover.nu +++ b/nulib/providers/discover.nu @@ -14,29 +14,29 @@ export def discover-providers []: nothing -> list<record> { error make { msg: $"Providers path not found: ($providers_path)" } } - # Find all provider directories with KCL modules + # Find all provider directories with Nickel modules ls $providers_path | where type == "dir" | each { |dir| let provider_name = ($dir.name | path basename) - let kcl_path = ($dir.name | path join "kcl") - let kcl_mod_path = ($kcl_path | path join "kcl.mod") + let schema_path = ($dir.name | path join "nickel") + let mod_path = ($schema_path | path join "nickel.mod") - if ($kcl_mod_path | path exists) { - extract_provider_metadata $provider_name $kcl_path + if ($mod_path | path exists) { + extract_provider_metadata $provider_name $schema_path } } | compact | sort-by name } -# Extract metadata from a provider's KCL module -def extract_provider_metadata [name: string, kcl_path: string]: nothing -> record { - let kcl_mod_path = ($kcl_path | path join "kcl.mod") - let mod_content = (open $kcl_mod_path | from toml) +# Extract metadata from a provider's Nickel module +def extract_provider_metadata [name: string, schema_path: string]: nothing -> record { + let mod_path = ($schema_path | path join "nickel.mod") + let mod_content = (open $mod_path | from toml) - # Find KCL schema files - let schema_files = (glob ($kcl_path | path join "*.k")) + # Find Nickel schema files + let schema_files = (glob ($schema_path | path join "*.ncl")) let main_schema = ($schema_files | where ($it | str contains $name) | first | default "") # Extract dependencies @@ -64,16 +64,16 @@ def extract_provider_metadata [name: string, kcl_path: string]: nothing -> recor type: "provider" provider_type: $provider_type version: $mod_content.package.version - kcl_path: $kcl_path + schema_path: $schema_path main_schema: $main_schema dependencies: $dependencies description: $description available: true - last_updated: (ls $kcl_mod_path | get 0.modified) + last_updated: (ls $mod_path | get 0.modified) } } -# Extract description from KCL schema file +# Extract description from Nickel schema file def extract_schema_description [schema_file: string]: nothing -> string { if not ($schema_file | path exists) { return "" @@ -140,4 +140,4 @@ export def get-default-provider []: nothing -> string { } else { $cloud_providers | first | get name } -} \ No newline at end of file +} diff --git a/nulib/providers/load.nu b/nulib/providers/load.nu index 7add81b..e67197b 100644 --- a/nulib/providers/load.nu +++ b/nulib/providers/load.nu @@ -70,9 +70,9 @@ def load-single-provider [target_path: string, name: string, force: bool, layer: } } - # Copy KCL files and directories + # Copy Nickel files and directories mkdir $target_dir - let source_items = (ls $provider_info.kcl_path | get name) + let source_items = (ls $provider_info.schema_path | get name) for $item in $source_items { cp -r $item $target_dir } @@ -99,16 +99,16 @@ def load-single-provider [target_path: string, name: string, force: bool, layer: } } -# Generate providers.k import file +# Generate providers.ncl import file def generate-providers-imports [target_path: string, providers: list<string>, layer: string] { # Generate individual imports for each provider let imports = ($providers | each { |name| # Check provider structure and import appropriately let main_files = [ - ($target_path | path join ".providers" $name ($"provision_($name).k")) - ($target_path | path join ".providers" $name ($"server_($name).k")) - ($target_path | path join ".providers" $name ($"defaults_($name).k")) - ($target_path | path join ".providers" $name ($name + ".k")) + ($target_path | path join ".providers" $name ($"provision_($name).ncl")) + ($target_path | path join ".providers" $name ($"server_($name).ncl")) + ($target_path | path join ".providers" $name ($"defaults_($name).ncl")) + ($target_path | path join ".providers" $name ($name + ".ncl")) ] # Find the main provider file @@ -116,7 +116,7 @@ def generate-providers-imports [target_path: string, providers: list<string>, la if ($main_file | is-empty) { $"import .providers.($name) as ($name)_provider" } else { - let file_stem = ($main_file | path basename | str replace '.k' '') + let file_stem = ($main_file | path basename | str replace '.ncl' '') $"import .providers.($name).($file_stem) as ($name)_provider" } } | str join "\n") @@ -141,7 +141,7 @@ providers = { providers" # Save the imports file - $content | save -f ($target_path | path join "providers.k") + $content | save -f ($target_path | path join "providers.ncl") # Also create individual alias files for easier direct imports for $name in $providers { @@ -153,7 +153,7 @@ import .providers.($name) as ($name) # Re-export for convenience ($name)" - $alias_content | save -f ($target_path | path join $"provider_($name).k") + $alias_content | save -f ($target_path | path join $"provider_($name).ncl") } } @@ -176,7 +176,7 @@ def update-providers-manifest [target_path: string, providers: list<string>, lay type: $info.provider_type layer: $layer loaded_at: (date now | format date '%Y-%m-%d %H:%M:%S') - source_path: $info.kcl_path + source_path: $info.schema_path } }) @@ -208,7 +208,7 @@ export def unload-provider [workspace: string, name: string]: nothing -> record if ($updated_providers | is-empty) { rm $manifest_path - rm ($workspace | path join "providers.k") + rm ($workspace | path join "providers.ncl") } else { let updated_manifest = ($manifest | update loaded_providers $updated_providers) $updated_manifest | to yaml | save $manifest_path @@ -268,4 +268,4 @@ export def set-default-provider [workspace: string, name: string]: nothing -> re default_provider: $name status: "updated" } -} \ No newline at end of file +} diff --git a/nulib/provisioning b/nulib/provisioning index 4a509c9..55a936a 100755 --- a/nulib/provisioning +++ b/nulib/provisioning @@ -27,6 +27,12 @@ export-env { # Combine paths: use default paths first, then add any from current $env.NU_LIB_DIRS = ($default_paths | append $current_lib_dirs) + + # Auto-load tera plugin BEFORE loading any modules + # This ensures tera-render is available throughout the script + if ( (version).installed_plugins | str contains "tera" ) { + (plugin use tera) + } } use std log @@ -41,7 +47,8 @@ use main_provisioning * use servers/ssh.nu * use servers/utils.nu * use taskservs/utils.nu find_taskserv -use lib_provisioning/platform/bootstrap.nu * +# Bootstrap will be loaded on-demand only when needed for real operations +# use lib_provisioning/platform/bootstrap.nu * # Helper: Reorder arguments to put flags before positional args # This allows: provisioning workspace update --yes @@ -156,6 +163,7 @@ def main [ # Bootstrap platform services (only if running actual commands, not help/info) # Skip bootstrap for help-like, guide, setup, discovery/info, and utility commands + # Updated for Phase 1: Fast-Path Expansion - Include read-only workspace commands let is_help_command = ( ($reordered_args | length) == 0 or ($reordered_args | get 0) in [ @@ -167,6 +175,8 @@ def main [ "guide", "guides", "howto", # Setup "setup", "st", + # Workspace commands (read-only, fast-path) + "workspace", "ws", # Discovery and module commands "mod", "module", "discover", "disc", "dt", "dp", "dc", @@ -187,7 +197,34 @@ def main [ ] ) - if not $is_help_command { + # Check if this is a command that doesn't need platform bootstrap + # VM commands and infrastructure commands can work without bootstrap + # Also skip bootstrap if --check flag is present (validation mode, no execution needed) + let skip_bootstrap = ( + (($reordered_args | length) > 0 and + ($reordered_args | get 0) in [ + # Interactive Nushell session (no bootstrap needed) + "nu", + # VM commands (info/list only, no bootstrap needed) + "vm", "vmi", "vmh", "vml", + # Infrastructure commands can work offline + "server", "s", + "taskserv", "task", "t", + "cluster", "cl", + # Create command (with various targets) + "create", "c", + # Delete command + "delete", "d", + # Update command + "update", "u" + ]) or + # Skip bootstrap if in check mode (validation/dry-run, no execution needed) + $final_check + ) + + if (not $is_help_command) and (not $skip_bootstrap) { + # Load bootstrap module dynamically when needed + use lib_provisioning/platform/bootstrap.nu * let bootstrap_result = (bootstrap-platform --auto-start --timeout=60 --verbose=($final_verbose)) if not $bootstrap_result.all_healthy { _print "" @@ -230,8 +267,48 @@ def main [ return } - # Dispatch command to appropriate handler - dispatch_command $reordered_args $parsed_flags + # Check if we're in module mode (invoked with -mod flag from bash wrapper) + # If so, bypass dispatcher and call the module directly + if ($env.PROVISIONING_MODULE? | default "" | is-not-empty) { + let module = $env.PROVISIONING_MODULE + # At this point, $reordered_args contains [create, ...] or whatever the user provided after -mod + # We need to invoke the module's main function + + match $module { + "server" => { + use servers/create.nu * + # Ensure tera plugin is loaded for template rendering + let tera_available = ((plugin list | where name == "tera" | length) > 0) + if $tera_available { + if ($env.PROVISIONING_DEBUG? | default false) { + _print "DEBUG: Loading tera plugin (-mod server)..." >&2 + } + (plugin use tera) + if ($env.PROVISIONING_DEBUG? | default false) { + _print "DEBUG: Tera plugin loaded for -mod server" >&2 + } + } + # Call server create module main function + # $reordered_args now has ["create"] or ["delete"] or ["list"] etc. + main ...$reordered_args --check=$final_check --wait=$final_wait --infra=($infra | default "") --settings=($settings | default "") --outfile=($outfile | default "") --debug=$debug --xm=$xm --xc=$xc --xr=$xr --xld=$xld --metadata=$metadata --notitles=$notitles --out=($out | default "") + } + "taskserv" | "task" => { + use taskservs/create.nu * + main ...$reordered_args --check=$final_check --wait=$final_wait --debug=$debug + } + "cluster" => { + use clusters/create.nu * + main ...$reordered_args --check=$final_check --debug=$debug + } + _ => { + print $"Unknown module: ($module)" + exit 1 + } + } + } else { + # Normal command dispatch through dispatcher + dispatch_command $reordered_args $parsed_flags + } # End run if not in debug mode if not ($env.PROVISIONING_DEBUG? | default false) { end_run "" } diff --git a/nulib/provisioning batch b/nulib/provisioning batch index 0e9cbf5..bbe54a3 100755 --- a/nulib/provisioning batch +++ b/nulib/provisioning batch @@ -153,7 +153,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } let workflow_param = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { ($ops | split row " " | get 0) } else { @@ -429,4 +429,4 @@ def output_result [result: any, format: string]: nothing -> nothing { print ($result | table) } } -} \ No newline at end of file +} diff --git a/nulib/provisioning cluster b/nulib/provisioning cluster index e340b7b..b4c9c34 100755 --- a/nulib/provisioning cluster +++ b/nulib/provisioning cluster @@ -1,17 +1,17 @@ -#!/usr/bin/env nu +#!/usr/bin/env nu # Info: Script to run Provisioning -# Author: JesusPerezLorenzo +# Author: JesusPerezLorenzo # Release: 1.0.4 # Date: 6-2-2024 #use std # assert use std log -use lib_provisioning * +use lib_provisioning * use env.nu * -#Load all main defs +#Load all main defs use clusters * # - > Help on Cluster @@ -19,58 +19,58 @@ export def "main help" [ --src: string = "" --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) -] { - if $notitles == null or not $notitles { show_titles } +] { + if $notitles == null or not $notitles { show_titles } ^($env.PROVISIONING_NAME) "-mod" "cluster" "--help" if ($out | is-not-empty) { $env.PROVISIONING_NO_TERMINAL = false } print (provisioning_options $src) - if not $env.PROVISIONING_DEBUG { end_run "" } + if not $env.PROVISIONING_DEBUG { end_run "" } } # > Cluster services def main [ - ...args: string # Other options, use help to get info + ...args: string # Other options, use help to get info -v # Show version - -i # Show Info + -i # Show Info --version (-V) # Show version with title --info (-I) # Show Info with title --about (-a) # Show About - --infra (-i): string # Infra directory - --settings (-s): string # Settings path - --serverpos (-p): int # Server position in settings + --infra (-i): string # Infra directory + --settings (-s): string # Settings path + --serverpos (-p): int # Server position in settings --yes (-y) # Confirm task - --check (-c) # Only check mode no servers will be created - --wait (-w) # Wait servers to be created - --select: string # Select with cluster as option + --check (-c) # Only check mode no servers will be created + --wait (-w) # Wait servers to be created + --select: string # Select with cluster as option --onsel: string # On selection: e (edit) | v (view) | l (list) --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for cluster and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug - --nc # Not clean working settings + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for cluster and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --nc # Not clean working settings --metadata # Error with metadata (-xm) --notitles # Do not show banner titles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "cluster" $args if $version or $v { ^$env.PROVISIONING_NAME -v ; exit } if $info or $i { ^$env.PROVISIONING_NAME -i ; exit } - if $about { + if $about { #use defs/about.nu [ about_info ] - _print (get_about_info) - exit + _print (get_about_info) + exit } - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } # for $arg in $args { print $arg } - let task = if ($args | length) > 0 { ($args| get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $"($task) " "" | str trim + let task = if ($args | length) > 0 { ($args| get 0) } else { "" } + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } match $task { "h" | "help" => { # Redirect to main categorized help system @@ -83,7 +83,7 @@ def main [ #server_ssh $curr_settings "" "pub" exec ($env.PROVISIONING_NAME) "-mod" "server" "status" ...($ops | split row " ") --notitles } - "sed" => { + "sed" => { if $ops == "" { (throw-error $"πŸ›‘ No file found" $"for (_ansi yellow_bold)sops(_ansi reset) edit") exit 1 @@ -94,31 +94,31 @@ def main [ if $env.PROVISIONING_SOPS? == null { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.CURRENT_INFRA_PATH = $"($curr_settings.infra_path)/($curr_settings.infra)" - use sops_env.nu + use sops_env.nu } #use sops on_sops on_sops "sed" $ops }, - "c" | "create" => { + "c" | "create" => { exec ($env.PROVISIONING_NAME) "-mod" "cluster" "create" ...($ops | split row " ") --notitles } - "d" | "delete" => { + "d" | "delete" => { exec ($env.PROVISIONING_NAME) "-mod" "cluster" "delete" ...($ops | split row " ") --notitles } - "g" | "generate" => { + "g" | "generate" => { exec ($env.PROVISIONING_NAME) "-mod" "cluster" "generate" ...($ops | split row " ") --notitles } - "list" => { + "list" => { #use defs/lists.nu on_list on_list "clusters" ($onsel | default "") "" }, - "qr" => { + "qr" => { #use utils/qr.nu * make_qr }, - _ => { + _ => { invalid_task "cluster" $task --end }, - } - if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + } + if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/provisioning complete b/nulib/provisioning complete index f55f513..53133cd 100755 --- a/nulib/provisioning complete +++ b/nulib/provisioning complete @@ -77,7 +77,7 @@ def main [ # Helper: Locate the provisioning-detector binary def detect-binary-path [] { let env_prov = ($env.PROVISIONING? | default "") - + let possible_paths = if ($env_prov | is-not-empty) { [ ($env_prov | path join "platform" "target" "debug" "provisioning-detector") diff --git a/nulib/provisioning detect b/nulib/provisioning detect index 24d0186..87a4840 100755 --- a/nulib/provisioning detect +++ b/nulib/provisioning detect @@ -76,7 +76,7 @@ def main [ # Helper: Locate the provisioning-detector binary def detect-binary-path [] { let env_prov = ($env.PROVISIONING? | default "") - + let possible_paths = if ($env_prov | is-not-empty) { [ ($env_prov | path join "platform" "target" "debug" "provisioning-detector") diff --git a/nulib/provisioning infra b/nulib/provisioning infra index 6f27f55..a94ad2f 100755 --- a/nulib/provisioning infra +++ b/nulib/provisioning infra @@ -1,13 +1,13 @@ -#!/usr/bin/env nu +#!/usr/bin/env nu # Info: Script to run Provisioning -# Author: JesusPerezLorenzo +# Author: JesusPerezLorenzo # Release: 1.0.4 # Date: 6-2-2024 #use std # assert use std log -use lib_provisioning * +use lib_provisioning * use servers/ssh.nu * use infras/utils.nu * @@ -23,70 +23,70 @@ export def "main help" [ --src: string = "" --notitles # not tittles --out: string # Print Output format: json, yaml, text (default) -] { - if $notitles == null or not $notitles { show_titles } +] { + if $notitles == null or not $notitles { show_titles } ^($env.PROVISIONING_NAME) "-mod" "infra" "--help" if ($out | is-not-empty) { $env.PROVISIONING_NO_TERMINAL = false } print (provisioning_infra_options) - if not $env.PROVISIONING_DEBUG { end_run "" } + if not $env.PROVISIONING_DEBUG { end_run "" } } # > Infras with Tasks and Services for servers def main [ - ...args: string # Other options, use help to get info + ...args: string # Other options, use help to get info --iptype: string = "public" # Ip type to connect -v # Show version - -i # Show Info + -i # Show Info --version (-V) # Show version with title --info (-I) # Show Info with title --about (-a) # Show About - --infra (-i): string # Infra directory + --infra (-i): string # Infra directory --infras: string # Infras list names separated by commas - --settings (-s): string # Settings path + --settings (-s): string # Settings path --iptype: string = "public" # Ip type to connect - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created --yes (-y) # Confirm task - --wait (-w) # Wait servers to be created - --select: string # Select with taskservice as option + --wait (-w) # Wait servers to be created + --select: string # Select with taskservice as option --onsel: string # On selection: e (edit) | v (view) | l (list) --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for taskservice and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug - --nc # Not clean working settings + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for taskservice and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --nc # Not clean working settings --metadata # Error with metadata (-xm) --notitles # Do not show banner titles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "infra" $args if $version or $v { ^$env.PROVISIONING_NAME -v ; exit } if $info or $i { ^$env.PROVISIONING_NAME -i ; exit } - if $about { + if $about { #use defs/about.nu [ about_info ] - _print (get_about_info) - exit + _print (get_about_info) + exit } - if $debug { $env.PROVISIONING_DEBUG = true } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + if $metadata { $env.PROVISIONING_METADATA = true } # for $arg in $args { print $arg } - let task = if ($args | length) > 0 { ($args| get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim - let infras_list = if $infras != null { + let task = if ($args | length) > 0 { ($args| get 0) } else { "" } + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } + let infras_list = if $infras != null { $infras | split row "," } else if ($ops | split row " " | get -o 0 | str contains ",") { - ($ops | split row " " | get -o 0 | split row ",") - } else if ($infra | is-not-empty) { + ($ops | split row " " | get -o 0 | split row ",") + } else if ($infra | is-not-empty) { [ $infra ] - } else { [] } + } else { [] } let ops = if ($ops | split row " " | get -o 0 | str contains ",") { - ($ops | str replace ($ops | split row " " | get -o 0 ) "") + ($ops | str replace ($ops | split row " " | get -o 0 ) "") } else { $ops } let name = if ($ops | str starts-with "-") { "" } else { ($ops | split row "-" | find -v -r "^--" | get -o 0 | default "" | str trim) } match $task { @@ -101,13 +101,13 @@ def main [ #server_ssh $curr_settings "" "pub" exec ($env.PROVISIONING_NAME) "-mod" "server" "status" ...($ops | split row " ") --notitles } - "c" | "create" => { + "c" | "create" => { let outfile = "" - on_create_infras $infras_list $check $wait $outfile $name $serverpos + on_create_infras $infras_list $check $wait $outfile $name $serverpos } - "d" | "delete" => { + "d" | "delete" => { if not $yes or not (($env.PROVISIONING_ARGS? | default "") | str contains "--yes") { - _print $"Run (_ansi red_bold)delete infras(_ansi reset) (_ansi cyan_bold)($infras_list)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " + _print $"Run (_ansi red_bold)delete infras(_ansi reset) (_ansi cyan_bold)($infras_list)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " let user_input = (input --numchar 3) if $user_input != "yes" and $user_input != "YES" { exit 1 @@ -116,29 +116,29 @@ def main [ let keep_storage = false on_delete_infras $infras_list $keep_storage $wait $name $serverpos } - "g" | "generate" => { + "g" | "generate" => { let outfile = "" - on_generate_infras $infras_list $check $wait $outfile $name $serverpos + on_generate_infras $infras_list $check $wait $outfile $name $serverpos } - "t" | "taskserv" => { + "t" | "taskserv" => { let hostname = if ($ops | str starts-with "-") { "" } else { ($ops | split row "-" | find -v -r "^--" | get -o 1 | default "" | str trim) } on_taskserv_infras $infras_list $check $name $hostname --iptype $iptype } - "cost" | "price" => { - let match_host = if ($name | str starts-with "-") { + "cost" | "price" => { + let match_host = if ($name | str starts-with "-") { "" - } else { + } else { $name } infras_walk_by $infras_list $match_host $check false } - "list" => { + "list" => { #use defs/lists.nu on_list on_list "infras" ($onsel | default "") "" }, - _ => { + _ => { invalid_task "infra" $task --end }, - } - if not $env.PROVISIONING_DEBUG { end_run "" } + } + if not $env.PROVISIONING_DEBUG { end_run "" } } diff --git a/nulib/provisioning layer b/nulib/provisioning layer index 64b8ca8..00f2889 100755 --- a/nulib/provisioning layer +++ b/nulib/provisioning layer @@ -56,7 +56,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } let workspace_name = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { ($ops | split row " " | get 0) diff --git a/nulib/provisioning module b/nulib/provisioning module index e994481..9b967c7 100755 --- a/nulib/provisioning module +++ b/nulib/provisioning module @@ -74,7 +74,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } $env.PROVISIONING_MODULE = "module" @@ -333,4 +333,4 @@ def main [ exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/provisioning orchestrator b/nulib/provisioning orchestrator index eb63850..cca0d95 100755 --- a/nulib/provisioning orchestrator +++ b/nulib/provisioning orchestrator @@ -332,4 +332,4 @@ def orchestrator_logs [ print $"πŸ“‹ Last ($lines) lines from orchestrator logs:" ^tail -n ($lines | into string) $log_file } -} \ No newline at end of file +} diff --git a/nulib/provisioning pack b/nulib/provisioning pack index d223ddf..50692e9 100755 --- a/nulib/provisioning pack +++ b/nulib/provisioning pack @@ -62,7 +62,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } let package_name = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { ($ops | split row " " | get 0) } else { @@ -177,4 +177,4 @@ def main [ exit 1 } } -} \ No newline at end of file +} diff --git a/nulib/provisioning server b/nulib/provisioning server index 0fc1c4b..669873f 100755 --- a/nulib/provisioning server +++ b/nulib/provisioning server @@ -70,9 +70,16 @@ def main [ } if $debug { $env.PROVISIONING_DEBUG = true } if $metadata { $env.PROVISIONING_METADATA = true } - # for $arg in $args { print $arg } + # DEBUG: Print received args + if ($env.PROVISIONING_DEBUG? | default false) { + print $"DEBUG provisioning server: args length = ($args | length)" >&2 + for arg in $args { print $"DEBUG provisioning server: arg = '($arg)'" >&2 } + } let task = if ($args | length) > 0 { ($args| get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } + if ($env.PROVISIONING_DEBUG? | default false) { + print $"DEBUG provisioning server: task = '($task)', ops = '($ops)'" >&2 + } $env.PROVISIONING_MODULE = "server" match $task { "upcloud" => { diff --git a/nulib/provisioning setup b/nulib/provisioning setup index ec26332..dbcd994 100755 --- a/nulib/provisioning setup +++ b/nulib/provisioning setup @@ -106,4 +106,4 @@ def show-setup-help [] { print "For help on specific commands:" print " provisioning setup <command> --help" print "" -} +} diff --git a/nulib/provisioning taskserv b/nulib/provisioning taskserv index 227956b..7d2a2ea 100755 --- a/nulib/provisioning taskserv +++ b/nulib/provisioning taskserv @@ -1,13 +1,13 @@ -#!/usr/bin/env nu +#!/usr/bin/env nu # Info: Script to run Provisioning -# Author: JesusPerezLorenzo +# Author: JesusPerezLorenzo # Release: 1.0.4 # Date: 6-2-2024 #use std # assert use std log -use lib_provisioning * +use lib_provisioning * use env.nu * @@ -17,65 +17,65 @@ use taskservs * export def "main help" [ --src: string = "" --notitles # not tittles -] { - if $notitles == null or not $notitles { show_titles } +] { + if $notitles == null or not $notitles { show_titles } ^($env.PROVISIONING_NAME) "-mod" "taskserv" "--help" _print (provisioning_options $src) - if not $env.PROVISIONING_DEBUG { end_run "" } + if not $env.PROVISIONING_DEBUG { end_run "" } } # > Task and Services for servers def main [ - ...args: string # Other options, use help to get info + ...args: string # Other options, use help to get info --iptype: string = "public" # Ip type to connect -v # Show version - -i # Show Info + -i # Show Info --version (-V) # Show version with title --info (-I) # Show Info with title --about (-a) # Show About - --infra (-i): string # Infra directory - --settings (-s): string # Settings path - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created + --infra (-i): string # Infra directory + --settings (-s): string # Settings path + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created --yes (-y) # Confirm task - --wait (-w) # Wait servers to be created - --select: string # Select with taskservice as option + --wait (-w) # Wait servers to be created + --select: string # Select with taskservice as option --onsel: string # On selection: e (edit) | v (view) | l (list) --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for taskservice and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug - --nc # Not clean working settings + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for taskservice and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --nc # Not clean working settings --metadata # Error with metadata (-xm) --notitles # Do not show banner titles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { - $env.PROVISIONING_OUT = $out + $env.PROVISIONING_OUT = $out $env.PROVISIONING_NO_TERMINAL = true } provisioning_init $helpinfo "taskserv" $args if $version or $v { ^$env.PROVISIONING_NAME -v ; exit } if $info or $i { ^$env.PROVISIONING_NAME -i ; exit } - if $about { + if $about { #use defs/about.nu [ about_info ] - _print (get_about_info) - exit + _print (get_about_info) + exit } - if $debug { $env.PROVISIONING_DEBUG = true } - let use_debug = if $debug or $env.PROVISIONING_DEBUG { "-x" } else { "" } - if $metadata { $env.PROVISIONING_METADATA = true } + if $debug { $env.PROVISIONING_DEBUG = true } + let use_debug = if $debug or $env.PROVISIONING_DEBUG { "-x" } else { "" } + if $metadata { $env.PROVISIONING_METADATA = true } # for $arg in $args { print $arg } - let task = if ($args | length) > 0 { ($args| get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let task = if ($args | length) > 0 { ($args| get 0) } else { "" } + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } match $task { "h" | "help" => { # Redirect to main categorized help system exec ($env.PROVISIONING_NAME) "help" "infrastructure" "--notitles" }, - "sed" => { + "sed" => { if $ops == "" { (throw-error $"πŸ›‘ No file found" $"for (_ansi yellow_bold)sops(_ansi reset) edit") exit 1 @@ -86,31 +86,31 @@ def main [ if $env.PROVISIONING_SOPS? == null { let curr_settings = (find_get_settings --infra $infra --settings $settings) $env.CURRENT_INFRA_PATH = $"($curr_settings.infra_path)/($curr_settings.infra)" - use sops_env.nu + use sops_env.nu } #use sops on_sops on_sops "sed" $ops }, - "c" | "create" => { + "c" | "create" => { exec ($env.PROVISIONING_NAME) $use_debug "-mod" "taskserv" "create" ...($ops | split row " ") --notitles } - "d" | "delete" => { + "d" | "delete" => { exec ($env.PROVISIONING_NAME) $use_debug "-mod" "taskserv" "delete" ...($ops | split row " ") --notitles } - "g" | "generate" => { + "g" | "generate" => { exec ($env.PROVISIONING_NAME) $use_debug "-mod" "taskserv" "generate" ...($ops | split row " ") --notitles } - "l"| "list" => { + "l"| "list" => { #use defs/lists.nu on_list on_list "taskservs" ($onsel | default "") "" }, - "qr" => { + "qr" => { #use utils/qr.nu * make_qr }, - _ => { + _ => { invalid_task "taskserv" $task --end }, - } - if not $env.PROVISIONING_DEBUG { end_run "" } + } + if not $env.PROVISIONING_DEBUG { end_run "" } } diff --git a/nulib/provisioning template b/nulib/provisioning template index fe60647..ae646f8 100755 --- a/nulib/provisioning template +++ b/nulib/provisioning template @@ -62,7 +62,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } let template_name = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { ($ops | split row " " | get 0) } else { @@ -437,4 +437,4 @@ def template_layer_info [ print "" } } -} \ No newline at end of file +} diff --git a/nulib/provisioning version b/nulib/provisioning version index a5e0cfb..cf9096d 100755 --- a/nulib/provisioning version +++ b/nulib/provisioning version @@ -85,7 +85,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } let component_name = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { ($ops | split row " " | get 0) } else { @@ -297,4 +297,4 @@ def version_tools [ print "πŸ”§ Tool Versions:" $results | select id configured installed status | table } -} \ No newline at end of file +} diff --git a/nulib/provisioning workflow b/nulib/provisioning workflow index 82c5793..64cc48e 100755 --- a/nulib/provisioning workflow +++ b/nulib/provisioning workflow @@ -33,9 +33,9 @@ def main [ print "STEP 1: Technology Detection" print "────────────────────────────" - + let detector_bin = (detect-binary-path) - + if not ($detector_bin | path exists) { print -e "❌ Detector binary not found" return @@ -57,7 +57,7 @@ def main [ # Run completion print "STEP 2: Infrastructure Completion" print "─────────────────────────────────" - + let complete_result = (^$detector_bin complete $project_path --format json | complete) if $complete_result.exit_code != 0 { @@ -80,7 +80,7 @@ def main [ # Helper: Locate the provisioning-detector binary def detect-binary-path [] { let env_prov = ($env.PROVISIONING? | default "") - + let possible_paths = if ($env_prov | is-not-empty) { [ ($env_prov | path join "platform" "target" "debug" "provisioning-detector") diff --git a/nulib/provisioning workspace b/nulib/provisioning workspace index 8365c51..2ff57b0 100755 --- a/nulib/provisioning workspace +++ b/nulib/provisioning workspace @@ -65,7 +65,7 @@ def main [ if $metadata { $env.PROVISIONING_METADATA = true } let task = if ($args | length) > 0 { ($args | get 0) } else { "" } - let ops = $"($env.PROVISIONING_ARGS? | default "") " | str replace $" ($task) " "" | str trim + let ops = if ($args | length) > 1 { ($args | skip 1 | str join " ") } else { "" } # Extract workspace path (first non-flag argument) let workspace_path = if ($ops | is-not-empty) and not ($ops | str starts-with "-") { diff --git a/nulib/provisioning-nu b/nulib/provisioning-nu new file mode 100755 index 0000000..5567b29 --- /dev/null +++ b/nulib/provisioning-nu @@ -0,0 +1,21 @@ +#!/usr/bin/env nu +# Lightweight entry point for interactive nu sessions +# Skips heavy module loading to start the prompt quickly + +# This script is loaded but doesn't execute - the shell continues interactively +# The export-env block runs during initialization + +export-env { + $env.NU_LIB_DIRS = [ + "/Users/Akasha/project-provisioning/provisioning/core/nulib", + "/opt/provisioning/core/nulib", + "/usr/local/provisioning/core/nulib" + ] + $env.PROVISIONING = "/Users/Akasha/project-provisioning/provisioning" +} + +# Load only essential utilities +use lib_provisioning * + +print "βœ“ Provisioning interactive shell ready" +print "" diff --git a/nulib/secrets_env.nu b/nulib/secrets_env.nu index 512db05..6dd0175 100644 --- a/nulib/secrets_env.nu +++ b/nulib/secrets_env.nu @@ -2,4 +2,4 @@ use lib_provisioning/secrets/lib.nu setup_secret_env export-env { setup_secret_env -} \ No newline at end of file +} diff --git a/nulib/servers/create.nu b/nulib/servers/create.nu index be2d2ad..3c089e3 100644 --- a/nulib/servers/create.nu +++ b/nulib/servers/create.nu @@ -36,11 +36,15 @@ export def "main create" [ set-provisioning-out $out set-provisioning-no-terminal true } - provisioning_init $helpinfo "servers create" $args + # Convert args to list of strings for provisioning_init + let string_args = ($args | each { $in | into string }) + provisioning_init $helpinfo "servers create" $string_args if $debug { set-debug-enabled true } if $metadata { set-metadata-enabled true } if $name != null and $name != "h" and $name != "help" { - let curr_settings = (find_get_settings --infra $infra --settings $settings) + let infra_arg = if ($infra | is-empty) { null } else { $infra } + let settings_arg = if ($settings | is-empty) { null } else { $settings } + let curr_settings = (find_get_settings --infra $infra_arg --settings $settings_arg) if ($curr_settings.data.servers | find $name| length) == 0 { _print $"πŸ›‘ invalid name ($name)" exit 1 @@ -60,7 +64,14 @@ export def "main create" [ let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $" ($task) " "" | str trim let run_create = { - let curr_settings = (find_get_settings --infra $infra --settings $settings) + # Convert empty strings to null for auto-detection to work + let infra_arg = if ($infra | is-empty) { null } else { $infra } + let settings_arg = if ($settings | is-empty) { null } else { $settings } + let curr_settings = (find_get_settings --infra $infra_arg --settings $settings_arg) + if ($curr_settings | is-empty) or ($curr_settings.wk_path? | is-empty) { + _print "πŸ›‘ Failed to load settings" + return { status: false, error: "settings_load_failed" } + } set-wk-cnprov $curr_settings.wk_path let match_name = if $name == null or $name == "" { "" } else { $name} on_create_servers $curr_settings $check $wait $outfile $match_name $serverpos --notitles=$notitles --orchestrated=$orchestrated --orchestrator=$orchestrator @@ -95,7 +106,7 @@ export def on_create_servers [ --orchestrator: string = "http://localhost:8080" # Orchestrator URL ]: nothing -> record { - # Authentication check for server creation + # Authentication check for server creation (only if actually creating, not in check mode) if not $check { let environment = (config-get "environment" "dev") let operation_name = $"server create (($hostname | default 'all'))" @@ -117,7 +128,7 @@ export def on_create_servers [ log-authenticated-operation "server_create" { hostname: ($hostname | default "all") infra: $settings.infra - environment: $env + environment: $environment orchestrated: $orchestrated } } @@ -208,8 +219,11 @@ export def on_create_servers [ mw_create_cache $ok_settings $it.item false } } - servers_walk_by_costs $ok_settings $match_hostname $check true - server_ssh $ok_settings "" "pub" false "" $check | ignore + # Skip pricing and SSH setup in check mode + if not $check { + servers_walk_by_costs $ok_settings $match_hostname $check true + server_ssh $ok_settings "" "pub" false "" $check | ignore + } # Show next-step hints after successful creation if not $check { @@ -228,6 +242,133 @@ export def create_server [ ]: nothing -> bool { ## Provider middleware now available through lib_provisioning #use utils.nu * + + # In check mode, show what would be created + if $check { + # Search for template in workspace .providers first, then in system providers + let workspace_infra_path = ($settings.src_path | path dirname | path dirname) + let workspace_template = ($workspace_infra_path | path join ".providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2") + let server_template = if ($workspace_template | path exists) { + $workspace_template + } else { + (get-base-path | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2") + } + + # Temporarily disable NO_TERMINAL to ensure check output is displayed + let old_no_terminal = ($env.PROVISIONING_NO_TERMINAL? | default false) + $env.PROVISIONING_NO_TERMINAL = false + + _print $"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + _print $"Check: Create server (_ansi cyan_bold)($server.hostname)(_ansi reset) with provider (_ansi green_bold)($server.provider)(_ansi reset)" + _print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if ($server_template | path exists) { + _print $"\nπŸ“‹ Template: ($server_template)" + + # Show template rendering info + _print $"\nπŸ”§ Generated script:" + _print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + # Build complete context record with all variables the template expects + # The template needs: servers (array), defaults (record), match_server, provisioning_vers, now, debug, use_time, wait, runset, wk_file + let template_context = { + servers: [$server] + defaults: {} + match_server: $server.hostname + provisioning_vers: "1.0.4" + now: (date now | format date '%Y-%m-%d %H:%M:%S') + debug: "no" + use_time: "false" + wait: false + runset: {output_format: "yaml"} + wk_file: ($settings.wk_path | path join "creation_script.sh") + } + + # Try to render the template with daemon first, fallback to plugin + if ($server_template | path exists) { + let absolute_template = (($server_template | path expand) | str trim) + let template_content = (open $absolute_template) + + # First try: Use Tera daemon (50-100x faster for batch operations) + let use_daemon = (is-tera-daemon-available) + let rendered = if $use_daemon { + let daemon_result = (do { tera-render-daemon $template_content $template_context --name ($server.hostname) } | complete) + if $daemon_result.exit_code == 0 { + $daemon_result.stdout + } else { + # Fallback to plugin if daemon fails + if (get-use-tera-plugin) { + let tera_loaded = (plugin list | where name == "tera" | length) > 0 + if not $tera_loaded { + (plugin use tera) + } + ($template_context | tera-render $absolute_template) + } else { + error make {msg: "Template rendering not available (no daemon, no plugin)"} + } + } + } else if (get-use-tera-plugin) { + # Fallback: Use tera plugin if daemon not available + let tera_loaded = (plugin list | where name == "tera" | length) > 0 + if not $tera_loaded { + (plugin use tera) + } + ($template_context | tera-render $absolute_template) + } else { + error make {msg: "Template rendering not available (no daemon, no plugin)"} + } + + # Handle outfile parameter: save to file if provided, otherwise print to stdout + let has_outfile = ($outfile != null and ($outfile | str length) > 0) + if $has_outfile { + # Expand the outfile path to absolute + let absolute_outfile = ($outfile | path expand) + # Create parent directories if they don't exist + let outfile_dir = ($absolute_outfile | path dirname) + if not ($outfile_dir | path exists) { + ^mkdir -p $outfile_dir + } + # Write rendered content to file + $rendered | save --force $absolute_outfile + _print $"βœ… Script saved to: ($absolute_outfile)" + } else { + _print $rendered + } + } else { + _print $"\n⚠️ Template file not found" + _print $" Template path: ($server_template)" + _print $" Server: ($server.hostname)" + } + + if false { + _print $"⚠️ Template rendering not available (tera plugin not installed)" + _print $"\nπŸ“ Template variables that would be used:" + _print $" β€’ hostname = ($server.hostname)" + _print $" β€’ provider = ($server.provider)" + _print $" β€’ plan = ($server.plan)" + _print $" β€’ zone = ($server.zone | default 'default')" + } + + _print $"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + _print $"\nβœ… Check completed successfully" + _print $" This server would be created with:" + _print $" β€’ Hostname: ($server.hostname)" + _print $" β€’ Provider: ($server.provider)" + _print $" β€’ Plan: ($server.plan)" + _print $" β€’ Zone: ($server.zone | default 'default')" + _print $"\n To actually create, run without --check flag" + } else { + _print $"\n⚠️ Template not found: ($server_template)" + $env.PROVISIONING_NO_TERMINAL = $old_no_terminal + return false + } + + # Restore original NO_TERMINAL setting + $env.PROVISIONING_NO_TERMINAL = $old_no_terminal + return true + } + let server_info = (mw_server_info $server true) # Check if server_info is a record, otherwise it's an error (empty or string) @@ -251,7 +392,6 @@ export def create_server [ (get-base-path | path join "extensions" | path join "providers" | path join $server.provider | path join "templates" | path join $"($server.provider)_servers.j2") } let create_result = on_server_template $server_template $server $index $check false $wait $settings $outfile - if $check { return true } if not $create_result { return false } let server_info = (mw_server_info $server true) check_server $settings $server $index $server_info $check $wait $settings $outfile @@ -355,4 +495,4 @@ export def check_server [ } } true -} \ No newline at end of file +} diff --git a/nulib/servers/delete.nu b/nulib/servers/delete.nu index 4cd1f99..8b626df 100644 --- a/nulib/servers/delete.nu +++ b/nulib/servers/delete.nu @@ -4,21 +4,21 @@ use ../lib_provisioning/config/accessor.nu * # > Delete Server export def "main delete" [ name?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # Infra directory + ...args # Args for create command + --infra (-i): string # Infra directory --keepstorage # keep storage - --settings (-s): string # Settings path + --settings (-s): string # Settings path --yes (-y) # confirm delete --outfile (-o): string # Output file - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created - --wait (-w) # Wait servers to be created - --select: string # Select with task as option + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created + --wait (-w) # Wait servers to be created + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --helpinfo (-h) # For more details use options "help" (no dashes) @@ -29,32 +29,32 @@ export def "main delete" [ set-provisioning-no-terminal true } provisioning_init $helpinfo "servers delete" $args - if $debug { set-debug-enabled true } - if $metadata { set-metadata-enabled true } + if $debug { set-debug-enabled true } + if $metadata { set-metadata-enabled true } if $name != null and $name != "h" and $name != "help" and not ($name | str contains "storage") { let curr_settings = (find_get_settings --infra $infra --settings $settings) if ($curr_settings.data.servers | find $name| length) == 0 { - _print $"πŸ›‘ invalid name ($name)" + _print $"πŸ›‘ invalid name ($name)" exit 1 } } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = (((get-provisioning-args) | str replace "delete " " " )) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = (((get-provisioning-args) | str replace "delete " " " )) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $"($task) " "" | str trim - let run_delete = { + let run_delete = { let curr_settings = (find_get_settings --infra $infra --settings $settings) set-wk-cnprov $curr_settings.wk_path - on_delete_servers $curr_settings $keepstorage $wait $name $serverpos + on_delete_servers $curr_settings $keepstorage $wait $name $serverpos } match $task { "" if $name == "h" => { @@ -64,13 +64,13 @@ export def "main delete" [ ^$"(get-provisioning-name)" -mod server delete --help _print (provisioning_options "delete") }, - "" if ($name | default "" | str contains "storage") => { + "" if ($name | default "" | str contains "storage") => { let curr_settings = (find_get_settings --infra $infra --settings $settings) on_delete_server_storage $curr_settings $wait "" $serverpos }, - "" | "d"| "delete" => { + "" | "d"| "delete" => { if not $yes or not ((get-provisioning-args | str contains "--yes")) { - _print $"Run (_ansi red_bold)delete servers(_ansi reset) (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " + _print $"Run (_ansi red_bold)delete servers(_ansi reset) (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " let user_input = (input --numchar 3) if $user_input != "yes" and $user_input != "YES" { exit 1 @@ -78,93 +78,93 @@ export def "main delete" [ } let result = desktop_run_notify $"(get-provisioning-name) servers delete" "-> " $run_delete --timeout 11sec }, - _ => { + _ => { invalid_task "servers delete" $task --end } - } - if not (is-debug-enabled) { end_run "" } -} + } + if not (is-debug-enabled) { end_run "" } +} export def on_delete_server_storage [ - settings: record # Settings record + settings: record # Settings record wait: bool # Wait for creation hostname?: string # Server hostname in settings - serverpos?: int # Server position in settings + serverpos?: int # Server position in settings ]: nothing -> list { - #use lib_provisioning * + #use lib_provisioning * #use utils.nu * - let match_hostname = if $hostname != null and $hostname != "" { - $hostname - } else if $serverpos != null { + let match_hostname = if $hostname != null and $hostname != "" { + $hostname + } else if $serverpos != null { let total = $settings.data.servers | length - let pos = if $serverpos == 0 { + let pos = if $serverpos == 0 { _print $"Use number form 1 to ($total)" $serverpos - } else if $serverpos <= $total { + } else if $serverpos <= $total { $serverpos - 1 - } else { - (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" + } else { + (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" "on_create" --span (metadata $serverpos).span) exit 1 } ($settings.data.servers | get $pos).hostname } _print $"Delete storage (_ansi blue_bold)($settings.data.servers | length)(_ansi reset) server\(s\) in parallel (_ansi blue_bold)>>> πŸŒ₯ >>> (_ansi reset)\n" - $settings.data.servers | enumerate | par-each { |it| - if ($match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname) { + $settings.data.servers | enumerate | par-each { |it| + if ($match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname) { if not (mw_delete_server_storage $settings $it.item false) { return false } _print $"\n(_ansi blue_reverse)----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- oOo ----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- (_ansi reset)\n" - } + } } } export def on_delete_servers [ - settings: record # Settings record + settings: record # Settings record keep_storage: bool # keep storage wait: bool # Wait for creation hostname?: string # Server hostname in settings - serverpos?: int # Server position in settings + serverpos?: int # Server position in settings ]: nothing -> record { - #use lib_provisioning * + #use lib_provisioning * #use utils.nu * - let match_hostname = if $hostname != null and $hostname != "" { - $hostname - } else if $serverpos != null { + let match_hostname = if $hostname != null and $hostname != "" { + $hostname + } else if $serverpos != null { let total = $settings.data.servers | length - let pos = if $serverpos == 0 { + let pos = if $serverpos == 0 { _print $"Use number form 1 to ($total)" $serverpos - } else if $serverpos <= $total { + } else if $serverpos <= $total { $serverpos - 1 - } else { - (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" + } else { + (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" "on_create" --span (metadata $serverpos).span) exit 1 } ($settings.data.servers | get $pos).hostname } _print $"Delete (_ansi blue_bold)($match_hostname | length)(_ansi reset) server\(s\) in parallel (_ansi blue_bold)>>> πŸŒ₯ >>> (_ansi reset)\n" - $settings.data.servers | enumerate | par-each { |it| - if ( $match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname) { + $settings.data.servers | enumerate | par-each { |it| + if ( $match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname) { if ($it.item | get lock? | default false) { _print ($"(_ansi green)($it.item.hostname)(_ansi reset) is set to (_ansi purple)lock state(_ansi reset).\n" + $"Set (_ansi red)lock(_ansi reset) to False to allow delete. ") - } else { + } else { if (mw_delete_server $settings $it.item $keep_storage false) { - if (is-debug-enabled) { _print $"\n(_ansi red) error ($it.item.hostname)(_ansi reset)\n" } + if (is-debug-enabled) { _print $"\n(_ansi red) error ($it.item.hostname)(_ansi reset)\n" } } } - } + } } _print $"\n(_ansi blue_reverse)----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- oOo ----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- (_ansi reset)\n" - for server in $settings.data.servers { + for server in $settings.data.servers { if ($server | get lock? | default false) { continue } let already_created = (mw_server_exists $server false) if ($already_created) { - if (is-debug-enabled) { _print $"\n(_ansi red) error ($server.hostname)(_ansi reset)\n" } + if (is-debug-enabled) { _print $"\n(_ansi red) error ($server.hostname)(_ansi reset)\n" } } else { mw_clean_cache $settings $server false } } { status: true, error: "" } -} \ No newline at end of file +} diff --git a/nulib/servers/generate.nu b/nulib/servers/generate.nu index d899e4a..262f264 100644 --- a/nulib/servers/generate.nu +++ b/nulib/servers/generate.nu @@ -115,7 +115,7 @@ export def on_generate_servers [ } # let servers_path_0 = if ($settings.data.servers_paths | length) > 1 { #TODO } let servers_path_0 = ($settings.data.servers_paths | first | default null) - let servers_path = if ($servers_path_0 | str ends-with ".k") { $servers_path_0 } else { $"($servers_path_0).k"} + let servers_path = if ($servers_path_0 | str ends-with ".ncl") { $servers_path_0 } else { $"($servers_path_0).ncl"} #if not ($servers_path | path exists) { #(throw-error $"πŸ›‘ servers path" $"($servers_path) not found in ($settings.infra)" # "on_generate" --span (metadata $servers_path).span) @@ -133,7 +133,7 @@ export def on_generate_servers [ mut $servers_length = ($settings.data.servers | length) while true { _print $"(_ansi yellow)($servers_length)(_ansi reset) servers " - let servers_kcl = (open -r $full_servers_path | str replace --multiline --regex '^]' '') + let servers_nickel = (open -r $full_servers_path | str replace --multiline --regex '^]' '') # TODO SAVE A COPY let item_select = if ($select | is-empty) { let selection_pos = ($providers_list | each {|it| @@ -162,29 +162,29 @@ export def on_generate_servers [ continue } let template_path = ($item_path | path join (get-provisioning-generate-dirpath)) - let new_created = if not ($target_path | path join $"($item_select.name)_defaults.k" | path exists) { - ^cp -pr ($template_path | path join $"($item_select.name)_defaults.k.j2") ($target_path) - _print $"copy (_ansi green)($item_select.name)_defaults.k.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" + let new_created = if not ($target_path | path join $"($item_select.name)_defaults.ncl" | path exists) { + ^cp -pr ($template_path | path join $"($item_select.name)_defaults.ncl.j2") ($target_path) + _print $"copy (_ansi green)($item_select.name)_defaults.ncl.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" true } else { false } - if not ($full_servers_path | path exists) or ($servers_kcl | is-empty) or $servers_length == 0 { - ($"import ($item_select.name)_prov\nservers = [\n" + (open -r ($template_path | path join "servers.k.j2")) + "\n]" ) + if not ($full_servers_path | path exists) or ($servers_nickel | is-empty) or $servers_length == 0 { + ($"import ($item_select.name)_prov\nservers = [\n" + (open -r ($template_path | path join "servers.ncl.j2")) + "\n]" ) | save -f $"($full_servers_path).j2" - _print $"create (_ansi green)($item_select.name) servers.k.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" + _print $"create (_ansi green)($item_select.name) servers.ncl.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" } else { - let head_text = if not ($servers_kcl | str contains $"import ($item_select.name)") { + let head_text = if not ($servers_nickel | str contains $"import ($item_select.name)") { $"import ($item_select.name)_prov\n" } else {"" } print $"import ($item_select.name)" print $head_text - ($head_text + $servers_kcl + (open -r ($template_path | path join "servers.k.j2")) + "\n]" ) + ($head_text + $servers_nickel + (open -r ($template_path | path join "servers.ncl.j2")) + "\n]" ) | save -f $"($full_servers_path).j2" - _print $"add (_ansi green)($item_select.name) servers.k.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" + _print $"add (_ansi green)($item_select.name) servers.ncl.j2(_ansi reset) to (_ansi green)($settings.infra)(_ansi reset)" } generate_data_def $item_path $settings.infra ($settings.src_path | path join ($full_servers_path | path dirname)) $new_created $inputfile - # TODO CHECK if compiles KCL OR RECOVERY + # TODO CHECK if compiles Nickel OR RECOVERY # TODO ADD tasks for server if ($inputfile | is-not-empty) { break } $servers_length += 1 @@ -323,4 +323,4 @@ export def check_server [ } } true -} \ No newline at end of file +} diff --git a/nulib/servers/list.nu b/nulib/servers/list.nu new file mode 100644 index 0000000..5af34b6 --- /dev/null +++ b/nulib/servers/list.nu @@ -0,0 +1,61 @@ +use lib_provisioning * +use utils.nu * +use ../lib_provisioning/config/accessor.nu * + +# List all servers +export def "main list" [ + ...args # Args for list command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path + --outfile (-o): string # Output file + --check (-c) # Only check mode + --debug (-x) # Use Debug mode + --xm # Debug with PROVISIONING_METADATA + --xc # Debug for task and services locally + --xr # Debug for remote servers + --xld # Log level with DEBUG + --metadata # Error with metadata + --notitles # not titles + --helpinfo (-h) # For more details use options "help" + --out: string # Print Output format: json, yaml, text (default) +]: nothing -> nothing { + if ($out | is-not-empty) { + set-provisioning-out $out + set-provisioning-no-terminal true + } + + provisioning_init $helpinfo "servers list" $args + + if $debug { set-debug-enabled true } + if $metadata { set-metadata-enabled true } + + # Load server settings + let curr_settings = (find_get_settings --infra $infra --settings $settings) + + # Get servers info + let servers_table = (mw_servers_info $curr_settings) + + # Check if any servers exist + if ($servers_table | length) == 0 { + if (get-provisioning-out | is-empty) { + _print "No servers configured" + } else { + _print ([] | to json) "json" "result" "table" + } + } else { + # Display servers + if ($out | is-empty) { + # Terminal output with formatting + _print ($servers_table | table -i false) + } else { + # Structured output (JSON, YAML) + match (get-provisioning-out) { + "json" => { _print ($servers_table | to json) "json" "result" "table" } + "yaml" => { _print ($servers_table | to yaml) "yaml" "result" "table" } + _ => { _print ($servers_table | table -i false) } + } + } + } + + if not $notitles and not (is-debug-enabled) { end_run "" } +} diff --git a/nulib/servers/mod.nu b/nulib/servers/mod.nu index c939fb9..804bd76 100644 --- a/nulib/servers/mod.nu +++ b/nulib/servers/mod.nu @@ -1,9 +1,10 @@ -export use ops.nu * +export use ops.nu * -export use create.nu * -export use delete.nu * -export use generate.nu * -export use status.nu * -export use state.nu * -export use ssh.nu * -export use utils.nu * +export use create.nu * +export use delete.nu * +export use generate.nu * +export use list.nu * +export use status.nu * +export use state.nu * +export use ssh.nu * +export use utils.nu * diff --git a/nulib/servers/state.nu b/nulib/servers/state.nu index 822ea58..bff8499 100644 --- a/nulib/servers/state.nu +++ b/nulib/servers/state.nu @@ -121,4 +121,4 @@ export def on_state_servers [ _print $"\n(_ansi blue_reverse)----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- oOo ----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- (_ansi reset)\n" } } -} \ No newline at end of file +} diff --git a/nulib/servers/status.nu b/nulib/servers/status.nu index cef40f9..c248b21 100644 --- a/nulib/servers/status.nu +++ b/nulib/servers/status.nu @@ -76,4 +76,4 @@ export def "main status" [ } # "" | "create" if not $notitles and not (is-debug-enabled) { end_run "" } -} \ No newline at end of file +} diff --git a/nulib/servers/utils.nu b/nulib/servers/utils.nu index cdce4ff..6ba7528 100644 --- a/nulib/servers/utils.nu +++ b/nulib/servers/utils.nu @@ -497,31 +497,31 @@ export def provider_data_cache [ } if ($outfile_path | path exists) { _print $"βœ… (_ansi green_bold)($server.provider)(_ansi reset) (_ansi cyan_bold)cache settings(_ansi reset) saved in (_ansi yellow_bold)($outfile_path)(_ansi reset)" - _print $"To create a (_ansi purple)kcl(_ansi reset) for (_ansi cyan)defs(_ansi reset) file use:" - let k_file_path = $"($outfile_path | str replace $'.($out_extension)' '').k" - ^kcl import ($outfile_path) -o ($k_file_path) --force + _print $"To create a (_ansi purple)nickel(_ansi reset) for (_ansi cyan)defs(_ansi reset) file use:" + let k_file_path = $"($outfile_path | str replace $'.($out_extension)' '').ncl" + ^nickel import ($outfile_path) -o ($k_file_path) --force ^sed -i '1,4d;s/^{/_data = {/' $k_file_path '{ main = _data.main, priv = _data.priv }' | tee {save -a $k_file_path} | ignore - let res = ( ^kcl $k_file_path | complete) + let res = ( ^nickel $k_file_path | complete) if $res.exit_code == 0 { $res.stdout | save $"($k_file_path).yaml" --force - ^kcl import $"($k_file_path).yaml" -o ($k_file_path) --force + ^nickel import $"($k_file_path).yaml" -o ($k_file_path) --force ^sed -i '1,4d;s/^{/_data = {/' $k_file_path let content = (open $k_file_path --raw) let comment = $"# ($server.provider)" + " environment settings, if not set will be autogenerated in 'provider_path' (data/" + $server.provider + "_cache.yaml)" let from_scratch = (mw_start_cache_info $settings $server) - ($"# Info: KCL Settings created by (get-provisioning-name)\n# Date: (date now | format date '%Y-%m-%d %H:%M:%S')\n\n" + + ($"# Info: Nickel Settings created by (get-provisioning-name)\n# Date: (date now | format date '%Y-%m-%d %H:%M:%S')\n\n" + $"($comment)\n($from_scratch)" + - $"# Use a command like: '(get-provisioning-name) server cache -o /tmp/data.yaml' to genereate '/tmp/($server.provider)_data.k' for 'defs' settings\n" + - $"# then you can move genereated '/tmp/($server.provider)_data.k' to '/defs/($server.provider)_data.k' \n\n" + + $"# Use a command like: '(get-provisioning-name) server cache -o /tmp/data.yaml' to genereate '/tmp/($server.provider)_data.ncl' for 'defs' settings\n" + + $"# then you can move genereated '/tmp/($server.provider)_data.ncl' to '/defs/($server.provider)_data.ncl' \n\n" + $"import ($server.provider)_prov\n" + ($content | str replace '_data = {' $"($server.provider)_prov.Provision_($server.provider) {") | save $k_file_path --force ) let result = (encrypt_secret $k_file_path --quiet) if ($result | is-not-empty) { ($result | save --force $k_file_path) } - _print $"(_ansi purple)kcl(_ansi reset) for (_ansi cyan)defs(_ansi reset) file has been created at (_ansi green)($k_file_path)(_ansi reset)" + _print $"(_ansi purple)nickel(_ansi reset) for (_ansi cyan)defs(_ansi reset) file has been created at (_ansi green)($k_file_path)(_ansi reset)" } - #show_clip_to $"kcl import ($outfile_path) -o ($k_file_path) --force ; sed -i '1,4d;s/^{/_data = {/' ($k_file_path) ; echo '{ main = _data.main, priv = _data.priv }' >> ($k_file_path)" true + #show_clip_to $"nickel import ($outfile_path) -o ($k_file_path) --force ; sed -i '1,4d;s/^{/_data = {/' ($k_file_path) ; echo '{ main = _data.main, priv = _data.priv }' >> ($k_file_path)" true } } else { let cmd = (get-file-viewer) @@ -566,7 +566,7 @@ export def find_serversdefs [ mut defs = [] for it in ($settings | get data? | default {} | get servers_paths? | default []) { let name = ($it| str replace "/" "_") - let it_path = if ($it | str ends-with ".k") { $it } else { $"($it).k" } + let it_path = if ($it | str ends-with ".ncl") { $it } else { $"($it).ncl" } let path_def = ($src_path | path join $it_path ) let defs_srvs = if ($path_def | path exists ) { (open -r $path_def) @@ -576,11 +576,11 @@ export def find_serversdefs [ } ) } - let defaults_path = (get-base-path | path join "kcl" | path join "defaults.k") + let defaults_path = (get-base-path | path join "nickel" | path join "defaults.ncl") let defaults = if ($defaults_path | path exists) { (open -r $defaults_path | default "") } else { "" } - let path_main = (get-base-path | path join "kcl" | path join "server.k") + let path_main = (get-base-path | path join "nickel" | path join "server.ncl") let main = if ($path_main | path exists) { (open -r $path_main | default "") } else { "" } @@ -598,23 +598,23 @@ export def find_serversdefs [ }) let defs_providers = ($providers_list | each {|it| let it_path = ($src_path| path join "defs") - let defaults = if ($it_path | path join $"($it.name)_defaults.k" | path exists) { - (open -r ($it_path | path join $"($it.name)_defaults.k")) + let defaults = if ($it_path | path join $"($it.name)_defaults.ncl" | path exists) { + (open -r ($it_path | path join $"($it.name)_defaults.ncl")) } else { "" } - let def = if ($it_path | path join "servers.k" | path exists) { - (open -r ($it_path | path join "servers.k")) + let def = if ($it_path | path join "servers.ncl" | path exists) { + (open -r ($it_path | path join "servers.ncl")) } else { "" } { name: $it.name, path_def: $it_path, def: $def, defaults: $defaults } } | default []) let providers = ($providers_list | each {|it| - let it_path = (get-providers-path | path join $it.name | path join "kcl") - let defaults = if ($it_path | path join $"defaults_($it.name).k" | path exists) { - (open -r ($it_path | path join $"defaults_($it.name).k")) + let it_path = (get-providers-path | path join $it.name | path join "nickel") + let defaults = if ($it_path | path join $"defaults_($it.name).ncl" | path exists) { + (open -r ($it_path | path join $"defaults_($it.name).ncl")) } else { "" } - let def = if ($it_path | path join $"server_($it.name).k" | path exists) { - (open -r ($it_path | path join $"server_($it.name).k")) + let def = if ($it_path | path join $"server_($it.name).ncl" | path exists) { + (open -r ($it_path | path join $"server_($it.name).ncl")) } else { "" } { name: $it.name, path_def: $it_path, def: $def, defaults: $defaults @@ -659,4 +659,4 @@ export def find_provgendefs [ $provdefs } $prov_defs -} \ No newline at end of file +} diff --git a/nulib/sops_env.nu b/nulib/sops_env.nu index 7fd23bd..0155e34 100644 --- a/nulib/sops_env.nu +++ b/nulib/sops_env.nu @@ -9,21 +9,21 @@ export-env { } else { $env.PROVISIONING_SOPS = (get_def_sops $env.CURRENT_INFRA_PATH) $env.PROVISIONING_KAGE = (get_def_age $env.CURRENT_INFRA_PATH) - # let context = (setup_user_context) + # let context = (setup_user_context) # let kage_path = ($context | try { get "kage_path" } catch { "" | str replace "KLOUD_PATH" $env.PROVISIONING_KLOUD_PATH) } - # if $kage_path != "" { + # if $kage_path != "" { # $env.PROVISIONING_KAGE = $kage_path # } - } + } print $env - if $env.PROVISIONING_KAGE? != null { + if $env.PROVISIONING_KAGE? != null { $env.SOPS_AGE_KEY_FILE = $env.PROVISIONING_KAGE let key_parts = (grep "public key:" $env.SOPS_AGE_KEY_FILE | split row ":") - $env.SOPS_AGE_RECIPIENTS = if ($key_parts | length) > 1 { $key_parts | get 1 | str trim } else { "" } - if $env.SOPS_AGE_RECIPIENTS == "" { + $env.SOPS_AGE_RECIPIENTS = if ($key_parts | length) > 1 { $key_parts | get 1 | str trim } else { "" } + if $env.SOPS_AGE_RECIPIENTS == "" { print $"❗Error no key found in (_ansi red_bold)($env.SOPS_AGE_KEY_FILE)(_ansi reset) file for secure AGE operations " exit 1 } } } -} \ No newline at end of file +} diff --git a/nulib/taskservs/README.md b/nulib/taskservs/README.md index 0b754aa..9063f68 100644 --- a/nulib/taskservs/README.md +++ b/nulib/taskservs/README.md @@ -18,7 +18,7 @@ provisioning taskserv create kubernetes --check # Sandbox testing provisioning taskserv test kubernetes --runtime docker -``` +```plaintext --- @@ -26,7 +26,7 @@ provisioning taskserv test kubernetes --runtime docker | Command | Description | |---------|-------------| -| `taskserv validate <name>` | Multi-level validation (KCL, templates, scripts, dependencies) | +| `taskserv validate <name>` | Multi-level validation (Nickel, templates, scripts, dependencies) | | `taskserv check-deps <name>` | Check dependencies against infrastructure | | `taskserv create <name> --check` | Dry-run with preview (no actual deployment) | | `taskserv test <name>` | Test in sandbox container | @@ -36,18 +36,21 @@ provisioning taskserv test kubernetes --runtime docker ## Validation Levels ### 1. **Static Validation** -- βœ… KCL schema syntax + +- βœ… Nickel schema syntax - βœ… Jinja2 template syntax - βœ… Shell script validation (shellcheck) - ⚑ **Fast** - No infrastructure needed ### 2. **Dependency Validation** + - βœ… Required dependencies available - βœ… Conflict detection - βœ… Resource requirements - βœ… Health check configuration ### 3. **Check Mode (Dry-Run)** + - βœ… All static validations - βœ… Dependency checking - βœ… Configuration preview @@ -55,6 +58,7 @@ provisioning taskserv test kubernetes --runtime docker - 🚫 **No actual deployment** ### 4. **Sandbox Testing** + - βœ… Package prerequisites - βœ… Configuration validity - βœ… Script execution @@ -82,14 +86,14 @@ provisioning taskserv test kubernetes --runtime docker # 5. Deploy (after all checks pass) provisioning taskserv create kubernetes -``` +```plaintext ### Quick Validation ```bash # All validations in one command provisioning taskserv validate kubernetes --level all -v -``` +```plaintext ### CI/CD Integration @@ -109,13 +113,13 @@ deploy: - provisioning taskserv create kubernetes only: - main -``` +```plaintext --- ## Module Structure -``` +```plaintext taskservs/ β”œβ”€β”€ validate.nu # Main validation framework β”œβ”€β”€ deps_validator.nu # Dependency validation @@ -130,35 +134,40 @@ taskservs/ β”œβ”€β”€ utils.nu # Utilities β”œβ”€β”€ ops.nu # Operations └── mod.nu # Module exports -``` +```plaintext --- ## Key Features ### βœ… **Multi-Level Validation** + - Static, dependency, prerequisite, and health check validation - Fail-fast with clear error messages - Verbose mode for detailed output ### 🎯 **Enhanced Check Mode** + - Comprehensive dry-run before deployment - Configuration preview - File listing - No SSH required in check mode ### 🐳 **Sandbox Testing** + - Isolated container environment - Docker and Podman support - Keep containers for debugging - Multiple test scenarios ### πŸ“Š **Multiple Output Formats** + - Text (default, human-readable) - JSON (for automation) - YAML (for configuration) ### πŸ”— **Dependency Management** + - Automatic dependency detection - Conflict resolution - Resource requirement validation @@ -169,10 +178,12 @@ taskservs/ ## Dependencies ### Required + - Nushell 0.107.1+ -- KCL 0.11.3+ +- Nickel 0.11.3+ ### Optional + - `shellcheck` - Enhanced script validation - `docker` or `podman` - Sandbox testing - `glow` or `bat` - Better markdown rendering @@ -182,7 +193,7 @@ taskservs/ ## Documentation - [Complete Validation Guide](../../../docs/user/taskserv-validation-guide.md) -- [KCL Schema Patterns](../../../.claude/kcl_idiomatic_patterns.md) +- [Nickel Schema Patterns](../../../.claude/kcl_idiomatic_patterns.md) - [Taskserv Development](../../../docs/development/taskserv-development.md) --- @@ -192,15 +203,15 @@ taskservs/ ### Validation Errors ```bash -# Check KCL syntax -kcl fmt <file>.k +# Check Nickel syntax +nickel fmt <file>.ncl # Validate dependencies manually -kcl run extensions/taskservs/<name>/kcl/dependencies.k +nickel run extensions/taskservs/<name>/nickel/dependencies.ncl # Run with verbose output provisioning taskserv validate <name> -v -``` +```plaintext ### Sandbox Testing Issues @@ -213,7 +224,7 @@ provisioning taskserv test <name> --keep # Connect to container docker exec -it taskserv-test-<name> bash -``` +```plaintext ### shellcheck not found @@ -223,7 +234,7 @@ brew install shellcheck # Ubuntu/Debian apt install shellcheck -``` +```plaintext --- diff --git a/nulib/taskservs/check_mode.nu b/nulib/taskservs/check_mode.nu index 6b3b323..c4a88b9 100644 --- a/nulib/taskservs/check_mode.nu +++ b/nulib/taskservs/check_mode.nu @@ -178,13 +178,13 @@ export def run-check-mode [ # 1. Static validation _print $"\n(_ansi yellow)β†’ Running static validation...(_ansi reset)" let static_validation = { - kcl: (validate-kcl-schemas $taskserv_name --verbose=$verbose) + nickel: (validate-nickel-schemas $taskserv_name --verbose=$verbose) templates: (validate-templates $taskserv_name --verbose=$verbose) scripts: (validate-scripts $taskserv_name --verbose=$verbose) } let static_valid = ( - $static_validation.kcl.valid and + $static_validation.nickel.valid and $static_validation.templates.valid and $static_validation.scripts.valid ) diff --git a/nulib/taskservs/create.nu b/nulib/taskservs/create.nu index a9883ae..1afab30 100644 --- a/nulib/taskservs/create.nu +++ b/nulib/taskservs/create.nu @@ -5,27 +5,27 @@ use ../lib_provisioning/utils/ssh.nu * use ../lib_provisioning/config/accessor.nu * # Provider middleware now available through lib_provisioning -# > TaskServs create +# > TaskServs create export def "main create" [ task_name?: string # task in settings server?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # Infra directory - --settings (-s): string # Settings path + ...args # Args for create command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path --iptype: string = "public" # Ip type to connect --outfile (-o): string # Output file - --taskserv_pos (-p): int # Server position in settings - --check (-c) # Only check mode no taskservs will be created - --wait (-w) # Wait taskservs to be created - --select: string # Select with task as option + --taskserv_pos (-p): int # Server position in settings + --check (-c) # Only check mode no taskservs will be created + --wait (-w) # Wait taskservs to be created + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { @@ -34,43 +34,43 @@ export def "main create" [ } provisioning_init $helpinfo "taskserv create" ([($task_name | default "") ($server | default "")] | append $args) if $debug { set-debug-enabled true } - if $metadata { set-metadata-enabled true } + if $metadata { set-metadata-enabled true } let curr_settings = (find_get_settings --infra $infra --settings $settings) let task = ((get-provisioning-args) | split row " "| try { get 0 } catch { null } - let options = if ($args | length) > 0 { - $args - } else { + let options = if ($args | length) > 0 { + $args + } else { let str_task = ((get-provisioning-args) | str replace $"($task) " "" | str replace $"($task_name) " "" | str replace $"($server) " "") ($str_task | split row "-" | try { get 0 } catch { "" | str trim ) } - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $"($task_name) " "" | str trim - let run_create = { + let run_create = { let curr_settings = (settings_with_env $curr_settings) set-wk-cnprov $curr_settings.wk_path - let arr_task = if $task_name == null or $task_name == "" or $task_name == "-" { [] } else { $task_name | split row "/" } - let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } - let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } - let match_server = if $server == null or $server == "" { "" } else { $server} + let arr_task = if $task_name == null or $task_name == "" or $task_name == "-" { [] } else { $task_name | split row "/" } + let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } + let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } + let match_server = if $server == null or $server == "" { "" } else { $server} on_taskservs $curr_settings $match_task $match_task_profile $match_server $iptype $check } match $task { - "" if $task_name == "h" => { + "" if $task_name == "h" => { ^$"((get-provisioning-name))" -mod taskserv update help --notitles }, - "" if $task_name == "help" => { + "" if $task_name == "help" => { ^$"((get-provisioning-name))" -mod taskserv update --help _print (provisioning_options "update") }, - "c" | "create" | "" => { + "c" | "create" | "" => { let result = desktop_run_notify $"((get-provisioning-name)) taskservs create" "-> " $run_create --timeout 11sec }, - _ => { - if $task_name != "" {_print $"πŸ›‘ invalid_option ($task_name)" } + _ => { + if $task_name != "" {_print $"πŸ›‘ invalid_option ($task_name)" } _print $"\nUse (_ansi blue_bold)((get-provisioning-name)) -h(_ansi reset) for help on commands and options" } - } + } # "" | "create" - #if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + #if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/taskservs/delete.nu b/nulib/taskservs/delete.nu index 7a5b83b..ea2a072 100644 --- a/nulib/taskservs/delete.nu +++ b/nulib/taskservs/delete.nu @@ -4,21 +4,21 @@ use ../lib_provisioning/config/accessor.nu * # > TaskServs Delete export def "main delete" [ name?: string # Server hostname in settings - ...args # Args for create command - --infra (-i): string # Infra directory + ...args # Args for create command + --infra (-i): string # Infra directory --keepstorage # keep storage - --settings (-s): string # Settings path + --settings (-s): string # Settings path --yes (-y) # confirm delete --outfile (-o): string # Output file - --serverpos (-p): int # Server position in settings - --check (-c) # Only check mode no servers will be created - --wait (-w) # Wait servers to be created - --select: string # Select with task as option + --serverpos (-p): int # Server position in settings + --check (-c) # Only check mode no servers will be created + --wait (-w) # Wait servers to be created + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote servers PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles --helpinfo (-h) # For more details use options "help" (no dashes) @@ -32,31 +32,31 @@ export def "main delete" [ #parse_help_command "server create" $name --ismod --end #print "on taskservs main delete" if $debug { set-debug-enabled true } - if $metadata { set-metadata-enabled true } + if $metadata { set-metadata-enabled true } if $name != null and $name != "h" and $name != "help" { let curr_settings = (find_get_settings --infra $infra --settings $settings) if ($curr_settings.data.servers | find $name| length) == 0 { - _print $"πŸ›‘ invalid name ($name)" + _print $"πŸ›‘ invalid name ($name)" exit 1 } } - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = ((get-provisioning-args) | str replace "delete " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = ((get-provisioning-args) | str replace "delete " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $"($task) " "" | str trim - let run_delete = { + let run_delete = { let curr_settings = (find_get_settings --infra $infra --settings $settings) set-wk-cnprov $curr_settings.wk_path - on_delete_taskservs $curr_settings $keepstorage $wait $name $serverpos + on_delete_taskservs $curr_settings $keepstorage $wait $name $serverpos } match $task { "" if $name == "h" => { @@ -66,9 +66,9 @@ export def "main delete" [ ^$"((get-provisioning-name))" -mod takserv delete --help _print (provisioning_options "delete") }, - "" => { + "" => { if not $yes or not ((get-provisioning-args) | str contains "--yes") { - _print $"Run (_ansi red_bold)delete servers(_ansi reset) (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " + _print $"Run (_ansi red_bold)delete servers(_ansi reset) (_ansi green_bold)($name)(_ansi reset) type (_ansi green_bold)yes(_ansi reset) ? " let user_input = (input --numchar 3) if $user_input != "yes" and $user_input != "YES" { exit 1 @@ -76,55 +76,55 @@ export def "main delete" [ } let result = desktop_run_notify $"((get-provisioning-name)) servers delete" "-> " $run_delete --timeout 11sec }, - _ => { - if $task != "" { _print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { _print $"πŸ›‘ invalid_option ($task)" } _print $"\nUse (_ansi blue_bold)((get-provisioning-name)) -h(_ansi reset) for help on commands and options" } - } - if not (is-debug-enabled) { end_run "" } -} + } + if not (is-debug-enabled) { end_run "" } +} export def on_delete_taskservs [ - settings: record # Settings record + settings: record # Settings record keep_storage: bool # keep storage wait: bool # Wait for creation hostname?: string # Server hostname in settings - serverpos?: int # Server position in settings + serverpos?: int # Server position in settings ]: nothing -> record { - #use lib_provisioning * + #use lib_provisioning * #use utils.nu * -# TODO review +# TODO review return { status: true, error: "" } - let match_hostname = if $hostname != null and $hostname != "" { - $hostname - } else if $serverpos != null { + let match_hostname = if $hostname != null and $hostname != "" { + $hostname + } else if $serverpos != null { let total = $settings.data.servers | length - let pos = if $serverpos == 0 { + let pos = if $serverpos == 0 { _print $"Use number form 1 to ($total)" $serverpos - } else if $serverpos <= $total { + } else if $serverpos <= $total { $serverpos - 1 - } else { - (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" + } else { + (throw-error $"πŸ›‘ server pos" $"($serverpos) from ($total) servers" "on_create" --span (metadata $serverpos).span) exit 1 } ($settings.data.servers | get $pos).hostname } _print $"Delete (_ansi blue_bold)($settings.data.servers | length)(_ansi reset) server\(s\) in parallel (_ansi blue_bold)>>> πŸŒ₯ >>> (_ansi reset)\n" - $settings.data.servers | enumerate | par-each { |it| - if $match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname { + $settings.data.servers | enumerate | par-each { |it| + if $match_hostname == null or $match_hostname == "" or $it.item.hostname == $match_hostname { if not (mw_delete_server $settings $it.item $keep_storage false) { return false } _print $"\n(_ansi blue_reverse)----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- oOo ----πŸŒ₯ ----πŸŒ₯ ----πŸŒ₯ ---- (_ansi reset)\n" - } + } } - for server in $settings.data.servers { + for server in $settings.data.servers { let already_created = (mw_server_exists $server false) if ($already_created) { return { status: false, error: $"($server.hostname) created" } } } { status: true, error: "" } -} \ No newline at end of file +} diff --git a/nulib/taskservs/deps_validator.nu b/nulib/taskservs/deps_validator.nu index dd9192f..170a8b3 100644 --- a/nulib/taskservs/deps_validator.nu +++ b/nulib/taskservs/deps_validator.nu @@ -5,17 +5,17 @@ use lib_provisioning * use utils.nu * use ../lib_provisioning/config/accessor.nu * -# Validate taskserv dependencies from KCL definition +# Validate taskserv dependencies from Nickel definition export def validate-dependencies [ taskserv_name: string settings: record --verbose (-v) ]: nothing -> record { let taskservs_path = (get-taskservs-path) - let taskserv_kcl_path = ($taskservs_path | path join $taskserv_name "kcl") + let taskserv_schema_path = ($taskservs_path | path join $taskserv_name "nickel") - # Check if taskserv has dependencies.k - let deps_file = ($taskserv_kcl_path | path join "dependencies.k") + # Check if taskserv has dependencies.ncl + let deps_file = ($taskserv_schema_path | path join "dependencies.ncl") if not ($deps_file | path exists) { return { @@ -31,22 +31,22 @@ export def validate-dependencies [ _print $"Validating dependencies for (_ansi yellow_bold)($taskserv_name)(_ansi reset)..." } - # Run KCL to extract dependency information - let kcl_result = (do { - kcl run $deps_file --format json | from json + # Run Nickel to extract dependency information + let decl_result = (do { + nickel export $deps_file --format json | from json } | complete) - if $kcl_result.exit_code != 0 { + if $decl_result.exit_code != 0 { return { valid: false taskserv: $taskserv_name has_dependencies: true warnings: [] - errors: [$"Failed to parse dependencies.k: ($kcl_result.stderr)"] + errors: [$"Failed to parse dependencies.ncl: ($decl_result.stderr)"] } } - let result = $kcl_result.stdout + let result = $decl_result.stdout # Extract dependency information let deps = ($result | try { get _dependencies) } catch { null } @@ -55,7 +55,7 @@ export def validate-dependencies [ valid: true taskserv: $taskserv_name has_dependencies: false - warnings: ["dependencies.k exists but no _dependencies defined"] + warnings: ["dependencies.ncl exists but no _dependencies defined"] errors: [] } } @@ -200,9 +200,9 @@ export def check-all-dependencies [ ]: nothing -> table { let taskservs_path = (get-taskservs-path) - # Find all taskservs with dependencies.k + # Find all taskservs with dependencies.ncl let all_taskservs = ( - ls ($taskservs_path | path join "**/kcl/dependencies.k") + ls ($taskservs_path | path join "**/nickel/dependencies.ncl") | get name | each {|path| $path | path dirname | path dirname | path basename @@ -266,4 +266,4 @@ export def print-validation-report [ _print $" βœ— ($err)" } } -} \ No newline at end of file +} diff --git a/nulib/taskservs/discover.nu b/nulib/taskservs/discover.nu index 9651d49..2857ff3 100644 --- a/nulib/taskservs/discover.nu +++ b/nulib/taskservs/discover.nu @@ -22,19 +22,19 @@ export def discover-taskservs []: nothing -> list<record> { for item in $items { let item_name = ($item.name | path basename) - let kcl_path = ($item.name | path join "kcl") - let kcl_mod_path = ($kcl_path | path join "kcl.mod") + let schema_path = ($item.name | path join "nickel") + let mod_path = ($schema_path | path join "nickel.mod") - # Check if this is a group directory with kcl/kcl.mod (has applications inside) - if ($kcl_mod_path | path exists) { + # Check if this is a group directory with nickel/nickel.mod (has applications inside) + if ($mod_path | path exists) { # This is a group - list the applications/profiles inside let group_result = (do { ls $item.name } | complete) let group_items = if $group_result.exit_code == 0 { $group_result.stdout } else { [] } - # Get all subdirectories (applications/profiles) except 'kcl' and 'images' + # Get all subdirectories (applications/profiles) except 'nickel' and 'images' for subitem in ($group_items | where type == "dir" | where { |it| let name = ($it.name | path basename) - $name != "kcl" and $name != "images" + $name != "nickel" and $name != "images" }) { let app_name = ($subitem.name | path basename) let metadata = { @@ -42,7 +42,7 @@ export def discover-taskservs []: nothing -> list<record> { type: "taskserv" group: $item_name version: "" - kcl_path: $kcl_path + schema_path: $schema_path main_schema: "" dependencies: [] description: "" @@ -57,24 +57,24 @@ export def discover-taskservs []: nothing -> list<record> { $taskservs | sort-by name } -# Extract metadata from a taskserv's KCL module (updated with group info) -def extract_taskserv_metadata [name: string, kcl_path: string, group: string]: nothing -> record { - let kcl_mod_path = ($kcl_path | path join "kcl.mod") +# Extract metadata from a taskserv's Nickel module (updated with group info) +def extract_taskserv_metadata [name: string, schema_path: string, group: string]: nothing -> record { + let mod_path = ($schema_path | path join "nickel.mod") # Try to parse TOML, skip if corrupted let toml_result = (do { - open $kcl_mod_path | from toml + open $mod_path | from toml } | complete) if $toml_result.exit_code != 0 { - print $"⚠️ Skipping ($name): corrupted kcl.mod file" + print $"⚠️ Skipping ($name): corrupted nickel.mod file" return null } let mod_content = $toml_result.stdout - # Find KCL schema files - let schema_files = (glob ($kcl_path | path join "*.k")) + # Find Nickel schema files + let schema_files = (glob ($schema_path | path join "*.ncl")) let main_schema = ($schema_files | where ($it | str contains $name) | first | default "") # Extract dependencies @@ -92,16 +92,16 @@ def extract_taskserv_metadata [name: string, kcl_path: string, group: string]: n type: "taskserv" group: $group version: $mod_content.package.version - kcl_path: $kcl_path + schema_path: $schema_path main_schema: $main_schema dependencies: $dependencies description: $description available: true - last_updated: (ls $kcl_mod_path | get 0.modified) + last_updated: (ls $mod_path | get 0.modified) } } -# Extract description from KCL schema file +# Extract description from Nickel schema file def extract_schema_description [schema_file: string]: nothing -> string { if not ($schema_file | path exists) { return "" @@ -183,4 +183,4 @@ export def get-taskserv-path [name: string]: nothing -> string { } else { $"($base_path)/($taskserv_info.group)/($name)" } -} \ No newline at end of file +} diff --git a/nulib/taskservs/generate.nu b/nulib/taskservs/generate.nu index 43b6c76..602ae1b 100644 --- a/nulib/taskservs/generate.nu +++ b/nulib/taskservs/generate.nu @@ -7,27 +7,27 @@ use ../lib_provisioning/config/accessor.nu * #use providers/prov_lib/middleware.nu * # Provider middleware now available through lib_provisioning -# > TaskServs generate +# > TaskServs generate export def "main generate" [ task_name?: string # task in settings server?: string # Server hostname in settings - ...args # Args for generate command - --infra (-i): string # Infra directory - --settings (-s): string # Settings path + ...args # Args for generate command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path --iptype: string = "public" # Ip type to connect --outfile (-o): string # Output file - --taskserv_pos (-p): int # Server position in settings - --check (-c) # Only check mode no taskservs will be generated - --wait (-w) # Wait taskservs to be generated - --select: string # Select with task as option + --taskserv_pos (-p): int # Server position in settings + --check (-c) # Only check mode no taskservs will be generated + --wait (-w) # Wait taskservs to be generated + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { @@ -36,46 +36,46 @@ export def "main generate" [ } provisioning_init $helpinfo "taskserv generate" ([($task_name | default "") ($server | default "")] | append $args) if $debug { set-debug-enabled true } - if $metadata { set-metadata-enabled true } + if $metadata { set-metadata-enabled true } let curr_settings = (find_get_settings --infra $infra --settings $settings) let task = ((get-provisioning-args) | split row " "| try { get 0 } catch { null } - let options = if ($args | length) > 0 { - $args - } else { + let options = if ($args | length) > 0 { + $args + } else { let str_task = ((get-provisioning-args) | str replace $"($task) " "" | str replace $"($task_name) " "" | str replace $"($server) " "") ($str_task | split row "-" | try { get 0 } catch { "" | str trim ) } - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $"($task_name) " "" | str trim - #print "GENEREATE" + #print "GENEREATE" # "/wuwei/repo-cnz/src/provisioning/taskservs/oci-reg/generate/defs.toml" - #exit - let run_generate = { + #exit + let run_generate = { let curr_settings = (settings_with_env $curr_settings) set-wk-cnprov $curr_settings.wk_path - let arr_task = if $task_name == null or $task_name == "" or $task_name == "-" { [] } else { $task_name | split row "/" } - let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } - let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } - let match_server = if $server == null or $server == "" { "" } else { $server} + let arr_task = if $task_name == null or $task_name == "" or $task_name == "-" { [] } else { $task_name | split row "/" } + let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } + let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } + let match_server = if $server == null or $server == "" { "" } else { $server} on_taskservs $curr_settings $match_task $match_task_profile $match_server $iptype $check } match $task { - "" if $task_name == "h" => { + "" if $task_name == "h" => { ^$"((get-provisioning-name))" -mod taskserv update help --notitles }, - "" if $task_name == "help" => { + "" if $task_name == "help" => { ^$"((get-provisioning-name))" -mod taskserv update --help _print (provisioning_options "update") }, - "g" | "generate" | "" => { + "g" | "generate" | "" => { let result = desktop_run_notify $"((get-provisioning-name)) taskservs generate" "-> " $run_generate --timeout 11sec }, - _ => { - if $task_name != "" {_print $"πŸ›‘ invalid_option ($task_name)" } + _ => { + if $task_name != "" {_print $"πŸ›‘ invalid_option ($task_name)" } _print $"\nUse (_ansi blue_bold)((get-provisioning-name)) -h(_ansi reset) for help on commands and options" } - } + } # "" | "generate" - #if not $env.PROVISIONING_DEBUG { end_run "" } -} \ No newline at end of file + #if not $env.PROVISIONING_DEBUG { end_run "" } +} diff --git a/nulib/taskservs/handlers.nu b/nulib/taskservs/handlers.nu index 286c00d..3444204 100644 --- a/nulib/taskservs/handlers.nu +++ b/nulib/taskservs/handlers.nu @@ -12,7 +12,7 @@ def install_from_server [ wk_server: string ]: nothing -> bool { _print ( - $"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " + + $"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " + $"($defs.server.hostname) (_ansi default_dimmed)install(_ansi reset) " + $"(_ansi purple_bold)from ($defs.taskserv_install_mode)(_ansi reset)" ) @@ -28,8 +28,8 @@ def install_from_library [ wk_server: string ]: nothing -> bool { _print ( - $"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " + - $"($defs.server.hostname) (_ansi default_dimmed)install(_ansi reset) " + + $"(_ansi yellow_bold)($defs.taskserv.name)(_ansi reset) (_ansi default_dimmed)on(_ansi reset) " + + $"($defs.server.hostname) (_ansi default_dimmed)install(_ansi reset) " + $"(_ansi purple_bold)from library(_ansi reset)" ) let taskservs_path = (get-taskservs-path) @@ -41,28 +41,28 @@ def install_from_library [ export def on_taskservs [ settings: record - match_taskserv: string - match_taskserv_profile: string - match_server: string + match_taskserv: string + match_taskserv_profile: string + match_server: string iptype: string check: bool ]: nothing -> bool { _print $"Running (_ansi yellow_bold)taskservs(_ansi reset) ..." let provisioning_sops = ($env.PROVISIONING_SOPS? | default "") if $provisioning_sops == "" { - # A SOPS load env + # A SOPS load env $env.CURRENT_INFRA_PATH = ($settings.infra_path | path join $settings.infra) - use ../sops_env.nu + use ../sops_env.nu } let ip_type = if $iptype == "" { "public" } else { $iptype } - let str_created_taskservs_dirpath = ( $settings.data.created_taskservs_dirpath | default (["/tmp"] | path join) | - str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME | str replace "NOW" $env.NOW + let str_created_taskservs_dirpath = ( $settings.data.created_taskservs_dirpath | default (["/tmp"] | path join) | + str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME | str replace "NOW" $env.NOW ) let created_taskservs_dirpath = if ($str_created_taskservs_dirpath | str starts-with "/" ) { $str_created_taskservs_dirpath } else { $settings.src_path | path join $str_created_taskservs_dirpath } let root_wk_server = ($created_taskservs_dirpath | path join "on-server") if not ($root_wk_server | path exists ) { ^mkdir "-p" $root_wk_server } - let dflt_clean_created_taskservs = ($settings.data.clean_created_taskservs? | default $created_taskservs_dirpath | - str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME + let dflt_clean_created_taskservs = ($settings.data.clean_created_taskservs? | default $created_taskservs_dirpath | + str replace "./" $"($settings.src_path)/" | str replace "~" $env.HOME ) let run_ops = if (is-debug-enabled) { "bash -x" } else { "" } $settings.data.servers @@ -181,4 +181,4 @@ export def on_taskservs [ } true -} \ No newline at end of file +} diff --git a/nulib/taskservs/load.nu b/nulib/taskservs/load.nu index 00b5a6b..21896f4 100644 --- a/nulib/taskservs/load.nu +++ b/nulib/taskservs/load.nu @@ -70,9 +70,9 @@ def load-single-taskserv [target_path: string, name: string, force: bool, layer: } } - # Copy KCL files and directories + # Copy Nickel files and directories mkdir $target_dir - let source_items = (ls $taskserv_info.kcl_path | get name) + let source_items = (ls $taskserv_info.schema_path | get name) for $item in $source_items { cp -r $item $target_dir } @@ -98,12 +98,12 @@ def load-single-taskserv [target_path: string, name: string, force: bool, layer: } } -# Generate taskservs.k import file +# Generate taskservs.ncl import file def generate-taskservs-imports [target_path: string, taskservs: list<string>, layer: string] { # Generate individual imports for each taskserv let imports = ($taskservs | each { |name| # Check if the taskserv main file exists - let main_file = ($target_path | path join ".taskservs" $name ($name + ".k")) + let main_file = ($target_path | path join ".taskservs" $name ($name + ".ncl")) if ($main_file | path exists) { $"import .taskservs.($name).($name) as ($name)_schema" } else { @@ -132,7 +132,7 @@ taskservs = { taskservs" # Save the imports file - $content | save -f ($target_path | path join "taskservs.k") + $content | save -f ($target_path | path join "taskservs.ncl") # Also create individual alias files for easier direct imports for $name in $taskservs { @@ -144,7 +144,7 @@ import .taskservs.($name) as ($name) # Re-export for convenience ($name)" - $alias_content | save -f ($target_path | path join $"taskserv_($name).k") + $alias_content | save -f ($target_path | path join $"taskserv_($name).ncl") } } @@ -166,7 +166,7 @@ def update-taskservs-manifest [target_path: string, taskservs: list<string>, lay version: $info.version layer: $layer loaded_at: (date now | format date '%Y-%m-%d %H:%M:%S') - source_path: $info.kcl_path + source_path: $info.schema_path } }) @@ -198,7 +198,7 @@ export def unload-taskserv [workspace: string, name: string]: nothing -> record if ($updated_taskservs | is-empty) { rm $manifest_path - rm ($workspace | path join "taskservs.k") + rm ($workspace | path join "taskservs.ncl") } else { let updated_manifest = ($manifest | update loaded_taskservs $updated_taskservs) $updated_manifest | to yaml | save $manifest_path @@ -229,4 +229,4 @@ export def list-loaded-taskservs [workspace: string]: nothing -> list<record> { let manifest = (open $manifest_path) $manifest.loaded_taskservs? | default [] -} \ No newline at end of file +} diff --git a/nulib/taskservs/mod.nu b/nulib/taskservs/mod.nu index e9e052e..b4b6f00 100644 --- a/nulib/taskservs/mod.nu +++ b/nulib/taskservs/mod.nu @@ -9,4 +9,4 @@ export use ops.nu * export use validate.nu * export use test.nu * export use deps_validator.nu * -export use check_mode.nu * \ No newline at end of file +export use check_mode.nu * diff --git a/nulib/taskservs/run.nu b/nulib/taskservs/run.nu index 55b6dc0..f97df23 100644 --- a/nulib/taskservs/run.nu +++ b/nulib/taskservs/run.nu @@ -71,11 +71,11 @@ export def run_taskserv_library [ if not ($taskserv_path | path exists) { return false } let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace "~" $env.HOME) let taskserv_server_name = $defs.server.hostname - rm -rf ...(glob ($taskserv_env_path | path join "*.k")) ($taskserv_env_path |path join "kcl") - mkdir ($taskserv_env_path | path join "kcl") + rm -rf ...(glob ($taskserv_env_path | path join "*.ncl")) ($taskserv_env_path |path join "nickel") + mkdir ($taskserv_env_path | path join "nickel") let err_out = ($taskserv_env_path | path join (mktemp --tmpdir-path $taskserv_env_path --suffix ".err" | path basename)) - let kcl_temp = ($taskserv_env_path | path join "kcl"| path join (mktemp --tmpdir-path $taskserv_env_path --suffix ".k" | path basename)) + let nickel_temp = ($taskserv_env_path | path join "nickel"| path join (mktemp --tmpdir-path $taskserv_env_path --suffix ".ncl" | path basename)) let wk_format = if (get-provisioning-wk-format) == "json" { "json" } else { "yaml" } let wk_data = { # providers: $defs.settings.providers, @@ -88,46 +88,46 @@ export def run_taskserv_library [ } else { $wk_data | to yaml | save --force $wk_vars } - if (get-use-kcl) { + if (get-use-nickel) { cd ($defs.settings.infra_path | path join $defs.settings.infra) - if ($kcl_temp | path exists) { rm -f $kcl_temp } - let res = (^kcl import -m $wk_format $wk_vars -o $kcl_temp | complete) + if ($nickel_temp | path exists) { rm -f $nickel_temp } + let res = (^nickel import -m $wk_format $wk_vars -o $nickel_temp | complete) if $res.exit_code != 0 { - _print $"❗KCL import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found " + _print $"❗Nickel import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found " _print $res.stdout - rm -f $kcl_temp + rm -f $nickel_temp cd $env.PWD return false } # Very important! Remove external block for import and re-format it - # ^sed -i "s/^{//;s/^}//" $kcl_temp - open $kcl_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $kcl_temp - let res = (^kcl fmt $kcl_temp | complete) - let kcl_taskserv_path = if ($taskserv_path | path join "kcl"| path join $"($defs.taskserv.name).k" | path exists) { - ($taskserv_path | path join "kcl"| path join $"($defs.taskserv.name).k") - } else if ($taskserv_path | path dirname | path join "kcl"| path join $"($defs.taskserv.name).k" | path exists) { - ($taskserv_path | path dirname | path join "kcl"| path join $"($defs.taskserv.name).k") - } else if ($taskserv_path | path dirname | path join "default" | path join "kcl"| path join $"($defs.taskserv.name).k" | path exists) { - ($taskserv_path | path dirname | path join "default" | path join "kcl"| path join $"($defs.taskserv.name).k") + # ^sed -i "s/^{//;s/^}//" $nickel_temp + open $nickel_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $nickel_temp + let res = (^nickel fmt $nickel_temp | complete) + let nickel_taskserv_path = if ($taskserv_path | path join "nickel"| path join $"($defs.taskserv.name).ncl" | path exists) { + ($taskserv_path | path join "nickel"| path join $"($defs.taskserv.name).ncl") + } else if ($taskserv_path | path dirname | path join "nickel"| path join $"($defs.taskserv.name).ncl" | path exists) { + ($taskserv_path | path dirname | path join "nickel"| path join $"($defs.taskserv.name).ncl") + } else if ($taskserv_path | path dirname | path join "default" | path join "nickel"| path join $"($defs.taskserv.name).ncl" | path exists) { + ($taskserv_path | path dirname | path join "default" | path join "nickel"| path join $"($defs.taskserv.name).ncl") } else { "" } - if $kcl_taskserv_path != "" and ($kcl_taskserv_path | path exists) { + if $nickel_taskserv_path != "" and ($nickel_taskserv_path | path exists) { if (is-debug-enabled) { - _print $"adding task name: ($defs.taskserv.name) -> ($kcl_taskserv_path)" + _print $"adding task name: ($defs.taskserv.name) -> ($nickel_taskserv_path)" } - cat $kcl_taskserv_path | save --append $kcl_temp + cat $nickel_taskserv_path | save --append $nickel_temp } - let kcl_taskserv_profile_path = if ($taskserv_path | path join "kcl"| path join $"($defs.taskserv.profile).k" | path exists) { - ($taskserv_path | path join "kcl"| path join $"($defs.taskserv.profile).k") - } else if ($taskserv_path | path dirname | path join "kcl"| path join $"($defs.taskserv.profile).k" | path exists) { - ($taskserv_path | path dirname | path join "kcl"| path join $"($defs.taskserv.profile).k") - } else if ($taskserv_path | path dirname | path join "default" | path join "kcl"| path join $"($defs.taskserv.profile).k" | path exists) { - ($taskserv_path | path dirname | path join "default" | path join "kcl"| path join $"($defs.taskserv.profile).k") + let nickel_taskserv_profile_path = if ($taskserv_path | path join "nickel"| path join $"($defs.taskserv.profile).ncl" | path exists) { + ($taskserv_path | path join "nickel"| path join $"($defs.taskserv.profile).ncl") + } else if ($taskserv_path | path dirname | path join "nickel"| path join $"($defs.taskserv.profile).ncl" | path exists) { + ($taskserv_path | path dirname | path join "nickel"| path join $"($defs.taskserv.profile).ncl") + } else if ($taskserv_path | path dirname | path join "default" | path join "nickel"| path join $"($defs.taskserv.profile).ncl" | path exists) { + ($taskserv_path | path dirname | path join "default" | path join "nickel"| path join $"($defs.taskserv.profile).ncl") } else { "" } - if $kcl_taskserv_profile_path != "" and ($kcl_taskserv_profile_path | path exists) { + if $nickel_taskserv_profile_path != "" and ($nickel_taskserv_profile_path | path exists) { if (is-debug-enabled) { - _print $"adding task profile: ($defs.taskserv.profile) -> ($kcl_taskserv_profile_path)" + _print $"adding task profile: ($defs.taskserv.profile) -> ($nickel_taskserv_profile_path)" } - cat $kcl_taskserv_profile_path | save --append $kcl_temp + cat $nickel_taskserv_profile_path | save --append $nickel_temp } let keys_path_config = (get-keys-path) if $keys_path_config != "" { @@ -141,36 +141,36 @@ export def run_taskserv_library [ } return false } - (on_sops d $keys_path) | save --append $kcl_temp - let kcl_defined_taskserv_path = if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).k") - } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).k") - } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).k") - } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).k") - } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile | path join $"($defs.taskserv.name).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile | path join $"($defs.taskserv.name).k") - } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).k" | path exists ) { - ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).k") + (on_sops d $keys_path) | save --append $nickel_temp + let nickel_defined_taskserv_path = if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl") + } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.profile).ncl") + } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $"($defs.taskserv.profile).ncl") + } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $"($defs.taskserv.name).ncl") + } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile | path join $"($defs.taskserv.name).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs" | path join $defs.server.hostname | path join $defs.taskserv.profile | path join $"($defs.taskserv.name).ncl") + } else if ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).ncl" | path exists ) { + ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).ncl") } else { "" } - if $kcl_defined_taskserv_path != "" and ($kcl_defined_taskserv_path | path exists) { + if $nickel_defined_taskserv_path != "" and ($nickel_defined_taskserv_path | path exists) { if (is-debug-enabled) { - _print $"adding defs taskserv: ($kcl_defined_taskserv_path)" + _print $"adding defs taskserv: ($nickel_defined_taskserv_path)" } - cat $kcl_defined_taskserv_path | save --append $kcl_temp + cat $nickel_defined_taskserv_path | save --append $nickel_temp } - let res = (^kcl $kcl_temp -o $wk_vars | complete) + let res = (^nickel $nickel_temp -o $wk_vars | complete) if $res.exit_code != 0 { - _print $"❗KCL errors (_ansi red_bold)($kcl_temp)(_ansi reset) found " + _print $"❗Nickel errors (_ansi red_bold)($nickel_temp)(_ansi reset) found " _print $res.stdout _print $res.stderr rm -f $wk_vars cd $env.PWD return false } - rm -f $kcl_temp $err_out + rm -f $nickel_temp $err_out } else if ( $defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).yaml" | path exists) { cat ($defs.settings.src_path | path join "extensions" | path join "taskservs"| path join $"($defs.taskserv.name).yaml") | tee { save -a $wk_vars } | ignore } @@ -196,7 +196,7 @@ export def run_taskserv_library [ } } } - rm -f ($taskserv_env_path | path join "kcl") ...(glob $"($taskserv_env_path)/*.k") + rm -f ($taskserv_env_path | path join "nickel") ...(glob $"($taskserv_env_path)/*.ncl") on_template_path $taskserv_env_path $wk_vars true true if ($taskserv_env_path | path join $"env-($defs.taskserv.name)" | path exists) { ^sed -i 's,\t,,g;s,^ ,,g;/^$/d' ($taskserv_env_path | path join $"env-($defs.taskserv.name)") @@ -208,7 +208,7 @@ export def run_taskserv_library [ } } if not (is-debug-enabled) { - rm -f ...(glob $"($taskserv_env_path)/*.j2") $err_out $kcl_temp + rm -f ...(glob $"($taskserv_env_path)/*.j2") $err_out $nickel_temp } true } @@ -231,7 +231,7 @@ export def run_taskserv [ if not ( $taskserv_env_path | path exists) { ^mkdir -p $taskserv_env_path } (^cp -pr ...(glob ($taskserv_path | path join "*")) $taskserv_env_path) - rm -rf ...(glob ($taskserv_env_path | path join "*.k")) ($taskserv_env_path | path join "kcl") + rm -rf ...(glob ($taskserv_env_path | path join "*.ncl")) ($taskserv_env_path | path join "nickel") let wk_vars = ($created_taskservs_dirpath | path join $"($defs.server.hostname).yaml") let require_j2 = (^ls ...(glob ($taskserv_env_path | path join "*.j2")) err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" })) @@ -259,7 +259,7 @@ export def run_taskserv [ if not (is-debug-enabled) { rm -f $wk_vars if $err_out != "" { rm -f $err_out } - rm -rf ...(glob $"($taskserv_env_path)/*.k") ($taskserv_env_path | path join join "kcl") + rm -rf ...(glob $"($taskserv_env_path)/*.ncl") ($taskserv_env_path | path join join "nickel") } return true } @@ -326,7 +326,7 @@ export def run_taskserv [ if not (is-debug-enabled) { rm -f $wk_vars if $err_out != "" { rm -f $err_out } - rm -rf ...(glob $"($taskserv_env_path)/*.k") ($taskserv_env_path | path join join "kcl") + rm -rf ...(glob $"($taskserv_env_path)/*.ncl") ($taskserv_env_path | path join join "nickel") } true -} \ No newline at end of file +} diff --git a/nulib/taskservs/test.nu b/nulib/taskservs/test.nu index ae4b755..93dad3b 100644 --- a/nulib/taskservs/test.nu +++ b/nulib/taskservs/test.nu @@ -294,20 +294,20 @@ def test-configuration-validity [ sandbox: record verbose: bool ]: nothing -> record { - # Run KCL validation - let kcl_result = (validate-kcl-schemas $taskserv_name --verbose=false) + # Run Nickel validation + let decl_result = (validate-nickel-schemas $taskserv_name --verbose=false) - if $kcl_result.valid { + if $decl_result.valid { { test: "Configuration validity" status: "passed" - message: $"($kcl_result.files_checked) configuration files validated" + message: $"($decl_result.files_checked) configuration files validated" } } else { { test: "Configuration validity" status: "failed" - message: $"KCL validation failed: (($kcl_result.errors | str join ', '))" + message: $"Nickel validation failed: (($decl_result.errors | str join ', '))" } } } diff --git a/nulib/taskservs/update.nu b/nulib/taskservs/update.nu index 707c219..7fc0f67 100644 --- a/nulib/taskservs/update.nu +++ b/nulib/taskservs/update.nu @@ -5,27 +5,27 @@ use ../lib_provisioning/utils/ssh.nu * use ../lib_provisioning/config/accessor.nu * # Provider middleware now available through lib_provisioning -# > TaskServs update +# > TaskServs update export def "main update" [ name?: string # task in settings server?: string # Server hostname in settings - ...args # Args for update command - --infra (-i): string # Infra directory - --settings (-s): string # Settings path + ...args # Args for update command + --infra (-i): string # Infra directory + --settings (-s): string # Settings path --iptype: string = "public" # Ip type to connect --outfile (-o): string # Output file - --taskserv_pos (-p): int # Server position in settings - --check (-c) # Only check mode no taskservs will be created + --taskserv_pos (-p): int # Server position in settings + --check (-c) # Only check mode no taskservs will be created --wait (-w) # Wait taskservs to be updated - --select: string # Select with task as option + --select: string # Select with task as option --debug (-x) # Use Debug mode - --xm # Debug with PROVISIONING_METADATA - --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK - --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE - --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug + --xm # Debug with PROVISIONING_METADATA + --xc # Debuc for task and services locally PROVISIONING_DEBUG_CHECK + --xr # Debug for remote taskservs PROVISIONING_DEBUG_REMOTE + --xld # Log level with DEBUG PROVISIONING_LOG_LEVEL=debug --metadata # Error with metadata (-xm) --notitles # not tittles - --helpinfo (-h) # For more details use options "help" (no dashes) + --helpinfo (-h) # For more details use options "help" (no dashes) --out: string # Print Output format: json, yaml, text (default) ]: nothing -> nothing { if ($out | is-not-empty) { @@ -34,46 +34,46 @@ export def "main update" [ } provisioning_init $helpinfo "taskserv update" $args if $debug { set-debug-enabled true } - if $metadata { set-metadata-enabled true } + if $metadata { set-metadata-enabled true } let curr_settings = (find_get_settings --infra $infra --settings $settings) - let task = if ($args | length) > 0 { - ($args| get 0) - } else { - let str_task = ((get-provisioning-args) | str replace "update " " " ) - let str_task = if $name != null { - ($str_task | str replace $name "") + let task = if ($args | length) > 0 { + ($args| get 0) + } else { + let str_task = ((get-provisioning-args) | str replace "update " " " ) + let str_task = if $name != null { + ($str_task | str replace $name "") } else { $str_task - } + } ($str_task | str trim | split row " " | first | default "" | split row "-" | first | default "" | str trim) - } - let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } + } + let other = if ($args | length) > 0 { ($args| skip 1) } else { "" } let ops = $"((get-provisioning-args)) " | str replace $"($task) " "" | str trim - let run_update = { + let run_update = { let curr_settings = (settings_with_env (find_get_settings --infra $infra --settings $settings)) set-wk-cnprov $curr_settings.wk_path - let arr_task = if $name == null or $name == "" or $name == $task { [] } else { $name | split row "/" } - let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } - let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } - let match_server = if $server == null or $server == "" { "" } else { $server} + let arr_task = if $name == null or $name == "" or $name == $task { [] } else { $name | split row "/" } + let match_task = if ($arr_task | length ) == 0 { "" } else { ($arr_task | try { get 0 } catch { null } } + let match_task_profile = if ($arr_task | length ) < 2 { "" } else { ($arr_task | try { get 1) } catch { null } } + let match_server = if $server == null or $server == "" { "" } else { $server} on_taskservs $curr_settings $match_task $match_task_profile $match_server $iptype $check } match $task { - "" if $name == "h" => { + "" if $name == "h" => { ^$"((get-provisioning-name))" -mod taskserv update help --notitles }, - "" if $name == "help" => { + "" if $name == "help" => { ^$"((get-provisioning-name))" -mod taskserv update --help print (provisioning_options "update") }, - "" | "u" | "update" => { + "" | "u" | "update" => { let result = desktop_run_notify $"((get-provisioning-name)) taskservs update" "-> " $run_update --timeout 11sec #do $run_update }, - _ => { - if $task != "" { print $"πŸ›‘ invalid_option ($task)" } + _ => { + if $task != "" { print $"πŸ›‘ invalid_option ($task)" } _print $"\nUse (_ansi blue_bold)((get-provisioning-name)) -h(_ansi reset) for help on commands and options" } - } - if not (is-debug-enabled) { end_run "" } -} \ No newline at end of file + } + if not (is-debug-enabled) { end_run "" } +} diff --git a/nulib/taskservs/utils.nu b/nulib/taskservs/utils.nu index dfdd59a..866f868 100644 --- a/nulib/taskservs/utils.nu +++ b/nulib/taskservs/utils.nu @@ -8,41 +8,41 @@ export def taskserv_get_file [ settings: record taskserv: record server: record - live_ip: string + live_ip: string req_sudo: bool local_mode: bool ]: nothing -> bool { let target_path = ($taskserv.target_path | default "") - if $target_path == "" { + if $target_path == "" { _print $"πŸ›‘ No (_ansi red_bold)target_path(_ansi reset) found in ($server.hostname) taskserv ($taskserv.name)" return false } let source_path = ($taskserv.soruce_path | default "") - if $source_path == "" { + if $source_path == "" { _print $"πŸ›‘ No (_ansi red_bold)source_path(_ansi reset) found in ($server.hostname) taskserv ($taskserv.name)" return false } - if $local_mode { - let res = (^cp $source_path $target_path | combine) - if $res.exit_code != 0 { + if $local_mode { + let res = (^cp $source_path $target_path | combine) + if $res.exit_code != 0 { _print $"πŸ›‘ Error get_file [ local-mode ] (_ansi red_bold)($source_path) to ($target_path)(_ansi reset) in ($server.hostname) taskserv ($taskserv.name)" _print $res.stdout return false - } + } return true } let ip = if $live_ip != "" { - $live_ip - } else { + $live_ip + } else { #use ../../../providers/prov_lib/middleware.nu mw_get_ip (mw_get_ip $settings $server $server.liveness_ip false) } let ssh_key_path = ($server.ssh_key_path | default "") - if $ssh_key_path == "" { + if $ssh_key_path == "" { _print $"πŸ›‘ No (_ansi red_bold)ssh_key_path(_ansi reset) found in ($server.hostname) taskserv ($taskserv.name)" return false } - if not ($ssh_key_path | path exists) { + if not ($ssh_key_path | path exists) { _print $"πŸ›‘ Error (_ansi red_bold)($ssh_key_path)(_ansi reset) not found for ($server.hostname) taskserv ($taskserv.name)" return false } @@ -54,10 +54,10 @@ export def taskserv_get_file [ if not (scp_from $settings $server $wk_path $target_path $ip ) { return false } - let rm_cmd = if $req_sudo { - $"sudo rm -f ($wk_path)" - } else { - $"rm -f ($wk_path)" + let rm_cmd = if $req_sudo { + $"sudo rm -f ($wk_path)" + } else { + $"rm -f ($wk_path)" } return ( ssh_cmd $settings $server false $rm_cmd $ip ) } @@ -66,7 +66,7 @@ export def find_taskserv [ settings: record, server: record, taskserv_name: string, - out: string + out: string ]: nothing -> record { let taskservs_list = ($server | get taskservs? | default []) let taskserv = ($taskservs_list | where {|t| ($t | get name? | default "") == $taskserv_name}) @@ -78,11 +78,11 @@ export def find_taskserv [ let hostname = ($server | get hostname? | default "") let run_taskservs_path = (get-run-taskservs-path) mut taskserv_host_path = ($src_path | path join $run_taskservs_path | - path join $hostname | path join $"($taskserv_name).k") + path join $hostname | path join $"($taskserv_name).ncl") let def_taskserv = if ($taskserv_host_path | path exists) { (open -r $taskserv_host_path) } else { - $taskserv_host_path = ($src_path | path join $run_taskservs_path | path join $"($taskserv_name).k") + $taskserv_host_path = ($src_path | path join $run_taskservs_path | path join $"($taskserv_name).ncl") if ($taskserv_host_path | path exists) { (open -r $taskserv_host_path) } else { @@ -92,10 +92,10 @@ export def find_taskserv [ } } let taskservs_path = (get-taskservs-path) - mut main_taskserv_path = ($taskservs_path | path join $taskserv_name | path join "kcl" | path join $"($taskserv_name).k") + mut main_taskserv_path = ($taskservs_path | path join $taskserv_name | path join "nickel" | path join $"($taskserv_name).ncl") if not ($main_taskserv_path | path exists) { $main_taskserv_path = ($taskservs_path | path join $taskserv_name | path join ($taskserv | - get -o profile | default "") | path join "kcl" | path join $"($taskserv_name).k") + get -o profile | default "") | path join "nickel" | path join $"($taskserv_name).ncl") } let def_main = if ($main_taskserv_path | path exists) { (open -r $main_taskserv_path) @@ -110,9 +110,9 @@ export def list_taskservs [ settings: record ]: nothing -> list { let list_taskservs = (taskservs_list) - if ($list_taskservs | length) == 0 { - _print $"πŸ›‘ no items found for (_ansi cyan)taskservs list(_ansi reset)" + if ($list_taskservs | length) == 0 { + _print $"πŸ›‘ no items found for (_ansi cyan)taskservs list(_ansi reset)" return - } + } $list_taskservs -} \ No newline at end of file +} diff --git a/nulib/taskservs/validate.nu b/nulib/taskservs/validate.nu index cfae210..a367451 100644 --- a/nulib/taskservs/validate.nu +++ b/nulib/taskservs/validate.nu @@ -8,69 +8,69 @@ use ../lib_provisioning/config/accessor.nu * # Validation levels const VALIDATION_LEVELS = { - static: "Static validation (KCL, templates, scripts)" + static: "Static validation (Nickel, templates, scripts)" dependencies: "Dependency validation" prerequisites: "Server prerequisites validation" health: "Health check validation" all: "Complete validation (all levels)" } -# Validate KCL schemas for taskserv -def validate-kcl-schemas [ +# Validate Nickel schemas for taskserv +def validate-nickel-schemas [ taskserv_name: string --verbose (-v) ]: nothing -> record { let taskservs_path = (get-taskservs-path) - let kcl_path = ($taskservs_path | path join $taskserv_name "kcl") + let schema_path = ($taskservs_path | path join $taskserv_name "nickel") - if not ($kcl_path | path exists) { + if not ($schema_path | path exists) { return { valid: false - level: "kcl" - errors: [$"KCL directory not found: ($kcl_path)"] + level: "nickel" + errors: [$"Nickel directory not found: ($schema_path)"] warnings: [] } } - # Find all .k files - let kcl_result = (do { - ls ($kcl_path | path join "*.k") | get name + # Find all .ncl files + let decl_result = (do { + ls ($schema_path | path join "*.ncl") | get name } | complete) - if $kcl_result.exit_code != 0 { + if $decl_result.exit_code != 0 { return { valid: false - level: "kcl" - errors: [$"No KCL files found in: ($kcl_path)"] + level: "nickel" + errors: [$"No Nickel files found in: ($schema_path)"] warnings: [] } } - let kcl_files = $kcl_result.stdout + let nickel_files = $decl_result.stdout if $verbose { - _print $"Validating KCL schemas for (_ansi yellow_bold)($taskserv_name)(_ansi reset)..." + _print $"Validating Nickel schemas for (_ansi yellow_bold)($taskserv_name)(_ansi reset)..." } mut errors = [] mut warnings = [] - for file in $kcl_files { + for file in $decl_files { if $verbose { _print $" Checking ($file | path basename)..." } - let kcl_check = (do { - kcl run $file --format json | from json + let decl_check = (do { + nickel export $file --format json | from json } | complete) - if $kcl_check.exit_code == 0 { + if $nickel_check.exit_code == 0 { if $verbose { _print $" βœ“ Valid" } } else { - let error_msg = $kcl_check.stderr - $errors = ($errors | append $"KCL error in ($file | path basename): ($error_msg)") + let error_msg = $nickel_check.stderr + $errors = ($errors | append $"Nickel error in ($file | path basename): ($error_msg)") if $verbose { _print $" βœ— Error: ($error_msg)" } @@ -79,8 +79,8 @@ def validate-kcl-schemas [ return { valid: (($errors | length) == 0) - level: "kcl" - files_checked: ($kcl_files | length) + level: "nickel" + files_checked: ($decl_files | length) errors: $errors warnings: $warnings } @@ -379,10 +379,10 @@ export def "main validate" [ mut all_results = [] - # Static validation (KCL, templates, scripts) + # Static validation (Nickel, templates, scripts) if $level in ["static", "all"] { - let kcl_result = (validate-kcl-schemas $taskserv_name --verbose=$verbose) - $all_results = ($all_results | append $kcl_result) + let decl_result = (validate-nickel-schemas $taskserv_name --verbose=$verbose) + $all_results = ($all_results | append $decl_result) let template_result = (validate-templates $taskserv_name --verbose=$verbose) $all_results = ($all_results | append $template_result) @@ -477,4 +477,4 @@ export def "main levels" []: nothing -> nothing { _print $"(_ansi yellow_bold)($level.name)(_ansi reset)" _print $" ($level.description)\n" } -} \ No newline at end of file +} diff --git a/nulib/test_environments_summary.md b/nulib/test-environments-summary.md similarity index 95% rename from nulib/test_environments_summary.md rename to nulib/test-environments-summary.md index 57dc9f2..2999b96 100644 --- a/nulib/test_environments_summary.md +++ b/nulib/test-environments-summary.md @@ -8,12 +8,15 @@ ## 🎯 What Was Built A complete **containerized test environment service** integrated into the orchestrator, enabling automated testing of: + - Single taskservs - Complete servers with multiple taskservs - Multi-node cluster topologies (Kubernetes, etcd, etc.) ### Key Innovation + **No manual Docker management** - The orchestrator automatically handles: + - Container lifecycle - Network isolation - Resource limits @@ -28,6 +31,7 @@ A complete **containerized test environment service** integrated into the orches ### Rust Components (Orchestrator) #### 1. **test_environment.rs** - Core Types + - Test environment types: Single/Server/Cluster - Resource limits configuration - Network configuration @@ -35,6 +39,7 @@ A complete **containerized test environment service** integrated into the orches - Test results tracking #### 2. **container_manager.rs** - Docker Integration + - Docker API client (bollard) - Container lifecycle management - Network creation/isolation @@ -43,6 +48,7 @@ A complete **containerized test environment service** integrated into the orches - Log collection #### 3. **test_orchestrator.rs** - Orchestration + - Environment provisioning logic - Single taskserv setup - Server simulation @@ -51,18 +57,20 @@ A complete **containerized test environment service** integrated into the orches - Cleanup automation #### 4. **API Endpoints** (main.rs) -``` + +```plaintext POST /test/environments/create GET /test/environments GET /test/environments/{id} POST /test/environments/{id}/run DELETE /test/environments/{id} GET /test/environments/{id}/logs -``` +```plaintext ### Nushell Integration #### 1. **test_environments.nu** - Core Commands + - `test env create` - Create from config - `test env single` - Single taskserv test - `test env server` - Server simulation @@ -74,11 +82,13 @@ GET /test/environments/{id}/logs - `test quick` - One-command test #### 2. **test/mod.nu** - CLI Dispatcher + - Command routing - Help system - Integration with main CLI #### 3. **CLI Integration** + - Added to main dispatcher - Registry shortcuts: `test`, `tst` - Full help documentation @@ -86,7 +96,9 @@ GET /test/environments/{id}/logs ### Configuration & Templates #### 1. **test-topologies.toml** - Predefined Topologies + Templates included: + - `kubernetes_3node` - K8s HA cluster (1 CP + 2 workers) - `kubernetes_single` - All-in-one K8s - `etcd_cluster` - 3-member etcd cluster @@ -94,6 +106,7 @@ Templates included: - `postgres_redis` - Database stack #### 2. **Cargo.toml** - Dependencies + - Added `bollard = "0.17"` for Docker API --- @@ -101,31 +114,37 @@ Templates included: ## πŸš€ Usage Examples ### 1. Quick Test (Fastest) + ```bash provisioning test quick kubernetes -``` +```plaintext ### 2. Single Taskserv + ```bash provisioning test env single postgres --auto-start --auto-cleanup -``` +```plaintext ### 3. Server Simulation + ```bash provisioning test env server web-01 [containerd kubernetes cilium] --auto-start -``` +```plaintext ### 4. Cluster from Template + ```bash provisioning test topology load kubernetes_3node | test env cluster kubernetes --auto-start -``` +```plaintext ### 5. Custom Resources + ```bash provisioning test env single redis --cpu 4000 --memory 8192 -``` +```plaintext ### 6. List & Manage + ```bash # List environments provisioning test env list @@ -138,13 +157,13 @@ provisioning test env logs <env-id> # Cleanup provisioning test env cleanup <env-id> -``` +```plaintext --- ## πŸ”§ Architecture -``` +```plaintext User Command ↓ Nushell CLI (test_environments.nu) @@ -162,13 +181,14 @@ Isolated Containers with: β€’ Resource limits β€’ Volume mounts β€’ Multi-node support -``` +```plaintext --- ## βœ… Features Delivered ### Core Capabilities + - βœ… Single taskserv testing - βœ… Server simulation (multiple taskservs) - βœ… Multi-node cluster topologies @@ -180,6 +200,7 @@ Isolated Containers with: - βœ… REST API ### Advanced Features + - βœ… Topology templates - βœ… Template loading system - βœ… Custom configurations @@ -189,6 +210,7 @@ Isolated Containers with: - βœ… Error handling ### Developer Experience + - βœ… Simple CLI commands - βœ… One-command quick tests - βœ… Comprehensive help system @@ -201,6 +223,7 @@ Isolated Containers with: ## πŸ“Š Comparison: Before vs After ### Before (Old test.nu) + - ❌ Manual Docker management - ❌ Single container only - ❌ No multi-node support @@ -209,6 +232,7 @@ Isolated Containers with: - ❌ Limited to single taskserv ### After (New Test Environment Service) + - βœ… Automated container orchestration - βœ… Single + Server + Cluster support - βœ… Multi-node topologies @@ -221,35 +245,40 @@ Isolated Containers with: ## πŸ“ Files Created/Modified ### New Files (Rust) -``` + +```plaintext provisioning/platform/orchestrator/src/ β”œβ”€β”€ test_environment.rs (280 lines) β”œβ”€β”€ container_manager.rs (350 lines) └── test_orchestrator.rs (320 lines) -``` +```plaintext ### New Files (Nushell) -``` + +```plaintext provisioning/core/nulib/ β”œβ”€β”€ test_environments.nu (250 lines) └── test/mod.nu (80 lines) -``` +```plaintext ### New Files (Config) -``` + +```plaintext provisioning/config/ └── test-topologies.toml (150 lines) -``` +```plaintext ### New Files (Docs) -``` + +```plaintext docs/user/ β”œβ”€β”€ test-environment-guide.md (500 lines) └── test_environments_summary.md (this file) -``` +```plaintext ### Modified Files -``` + +```plaintext provisioning/platform/orchestrator/ β”œβ”€β”€ Cargo.toml (added bollard) β”œβ”€β”€ src/lib.rs (added modules) @@ -257,28 +286,32 @@ provisioning/platform/orchestrator/ provisioning/core/nulib/main_provisioning/ └── dispatcher.nu (added test handler) -``` +```plaintext --- ## πŸ” Testing Scenarios Supported ### Development + - Test new taskservs before deployment - Validate configurations - Debug issues in isolation ### Integration + - Test taskserv combinations - Validate dependencies - Check compatibility ### Production-Like + - Simulate HA clusters - Test failover scenarios - Validate multi-node setups ### CI/CD + ```yaml # Example GitLab CI test-infrastructure: @@ -286,7 +319,7 @@ test-infrastructure: - provisioning test quick kubernetes - provisioning test quick postgres - provisioning test quick redis -``` +```plaintext --- @@ -312,11 +345,13 @@ test-infrastructure: ## 🚦 Prerequisites 1. **Docker running:** + ```bash docker ps ``` -2. **Orchestrator running:** +1. **Orchestrator running:** + ```bash cd provisioning/platform/orchestrator ./scripts/start-orchestrator.nu --background @@ -347,6 +382,7 @@ test-infrastructure: ## πŸ”„ Next Steps (Optional Enhancements) Future improvements could include: + - Add more topology templates - Advanced health checks - Performance benchmarking diff --git a/nulib/test/PLUGIN_TEST_README.md b/nulib/test/README.md similarity index 98% rename from nulib/test/PLUGIN_TEST_README.md rename to nulib/test/README.md index df95776..c230781 100644 --- a/nulib/test/PLUGIN_TEST_README.md +++ b/nulib/test/README.md @@ -5,6 +5,7 @@ Comprehensive test suite for the Provisioning platform's plugin system, covering ## Overview This test suite validates: + - **Plugin Availability**: Detection of installed Nushell plugins - **Fallback Behavior**: Graceful degradation to HTTP/SOPS when plugins unavailable - **Complete Workflows**: End-to-end authentication, encryption, and orchestration @@ -46,7 +47,7 @@ nu ../lib_provisioning/plugins/auth_test.nu nu ../lib_provisioning/plugins/kms_test.nu nu ../lib_provisioning/plugins/orchestrator_test.nu nu test_plugin_integration.nu -``` +```plaintext ### Test Options @@ -59,7 +60,7 @@ nu run_plugin_tests.nu --verbose # Skip integration tests (faster) nu run_plugin_tests.nu --skip-integration -``` +```plaintext ### CI/CD Integration @@ -77,7 +78,7 @@ test:plugins: when: always paths: - plugin-test-report.json -``` +```plaintext ## Test Coverage @@ -118,7 +119,7 @@ test:plugins: βœ… Workflow status query βœ… Batch operations βœ… Statistics retrieval -βœ… KCL validation +βœ… Nickel validation βœ… Configuration integration βœ… Error handling βœ… Performance benchmarking @@ -138,6 +139,7 @@ test:plugins: ### Graceful Degradation **All tests pass regardless of plugin availability:** + - βœ… Plugins installed β†’ Use plugins, test performance - βœ… Plugins missing β†’ Use HTTP/SOPS fallback, warn user - βœ… Services unavailable β†’ Skip service-dependent tests, report status @@ -145,6 +147,7 @@ test:plugins: ### No Hard Dependencies Tests never fail due to: + - Missing plugins (fallback tested) - Services not running (gracefully reported) - Network issues (error handling tested) @@ -152,6 +155,7 @@ Tests never fail due to: ### Performance Awareness Tests measure and report performance: + - **Plugin mode**: <50ms (excellent) - **HTTP fallback**: <200ms (good) - **SOPS fallback**: <500ms (acceptable) @@ -160,7 +164,7 @@ Tests measure and report performance: ### Successful Run (All Plugins Available) -``` +```plaintext ================================================================== πŸš€ Running Complete Plugin Integration Test Suite ================================================================== @@ -212,8 +216,8 @@ Tests measure and report performance: βœ… Statistics retrieved Step 7: List batch operations βœ… Batch operations listed - Step 8: Validate KCL content - βœ… KCL validation passed + Step 8: Validate Nickel content + βœ… Nickel validation passed βœ… Orchestrator workflow tests completed πŸ§ͺ Running performance benchmarks... @@ -258,11 +262,11 @@ Expected Performance: ================================================================== βœ… All plugin integration tests completed successfully! ================================================================== -``` +```plaintext ### Fallback Mode (No Plugins) -``` +```plaintext ================================================================== πŸš€ Running Complete Plugin Integration Test Suite ================================================================== @@ -325,7 +329,7 @@ Expected Performance: ================================================================== βœ… All plugin integration tests completed successfully! ================================================================== -``` +```plaintext ## Test Report Format @@ -363,7 +367,7 @@ Expected Performance: "arch": "aarch64" } } -``` +```plaintext ## Troubleshooting @@ -371,10 +375,11 @@ Expected Performance: **Problem**: `nu: command not found` **Solution**: Install Nushell 0.107.1+ + ```bash brew install nushell # macOS cargo install nu # Any platform -``` +```plaintext ### Plugin Tests Show Warnings @@ -385,10 +390,11 @@ cargo install nu # Any platform **Problem**: "Orchestrator not available" warnings **Solution**: Start orchestrator service: + ```bash cd provisioning/platform/orchestrator cargo run --release -``` +```plaintext ### KMS Backend Errors @@ -425,7 +431,7 @@ export def test_new_feature [] { print " ⚠️ Feature not available" } } -``` +```plaintext ## Performance Baselines @@ -452,15 +458,18 @@ export def test_new_feature [] { See: `.github/workflows/plugin-tests.yml` Tests run on: + - Push to main/develop - Pull requests - Manual trigger Platforms: + - Ubuntu latest - macOS latest Artifacts: + - Test reports (JSON) - Benchmark results - Logs (on failure) @@ -468,9 +477,10 @@ Artifacts: ### Badge Status Add to README: + ```markdown [![Plugin Tests](https://github.com/org/repo/workflows/Plugin%20Integration%20Tests/badge.svg)](https://github.com/org/repo/actions) -``` +```plaintext ## Maintenance @@ -484,6 +494,7 @@ Add to README: ### Test Metrics Track over time: + - Total test count - Average execution time - Plugin availability rate @@ -507,6 +518,7 @@ Same as Provisioning Platform (see root LICENSE) ## Support For issues or questions: + - GitHub Issues: [project-provisioning/issues](https://github.com/org/project-provisioning/issues) - Documentation: [docs/](../../docs/) - Plugin Docs: [docs/plugins/](../../docs/plugins/) diff --git a/nulib/test/test_plugin_integration.nu b/nulib/test/test_plugin_integration.nu index 59b4cbf..0d050a0 100644 --- a/nulib/test/test_plugin_integration.nu +++ b/nulib/test/test_plugin_integration.nu @@ -195,9 +195,9 @@ export def test_orch_workflow [] { print " βœ… Batch operations listed" } - # Test 8: Validate KCL content - print " Step 8: Validate KCL content" - let kcl_test = ''' + # Test 8: Validate Nickel content + print " Step 8: Validate Nickel content" + let nickel_test = ''' schema TestConfig: name: str enabled: bool = true @@ -207,13 +207,13 @@ config: TestConfig = { } ''' let validate = (do { - plugin-orch-validate-kcl $kcl_test + plugin-orch-validate-nickel $nickel_test } | complete) if $validate.exit_code == 0 { - print " βœ… KCL validation passed" + print " βœ… Nickel validation passed" } else { - print " ⚠️ KCL validation failed" + print " ⚠️ Nickel validation failed" } } else { print " ⚠️ Orchestrator not available" diff --git a/nulib/tests/mod.nu b/nulib/tests/mod.nu index 655a261..8ca3f6f 100644 --- a/nulib/tests/mod.nu +++ b/nulib/tests/mod.nu @@ -3,4 +3,3 @@ use std assert export def test_addition [] { assert equal (1 + 2) 3 } - diff --git a/nulib/tests/test_gitea.nu b/nulib/tests/test_gitea.nu index 6e1a601..c45b60b 100644 --- a/nulib/tests/test_gitea.nu +++ b/nulib/tests/test_gitea.nu @@ -241,7 +241,7 @@ export def test-extension-publishing-mock [] { # Create temporary extension let temp_ext = $"/tmp/test-extension-(random chars -l 8)" mkdir $temp_ext - mkdir $"($temp_ext)/kcl" + mkdir $"($temp_ext)/nickel" # Create minimal extension structure { @@ -249,9 +249,9 @@ export def test-extension-publishing-mock [] { name = "test-extension" version = "1.0.0" } - } | save -f $"($temp_ext)/kcl/kcl.mod" + } | save -f $"($temp_ext)/nickel/nickel.mod" - "schema TestExtension:\n name: str" | save -f $"($temp_ext)/kcl/test.k" + "schema TestExtension:\n name: str" | save -f $"($temp_ext)/nickel/test.ncl" # Validate extension let validation = validate-extension $temp_ext diff --git a/nulib/tests/verify_services.nu b/nulib/tests/verify_services.nu index 5011a02..67daab2 100644 --- a/nulib/tests/verify_services.nu +++ b/nulib/tests/verify_services.nu @@ -28,15 +28,15 @@ if ($services_toml | path exists) { print "" -# Test 2: KCL schema exists and is valid -print "Test 2: KCL services schema" -let services_kcl = "provisioning/kcl/services.k" +# Test 2: Nickel schema exists and is valid +print "Test 2: Nickel services schema" +let services_nickel = "provisioning/nickel/services.ncl" -if ($services_kcl | path exists) { - print $"βœ… KCL schema exists: ($services_kcl)" +if ($services_nickel | path exists) { + print $"βœ… Nickel schema exists: ($services_nickel)" # Check schema content - let content = (open $services_kcl | str trim) + let content = (open $services_nickel | str trim) if ($content | str contains "schema ServiceRegistry") { print "βœ… ServiceRegistry schema defined" } @@ -47,7 +47,7 @@ if ($services_kcl | path exists) { print "βœ… HealthCheck schema defined" } } else { - print $"❌ KCL schema not found: ($services_kcl)" + print $"❌ Nickel schema not found: ($services_nickel)" } print "" diff --git a/nulib/workflows/batch.nu b/nulib/workflows/batch.nu index e0e94fd..85ee966 100644 --- a/nulib/workflows/batch.nu +++ b/nulib/workflows/batch.nu @@ -30,13 +30,13 @@ def get-storage-backend []: nothing -> string { config-get "workflows.storage.backend" "filesystem" } -# Validate KCL workflow definition +# Validate Nickel workflow definition export def "batch validate" [ - workflow_file: string # Path to KCL workflow definition + workflow_file: string # Path to Nickel workflow definition --check-syntax (-s) # Check syntax only --check-dependencies (-d) # Validate dependencies ]: nothing -> record { - _print $"Validating KCL workflow: ($workflow_file)" + _print $"Validating Nickel workflow: ($workflow_file)" if not ($workflow_file | path exists) { return { @@ -53,13 +53,13 @@ export def "batch validate" [ warnings: [] } - # Check KCL syntax + # Check Nickel syntax if $check_syntax or (not $check_dependencies) { - let kcl_result = (run-external "kcl" ["fmt", "--check", $workflow_file] | complete) - if $kcl_result.exit_code == 0 { + let decl_result = (run-external "nickel" ["fmt", "--check", $workflow_file] | complete) + if $decl_result.exit_code == 0 { $validation_result | update syntax_valid true } else { - $validation_result | update errors ($validation_result.errors | append $"KCL syntax error: ($kcl_result.stderr)") + $validation_result | update errors ($validation_result.errors | append $"Nickel syntax error: ($decl_result.stderr)") } } @@ -90,9 +90,9 @@ export def "batch validate" [ $validation_result | update valid $is_valid } -# Submit KCL workflow to orchestrator +# Submit Nickel workflow to orchestrator export def "batch submit" [ - workflow_file: string # Path to KCL workflow definition + workflow_file: string # Path to Nickel workflow definition --name (-n): string # Custom workflow name --priority: int = 5 # Workflow priority (1-10) --environment: string # Target environment (dev/test/prod) @@ -675,4 +675,4 @@ export def "batch health" []: nothing -> record { } } } -} \ No newline at end of file +} diff --git a/nulib/workflows/cluster.nu b/nulib/workflows/cluster.nu index 2116501..327d246 100644 --- a/nulib/workflows/cluster.nu +++ b/nulib/workflows/cluster.nu @@ -106,4 +106,4 @@ def wait_for_workflow_completion [orchestrator: string, task_id: string]: nothin } return $result -} \ No newline at end of file +} diff --git a/nulib/workflows/management.nu b/nulib/workflows/management.nu index fee2f59..b2aa52d 100644 --- a/nulib/workflows/management.nu +++ b/nulib/workflows/management.nu @@ -295,4 +295,4 @@ export def "workflow submit" [ { status: "error", message: $"Unknown workflow type: ($workflow_type)" } } } -} \ No newline at end of file +} diff --git a/nulib/workflows/server_create.nu b/nulib/workflows/server_create.nu index 52b6934..7deb476 100644 --- a/nulib/workflows/server_create.nu +++ b/nulib/workflows/server_create.nu @@ -248,4 +248,4 @@ export def "workflow health" [ } else { { status: "unhealthy", message: "Orchestrator returned error" } } -} \ No newline at end of file +} diff --git a/nulib/workflows/taskserv.nu b/nulib/workflows/taskserv.nu index 62d79f1..38869e0 100644 --- a/nulib/workflows/taskserv.nu +++ b/nulib/workflows/taskserv.nu @@ -152,4 +152,4 @@ def wait_for_workflow_completion [orchestrator: string, task_id: string]: nothin } return $result -} \ No newline at end of file +} diff --git a/versions b/versions new file mode 100644 index 0000000..c77b243 --- /dev/null +++ b/versions @@ -0,0 +1,25 @@ +NUSHELL_VERSION="0.109.1" +NUSHELL_SOURCE="https://github.com/nushell/nushell/releases" +NU_VERSION="0.109.1" +NU_SOURCE="https://github.com/nushell/nushell/releases" + +NICKEL_VERSION="1.15.1" +NICKEL_SOURCE="https://github.com/tweag/nickel/releases" + +SOPS_VERSION="3.10.2" +SOPS_SOURCE="https://github.com/getsops/sops/releases" + +AGE_VERSION="1.2.1" +AGE_SOURCE="https://github.com/FiloSottile/age/releases" + +K9S_VERSION="0.50.6" +K9S_SOURCE="https://github.com/derailed/k9s/releases" + +PROVIDER_AWS_VERSION="2.32.11" +PROVIDER_AWS_SOURCE="https://github.com/aws/aws-cli/releases" + +PROVIDER_HCLOUD_VERSION="1.57.0" +PROVIDER_HCLOUD_SOURCE="https://github.com/hetznercloud/cli/releases" + +PROVIDER_UPCTL_VERSION="3.26.0" +PROVIDER_UPCTL_SOURCE="https://github.com/UpCloudLtd/upcloud-cli/releases" diff --git a/versions.ncl b/versions.ncl new file mode 100644 index 0000000..e6c6103 --- /dev/null +++ b/versions.ncl @@ -0,0 +1,73 @@ +# Core tools versions for provisioning system (Nickel IaC) +# Migrated from KCL - defines tool versions with detection methods + +{ + core_versions = [ + { + name = "nushell", + version = { current = "0.109.1", source = "https://github.com/nushell/nushell/releases", tags = "https://github.com/nushell/nushell/tags", site = "https://www.nushell.sh/", check_latest = false, grace_period = 86400 }, + dependencies = [], + detector = { method = "command", command = "nu -v", pattern = "(?P<capture0>[\\d.]+\\.[\\d.]+)", capture = "capture0" }, + }, + { + name = "nickel", + version = { current = "1.15.1", source = "https://github.com/tweag/nickel/releases", tags = "https://github.com/tweag/nickel/tags", site = "https://nickel-lang.org", check_latest = false, grace_period = 86400 }, + dependencies = [], + detector = { method = "command", command = "nickel --version", pattern = "nickel\\s+(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "sops", + version = { current = "3.10.2", source = "https://github.com/getsops/sops/releases", tags = "https://github.com/getsops/sops/tags", site = "https://github.com/getsops/sops", check_latest = false, grace_period = 86400 }, + dependencies = ["age"], + detector = { method = "command", command = "sops -v", pattern = "sops\\s+(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "age", + version = { current = "1.2.1", source = "https://github.com/FiloSottile/age/releases", tags = "https://github.com/FiloSottile/age/tags", site = "https://github.com/FiloSottile/age", check_latest = false, grace_period = 86400 }, + dependencies = [], + detector = { method = "command", command = "age --version", pattern = "v(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "k9s", + version = { current = "0.50.6", source = "https://github.com/derailed/k9s/releases", tags = "https://github.com/derailed/k9s/tags", site = "https://k9scli.io/", check_latest = true, grace_period = 86400 }, + dependencies = [], + detector = { method = "command", command = "k9s version", pattern = "Version\\s+v(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = true, grace_period = 86400 }, + dependencies = [], + detector = { method = "command", command = "typedialog --version", pattern = "typedialog\\s+(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog-tui", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = false, grace_period = 86400 }, + dependencies = ["typedialog"], + detector = { method = "command", command = "typedialog-tui --version", pattern = "typedialog-tui\\s+(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog-web", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = false, grace_period = 86400 }, + dependencies = ["typedialog"], + detector = { method = "command", command = "typedialog-web --version", pattern = "typedialog-web\\s+(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog-ag", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = false, grace_period = 86400 }, + dependencies = ["typedialog"], + detector = { method = "command", command = "typedialog-ag --help", pattern = "(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog-ai", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = false, grace_period = 86400 }, + dependencies = ["typedialog"], + detector = { method = "command", command = "typedialog-ai --help", pattern = "(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + { + name = "typedialog-prov-gen", + version = { current = "0.1.0", source = "https://github.com/typedialog/typedialog/releases", tags = "https://github.com/typedialog/typedialog/tags", site = "https://github.com/typedialog/typedialog", check_latest = false, grace_period = 86400 }, + dependencies = ["typedialog"], + detector = { method = "command", command = "typedialog-prov-gen --help", pattern = "(?P<capture0>[\\d.]+)", capture = "capture0" }, + }, + ] +}