provisioning/docs/book/user/extension-development.html

1608 lines
52 KiB
HTML
Raw Normal View History

<!DOCTYPE HTML>
<html lang="en" class="ayu sidebar-visible" dir="ltr">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Extension Development - Provisioning Platform Documentation</title>
<!-- Custom HTML head -->
<meta name="description" content="Complete documentation for the Provisioning Platform - Infrastructure automation with Nushell, KCL, and Rust">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff">
<link rel="icon" href="../favicon.svg">
<link rel="shortcut icon" href="../favicon.png">
<link rel="stylesheet" href="../css/variables.css">
<link rel="stylesheet" href="../css/general.css">
<link rel="stylesheet" href="../css/chrome.css">
<link rel="stylesheet" href="../css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="../fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
<!-- Custom theme stylesheets -->
<!-- Provide site root and default themes to javascript -->
<script>
const path_to_root = "../";
const default_light_theme = "ayu";
const default_dark_theme = "navy";
</script>
<!-- Start loading toc.js asap -->
<script src="../toc.js"></script>
</head>
<body>
<div id="mdbook-help-container">
<div id="mdbook-help-popup">
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
<div>
<p>Press <kbd></kbd> or <kbd></kbd> to navigate between chapters</p>
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
<p>Press <kbd>?</kbd> to show this help</p>
<p>Press <kbd>Esc</kbd> to hide this help</p>
</div>
</div>
</div>
<div id="body-container">
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script>
try {
let theme = localStorage.getItem('mdbook-theme');
let sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script>
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
let theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
const html = document.documentElement;
html.classList.remove('ayu')
html.classList.add(theme);
html.classList.add("js");
</script>
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
<!-- Hide / unhide sidebar before it is displayed -->
<script>
let sidebar = null;
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
} else {
sidebar = 'hidden';
}
sidebar_toggle.checked = sidebar === 'visible';
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<!-- populated by js -->
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
<noscript>
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
</noscript>
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
<div class="sidebar-resize-indicator"></div>
</div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky">
<div class="left-buttons">
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</label>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">Provisioning Platform Documentation</h1>
<div class="right-buttons">
<a href="../print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
<a href="https://github.com/provisioning/provisioning-platform" title="Git repository" aria-label="Git repository">
<i id="git-repository-button" class="fa fa-github"></i>
</a>
<a href="https://github.com/provisioning/provisioning-platform/edit/main/provisioning/docs/src/user/extension-development.md" title="Suggest an edit" aria-label="Suggest an edit">
<i id="git-edit-button" class="fa fa-edit"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script>
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="extension-development-guide"><a class="header" href="#extension-development-guide">Extension Development Guide</a></h1>
<p>This guide will help you create custom providers, task services, and cluster configurations to extend provisioning for your specific needs.</p>
<h2 id="what-youll-learn"><a class="header" href="#what-youll-learn">What Youll Learn</a></h2>
<ul>
<li>Extension architecture and concepts</li>
<li>Creating custom cloud providers</li>
<li>Developing task services</li>
<li>Building cluster configurations</li>
<li>Publishing and sharing extensions</li>
<li>Best practices and patterns</li>
<li>Testing and validation</li>
</ul>
<h2 id="extension-architecture"><a class="header" href="#extension-architecture">Extension Architecture</a></h2>
<h3 id="extension-types"><a class="header" href="#extension-types">Extension Types</a></h3>
<div class="table-wrapper"><table><thead><tr><th>Extension Type</th><th>Purpose</th><th>Examples</th></tr></thead><tbody>
<tr><td><strong>Providers</strong></td><td>Cloud platform integrations</td><td>Custom cloud, on-premises</td></tr>
<tr><td><strong>Task Services</strong></td><td>Software components</td><td>Custom databases, monitoring</td></tr>
<tr><td><strong>Clusters</strong></td><td>Service orchestration</td><td>Application stacks, platforms</td></tr>
<tr><td><strong>Templates</strong></td><td>Reusable configurations</td><td>Standard deployments</td></tr>
</tbody></table>
</div>
<h3 id="extension-structure"><a class="header" href="#extension-structure">Extension Structure</a></h3>
<pre><code>my-extension/
├── kcl/ # KCL schemas and models
│ ├── models/ # Data models
│ ├── providers/ # Provider definitions
│ ├── taskservs/ # Task service definitions
│ └── clusters/ # Cluster definitions
├── nulib/ # Nushell implementation
│ ├── providers/ # Provider logic
│ ├── taskservs/ # Task service logic
│ └── utils/ # Utility functions
├── templates/ # Configuration templates
├── tests/ # Test files
├── docs/ # Documentation
├── extension.toml # Extension metadata
└── README.md # Extension documentation
</code></pre>
<h3 id="extension-metadata"><a class="header" href="#extension-metadata">Extension Metadata</a></h3>
<p><code>extension.toml</code>:</p>
<pre><code class="language-toml">[extension]
name = "my-custom-provider"
version = "1.0.0"
description = "Custom cloud provider integration"
author = "Your Name &lt;you@example.com&gt;"
license = "MIT"
[compatibility]
provisioning_version = "&gt;=1.0.0"
kcl_version = "&gt;=0.11.2"
[provides]
providers = ["custom-cloud"]
taskservs = ["custom-database"]
clusters = ["custom-stack"]
[dependencies]
extensions = []
system_packages = ["curl", "jq"]
[configuration]
required_env = ["CUSTOM_CLOUD_API_KEY"]
optional_env = ["CUSTOM_CLOUD_REGION"]
</code></pre>
<h2 id="creating-custom-providers"><a class="header" href="#creating-custom-providers">Creating Custom Providers</a></h2>
<h3 id="provider-architecture"><a class="header" href="#provider-architecture">Provider Architecture</a></h3>
<p>A provider handles:</p>
<ul>
<li>Authentication with cloud APIs</li>
<li>Resource lifecycle management (create, read, update, delete)</li>
<li>Provider-specific configurations</li>
<li>Cost estimation and billing integration</li>
</ul>
<h3 id="step-1-define-provider-schema"><a class="header" href="#step-1-define-provider-schema">Step 1: Define Provider Schema</a></h3>
<p><code>kcl/providers/custom_cloud.k</code>:</p>
<pre><code class="language-kcl"># Custom cloud provider schema
import models.base
schema CustomCloudConfig(base.ProviderConfig):
"""Configuration for Custom Cloud provider"""
# Authentication
api_key: str
api_secret?: str
region?: str = "us-west-1"
# Provider-specific settings
project_id?: str
organization?: str
# API configuration
api_url?: str = "https://api.custom-cloud.com/v1"
timeout?: int = 30
# Cost configuration
billing_account?: str
cost_center?: str
schema CustomCloudServer(base.ServerConfig):
"""Server configuration for Custom Cloud"""
# Instance configuration
machine_type: str
zone: str
disk_size?: int = 20
disk_type?: str = "ssd"
# Network configuration
vpc?: str
subnet?: str
external_ip?: bool = true
# Custom Cloud specific
preemptible?: bool = false
labels?: {str: str} = {}
# Validation rules
check:
len(machine_type) &gt; 0, "machine_type cannot be empty"
disk_size &gt;= 10, "disk_size must be at least 10GB"
# Provider capabilities
provider_capabilities = {
"name": "custom-cloud"
"supports_auto_scaling": True
"supports_load_balancing": True
"supports_managed_databases": True
"regions": [
"us-west-1", "us-west-2", "us-east-1", "eu-west-1"
]
"machine_types": [
"micro", "small", "medium", "large", "xlarge"
]
}
</code></pre>
<h3 id="step-2-implement-provider-logic"><a class="header" href="#step-2-implement-provider-logic">Step 2: Implement Provider Logic</a></h3>
<p><code>nulib/providers/custom_cloud.nu</code>:</p>
<pre><code class="language-nushell"># Custom Cloud provider implementation
# Provider initialization
export def custom_cloud_init [] {
# Validate environment variables
if ($env.CUSTOM_CLOUD_API_KEY | is-empty) {
error make {
msg: "CUSTOM_CLOUD_API_KEY environment variable is required"
}
}
# Set up provider context
$env.CUSTOM_CLOUD_INITIALIZED = true
}
# Create server instance
export def custom_cloud_create_server [
server_config: record
--check: bool = false # Dry run mode
] -&gt; record {
custom_cloud_init
print $"Creating server: ($server_config.name)"
if $check {
return {
action: "create"
resource: "server"
name: $server_config.name
status: "planned"
estimated_cost: (calculate_server_cost $server_config)
}
}
# Make API call to create server
let api_response = (custom_cloud_api_call "POST" "instances" $server_config)
if ($api_response.status | str contains "error") {
error make {
msg: $"Failed to create server: ($api_response.message)"
}
}
# Wait for server to be ready
let server_id = $api_response.instance_id
custom_cloud_wait_for_server $server_id "running"
return {
id: $server_id
name: $server_config.name
status: "running"
ip_address: $api_response.ip_address
created_at: (date now | format date "%Y-%m-%d %H:%M:%S")
}
}
# Delete server instance
export def custom_cloud_delete_server [
server_name: string
--keep_storage: bool = false
] -&gt; record {
custom_cloud_init
let server = (custom_cloud_get_server $server_name)
if ($server | is-empty) {
error make {
msg: $"Server not found: ($server_name)"
}
}
print $"Deleting server: ($server_name)"
# Delete the instance
let delete_response = (custom_cloud_api_call "DELETE" $"instances/($server.id)" {
keep_storage: $keep_storage
})
return {
action: "delete"
resource: "server"
name: $server_name
status: "deleted"
}
}
# List servers
export def custom_cloud_list_servers [] -&gt; list&lt;record&gt; {
custom_cloud_init
let response = (custom_cloud_api_call "GET" "instances" {})
return ($response.instances | each {|instance|
{
id: $instance.id
name: $instance.name
status: $instance.status
machine_type: $instance.machine_type
zone: $instance.zone
ip_address: $instance.ip_address
created_at: $instance.created_at
}
})
}
# Get server details
export def custom_cloud_get_server [server_name: string] -&gt; record {
let servers = (custom_cloud_list_servers)
return ($servers | where name == $server_name | first)
}
# Calculate estimated costs
export def calculate_server_cost [server_config: record] -&gt; float {
# Cost calculation logic based on machine type
let base_costs = {
micro: 0.01
small: 0.05
medium: 0.10
large: 0.20
xlarge: 0.40
}
let machine_cost = ($base_costs | get $server_config.machine_type)
let storage_cost = ($server_config.disk_size | default 20) * 0.001
return ($machine_cost + $storage_cost)
}
# Make API call to Custom Cloud
def custom_cloud_api_call [
method: string
endpoint: string
data: record
] -&gt; record {
let api_url = ($env.CUSTOM_CLOUD_API_URL | default "https://api.custom-cloud.com/v1")
let api_key = $env.CUSTOM_CLOUD_API_KEY
let headers = {
"Authorization": $"Bearer ($api_key)"
"Content-Type": "application/json"
}
let url = $"($api_url)/($endpoint)"
match $method {
"GET" =&gt; {
http get $url --headers $headers
}
"POST" =&gt; {
http post $url --headers $headers ($data | to json)
}
"PUT" =&gt; {
http put $url --headers $headers ($data | to json)
}
"DELETE" =&gt; {
http delete $url --headers $headers
}
_ =&gt; {
error make {
msg: $"Unsupported HTTP method: ($method)"
}
}
}
}
# Wait for server to reach desired state
def custom_cloud_wait_for_server [
server_id: string
target_status: string
--timeout: int = 300
] {
let start_time = (date now)
loop {
let response = (custom_cloud_api_call "GET" $"instances/($server_id)" {})
let current_status = $response.status
if $current_status == $target_status {
print $"Server ($server_id) reached status: ($target_status)"
break
}
let elapsed = ((date now) - $start_time) / 1000000000 # Convert to seconds
if $elapsed &gt; $timeout {
error make {
msg: $"Timeout waiting for server ($server_id) to reach ($target_status)"
}
}
sleep 10sec
print $"Waiting for server status: ($current_status) -&gt; ($target_status)"
}
}
</code></pre>
<h3 id="step-3-provider-registration"><a class="header" href="#step-3-provider-registration">Step 3: Provider Registration</a></h3>
<p><code>nulib/providers/mod.nu</code>:</p>
<pre><code class="language-nushell"># Provider module exports
export use custom_cloud.nu *
# Provider registry
export def get_provider_info [] -&gt; record {
{
name: "custom-cloud"
version: "1.0.0"
capabilities: {
servers: true
load_balancers: true
databases: false
storage: true
}
regions: ["us-west-1", "us-west-2", "us-east-1", "eu-west-1"]
auth_methods: ["api_key", "oauth"]
}
}
</code></pre>
<h2 id="creating-custom-task-services"><a class="header" href="#creating-custom-task-services">Creating Custom Task Services</a></h2>
<h3 id="task-service-architecture"><a class="header" href="#task-service-architecture">Task Service Architecture</a></h3>
<p>Task services handle:</p>
<ul>
<li>Software installation and configuration</li>
<li>Service lifecycle management</li>
<li>Health checking and monitoring</li>
<li>Version management and updates</li>
</ul>
<h3 id="step-1-define-service-schema"><a class="header" href="#step-1-define-service-schema">Step 1: Define Service Schema</a></h3>
<p><code>kcl/taskservs/custom_database.k</code>:</p>
<pre><code class="language-kcl"># Custom database task service
import models.base
schema CustomDatabaseConfig(base.TaskServiceConfig):
"""Configuration for Custom Database service"""
# Database configuration
version?: str = "14.0"
port?: int = 5432
max_connections?: int = 100
memory_limit?: str = "512MB"
# Data configuration
data_directory?: str = "/var/lib/customdb"
log_directory?: str = "/var/log/customdb"
# Replication
replication?: {
enabled?: bool = false
mode?: str = "async" # async, sync
replicas?: int = 1
}
# Backup configuration
backup?: {
enabled?: bool = true
schedule?: str = "0 2 * * *" # Daily at 2 AM
retention_days?: int = 7
storage_location?: str = "local"
}
# Security
ssl?: {
enabled?: bool = true
cert_file?: str = "/etc/ssl/certs/customdb.crt"
key_file?: str = "/etc/ssl/private/customdb.key"
}
# Monitoring
monitoring?: {
enabled?: bool = true
metrics_port?: int = 9187
log_level?: str = "info"
}
check:
port &gt; 1024 and port &lt; 65536, "port must be between 1024 and 65535"
max_connections &gt; 0, "max_connections must be positive"
# Service metadata
service_metadata = {
"name": "custom-database"
"description": "Custom Database Server"
"version": "14.0"
"category": "database"
"dependencies": ["systemd"]
"supported_os": ["ubuntu", "debian", "centos", "rhel"]
"ports": [5432, 9187]
"data_directories": ["/var/lib/customdb"]
}
</code></pre>
<h3 id="step-2-implement-service-logic"><a class="header" href="#step-2-implement-service-logic">Step 2: Implement Service Logic</a></h3>
<p><code>nulib/taskservs/custom_database.nu</code>:</p>
<pre><code class="language-nushell"># Custom Database task service implementation
# Install custom database
export def install_custom_database [
config: record
--check: bool = false
] -&gt; record {
print "Installing Custom Database..."
if $check {
return {
action: "install"
service: "custom-database"
version: ($config.version | default "14.0")
status: "planned"
changes: [
"Install Custom Database packages"
"Configure database server"
"Start database service"
"Set up monitoring"
]
}
}
# Check prerequisites
validate_prerequisites $config
# Install packages
install_packages $config
# Configure service
configure_service $config
# Initialize database
initialize_database $config
# Set up monitoring
if ($config.monitoring?.enabled | default true) {
setup_monitoring $config
}
# Set up backups
if ($config.backup?.enabled | default true) {
setup_backups $config
}
# Start service
start_service
# Verify installation
let status = (verify_installation $config)
return {
action: "install"
service: "custom-database"
version: ($config.version | default "14.0")
status: $status.status
endpoint: $"localhost:($config.port | default 5432)"
data_directory: ($config.data_directory | default "/var/lib/customdb")
}
}
# Configure custom database
export def configure_custom_database [
config: record
] {
print "Configuring Custom Database..."
# Generate configuration file
let db_config = generate_config $config
$db_config | save "/etc/customdb/customdb.conf"
# Set up SSL if enabled
if ($config.ssl?.enabled | default true) {
setup_ssl $config
}
# Configure replication if enabled
if ($config.replication?.enabled | default false) {
setup_replication $config
}
# Restart service to apply configuration
restart_service
}
# Start service
export def start_custom_database [] {
print "Starting Custom Database service..."
^systemctl start customdb
^systemctl enable customdb
}
# Stop service
export def stop_custom_database [] {
print "Stopping Custom Database service..."
^systemctl stop customdb
}
# Check service status
export def status_custom_database [] -&gt; record {
let systemd_status = (^systemctl is-active customdb | str trim)
let port_check = (check_port 5432)
let version = (get_database_version)
return {
service: "custom-database"
status: $systemd_status
port_accessible: $port_check
version: $version
uptime: (get_service_uptime)
connections: (get_active_connections)
}
}
# Health check
export def health_custom_database [] -&gt; record {
let status = (status_custom_database)
let health_checks = [
{
name: "Service Running"
status: ($status.status == "active")
message: $"Systemd status: ($status.status)"
}
{
name: "Port Accessible"
status: $status.port_accessible
message: "Database port 5432 is accessible"
}
{
name: "Database Responsive"
status: (test_database_connection)
message: "Database responds to queries"
}
]
let healthy = ($health_checks | all {|check| $check.status})
return {
service: "custom-database"
healthy: $healthy
checks: $health_checks
last_check: (date now | format date "%Y-%m-%d %H:%M:%S")
}
}
# Update service
export def update_custom_database [
target_version: string
] -&gt; record {
print $"Updating Custom Database to version ($target_version)..."
# Create backup before update
backup_database "pre-update"
# Stop service
stop_custom_database
# Update packages
update_packages $target_version
# Migrate database if needed
migrate_database $target_version
# Start service
start_custom_database
# Verify update
let new_version = (get_database_version)
return {
action: "update"
service: "custom-database"
old_version: (get_previous_version)
new_version: $new_version
status: "completed"
}
}
# Remove service
export def remove_custom_database [
--keep_data: bool = false
] -&gt; record {
print "Removing Custom Database..."
# Stop service
stop_custom_database
# Remove packages
^apt remove --purge -y customdb-server customdb-client
# Remove configuration
rm -rf "/etc/customdb"
# Remove data (optional)
if not $keep_data {
print "Removing database data..."
rm -rf "/var/lib/customdb"
rm -rf "/var/log/customdb"
}
return {
action: "remove"
service: "custom-database"
data_preserved: $keep_data
status: "completed"
}
}
# Helper functions
def validate_prerequisites [config: record] {
# Check operating system
let os_info = (^lsb_release -is | str trim | str downcase)
let supported_os = ["ubuntu", "debian"]
if not ($os_info in $supported_os) {
error make {
msg: $"Unsupported OS: ($os_info). Supported: ($supported_os | str join ', ')"
}
}
# Check system resources
let memory_mb = (^free -m | lines | get 1 | split row ' ' | get 1 | into int)
if $memory_mb &lt; 512 {
error make {
msg: $"Insufficient memory: ($memory_mb)MB. Minimum 512MB required."
}
}
}
def install_packages [config: record] {
let version = ($config.version | default "14.0")
# Update package list
^apt update
# Install packages
^apt install -y $"customdb-server-($version)" $"customdb-client-($version)"
}
def configure_service [config: record] {
let config_content = generate_config $config
$config_content | save "/etc/customdb/customdb.conf"
# Set permissions
^chown -R customdb:customdb "/etc/customdb"
^chmod 600 "/etc/customdb/customdb.conf"
}
def generate_config [config: record] -&gt; string {
let port = ($config.port | default 5432)
let max_connections = ($config.max_connections | default 100)
let memory_limit = ($config.memory_limit | default "512MB")
return $"
# Custom Database Configuration
port = ($port)
max_connections = ($max_connections)
shared_buffers = ($memory_limit)
data_directory = '($config.data_directory | default "/var/lib/customdb")'
log_directory = '($config.log_directory | default "/var/log/customdb")'
# Logging
log_level = '($config.monitoring?.log_level | default "info")'
# SSL Configuration
ssl = ($config.ssl?.enabled | default true)
ssl_cert_file = '($config.ssl?.cert_file | default "/etc/ssl/certs/customdb.crt")'
ssl_key_file = '($config.ssl?.key_file | default "/etc/ssl/private/customdb.key")'
"
}
def initialize_database [config: record] {
print "Initializing database..."
# Create data directory
let data_dir = ($config.data_directory | default "/var/lib/customdb")
mkdir $data_dir
^chown -R customdb:customdb $data_dir
# Initialize database
^su - customdb -c $"customdb-initdb -D ($data_dir)"
}
def setup_monitoring [config: record] {
if ($config.monitoring?.enabled | default true) {
print "Setting up monitoring..."
# Install monitoring exporter
^apt install -y customdb-exporter
# Configure exporter
let exporter_config = $"
port: ($config.monitoring?.metrics_port | default 9187)
database_url: postgresql://localhost:($config.port | default 5432)/postgres
"
$exporter_config | save "/etc/customdb-exporter/config.yaml"
# Start exporter
^systemctl enable customdb-exporter
^systemctl start customdb-exporter
}
}
def setup_backups [config: record] {
if ($config.backup?.enabled | default true) {
print "Setting up backups..."
let schedule = ($config.backup?.schedule | default "0 2 * * *")
let retention = ($config.backup?.retention_days | default 7)
# Create backup script
let backup_script = $"#!/bin/bash
customdb-dump --all-databases &gt; /var/backups/customdb-$(date +%Y%m%d_%H%M%S).sql
find /var/backups -name 'customdb-*.sql' -mtime +($retention) -delete
"
$backup_script | save "/usr/local/bin/customdb-backup.sh"
^chmod +x "/usr/local/bin/customdb-backup.sh"
# Add to crontab
$"($schedule) /usr/local/bin/customdb-backup.sh" | ^crontab -u customdb -
}
}
def test_database_connection [] -&gt; bool {
let result = (^customdb-cli -h localhost -c "SELECT 1;" | complete)
return ($result.exit_code == 0)
}
def get_database_version [] -&gt; string {
let result = (^customdb-cli -h localhost -c "SELECT version();" | complete)
if ($result.exit_code == 0) {
return ($result.stdout | lines | first | parse "Custom Database {version}" | get version.0)
} else {
return "unknown"
}
}
def check_port [port: int] -&gt; bool {
let result = (^nc -z localhost $port | complete)
return ($result.exit_code == 0)
}
</code></pre>
<h2 id="creating-custom-clusters"><a class="header" href="#creating-custom-clusters">Creating Custom Clusters</a></h2>
<h3 id="cluster-architecture"><a class="header" href="#cluster-architecture">Cluster Architecture</a></h3>
<p>Clusters orchestrate multiple services to work together as a cohesive application stack.</p>
<h3 id="step-1-define-cluster-schema"><a class="header" href="#step-1-define-cluster-schema">Step 1: Define Cluster Schema</a></h3>
<p><code>kcl/clusters/custom_web_stack.k</code>:</p>
<pre><code class="language-kcl"># Custom web application stack
import models.base
import models.server
import models.taskserv
schema CustomWebStackConfig(base.ClusterConfig):
"""Configuration for Custom Web Application Stack"""
# Application configuration
app_name: str
app_version?: str = "latest"
environment?: str = "production"
# Web tier configuration
web_tier: {
replicas?: int = 3
instance_type?: str = "t3.medium"
load_balancer?: {
enabled?: bool = true
ssl?: bool = true
health_check_path?: str = "/health"
}
}
# Application tier configuration
app_tier: {
replicas?: int = 5
instance_type?: str = "t3.large"
auto_scaling?: {
enabled?: bool = true
min_replicas?: int = 2
max_replicas?: int = 10
cpu_threshold?: int = 70
}
}
# Database tier configuration
database_tier: {
type?: str = "postgresql" # postgresql, mysql, custom-database
instance_type?: str = "t3.xlarge"
high_availability?: bool = true
backup_enabled?: bool = true
}
# Monitoring configuration
monitoring: {
enabled?: bool = true
metrics_retention?: str = "30d"
alerting?: bool = true
}
# Networking
network: {
vpc_cidr?: str = "10.0.0.0/16"
public_subnets?: [str] = ["10.0.1.0/24", "10.0.2.0/24"]
private_subnets?: [str] = ["10.0.10.0/24", "10.0.20.0/24"]
database_subnets?: [str] = ["10.0.100.0/24", "10.0.200.0/24"]
}
check:
len(app_name) &gt; 0, "app_name cannot be empty"
web_tier.replicas &gt;= 1, "web_tier replicas must be at least 1"
app_tier.replicas &gt;= 1, "app_tier replicas must be at least 1"
# Cluster blueprint
cluster_blueprint = {
"name": "custom-web-stack"
"description": "Custom web application stack with load balancer, app servers, and database"
"version": "1.0.0"
"components": [
{
"name": "load-balancer"
"type": "taskserv"
"service": "haproxy"
"tier": "web"
}
{
"name": "web-servers"
"type": "server"
"tier": "web"
"scaling": "horizontal"
}
{
"name": "app-servers"
"type": "server"
"tier": "app"
"scaling": "horizontal"
}
{
"name": "database"
"type": "taskserv"
"service": "postgresql"
"tier": "database"
}
{
"name": "monitoring"
"type": "taskserv"
"service": "prometheus"
"tier": "monitoring"
}
]
}
</code></pre>
<h3 id="step-2-implement-cluster-logic"><a class="header" href="#step-2-implement-cluster-logic">Step 2: Implement Cluster Logic</a></h3>
<p><code>nulib/clusters/custom_web_stack.nu</code>:</p>
<pre><code class="language-nushell"># Custom Web Stack cluster implementation
# Deploy web stack cluster
export def deploy_custom_web_stack [
config: record
--check: bool = false
] -&gt; record {
print $"Deploying Custom Web Stack: ($config.app_name)"
if $check {
return {
action: "deploy"
cluster: "custom-web-stack"
app_name: $config.app_name
status: "planned"
components: [
"Network infrastructure"
"Load balancer"
"Web servers"
"Application servers"
"Database"
"Monitoring"
]
estimated_cost: (calculate_cluster_cost $config)
}
}
# Deploy in order
let network = (deploy_network $config)
let database = (deploy_database $config)
let app_servers = (deploy_app_tier $config)
let web_servers = (deploy_web_tier $config)
let load_balancer = (deploy_load_balancer $config)
let monitoring = (deploy_monitoring $config)
# Configure service discovery
configure_service_discovery $config
# Set up health checks
setup_health_checks $config
return {
action: "deploy"
cluster: "custom-web-stack"
app_name: $config.app_name
status: "deployed"
components: {
network: $network
database: $database
app_servers: $app_servers
web_servers: $web_servers
load_balancer: $load_balancer
monitoring: $monitoring
}
endpoints: {
web: $load_balancer.public_ip
monitoring: $monitoring.grafana_url
}
}
}
# Scale cluster
export def scale_custom_web_stack [
app_name: string
tier: string
replicas: int
] -&gt; record {
print $"Scaling ($tier) tier to ($replicas) replicas for ($app_name)"
match $tier {
"web" =&gt; {
scale_web_tier $app_name $replicas
}
"app" =&gt; {
scale_app_tier $app_name $replicas
}
_ =&gt; {
error make {
msg: $"Invalid tier: ($tier). Valid options: web, app"
}
}
}
return {
action: "scale"
cluster: "custom-web-stack"
app_name: $app_name
tier: $tier
new_replicas: $replicas
status: "completed"
}
}
# Update cluster
export def update_custom_web_stack [
app_name: string
config: record
] -&gt; record {
print $"Updating Custom Web Stack: ($app_name)"
# Rolling update strategy
update_app_tier $app_name $config
update_web_tier $app_name $config
update_load_balancer $app_name $config
return {
action: "update"
cluster: "custom-web-stack"
app_name: $app_name
status: "completed"
}
}
# Delete cluster
export def delete_custom_web_stack [
app_name: string
--keep_data: bool = false
] -&gt; record {
print $"Deleting Custom Web Stack: ($app_name)"
# Delete in reverse order
delete_load_balancer $app_name
delete_web_tier $app_name
delete_app_tier $app_name
if not $keep_data {
delete_database $app_name
}
delete_monitoring $app_name
delete_network $app_name
return {
action: "delete"
cluster: "custom-web-stack"
app_name: $app_name
data_preserved: $keep_data
status: "completed"
}
}
# Cluster status
export def status_custom_web_stack [
app_name: string
] -&gt; record {
let web_status = (get_web_tier_status $app_name)
let app_status = (get_app_tier_status $app_name)
let db_status = (get_database_status $app_name)
let lb_status = (get_load_balancer_status $app_name)
let monitoring_status = (get_monitoring_status $app_name)
let overall_healthy = (
$web_status.healthy and
$app_status.healthy and
$db_status.healthy and
$lb_status.healthy and
$monitoring_status.healthy
)
return {
cluster: "custom-web-stack"
app_name: $app_name
healthy: $overall_healthy
components: {
web_tier: $web_status
app_tier: $app_status
database: $db_status
load_balancer: $lb_status
monitoring: $monitoring_status
}
last_check: (date now | format date "%Y-%m-%d %H:%M:%S")
}
}
# Helper functions for deployment
def deploy_network [config: record] -&gt; record {
print "Deploying network infrastructure..."
# Create VPC
let vpc_config = {
cidr: ($config.network.vpc_cidr | default "10.0.0.0/16")
name: $"($config.app_name)-vpc"
}
# Create subnets
let subnets = [
{name: "public-1", cidr: ($config.network.public_subnets | get 0)}
{name: "public-2", cidr: ($config.network.public_subnets | get 1)}
{name: "private-1", cidr: ($config.network.private_subnets | get 0)}
{name: "private-2", cidr: ($config.network.private_subnets | get 1)}
{name: "database-1", cidr: ($config.network.database_subnets | get 0)}
{name: "database-2", cidr: ($config.network.database_subnets | get 1)}
]
return {
vpc: $vpc_config
subnets: $subnets
status: "deployed"
}
}
def deploy_database [config: record] -&gt; record {
print "Deploying database tier..."
let db_config = {
name: $"($config.app_name)-db"
type: ($config.database_tier.type | default "postgresql")
instance_type: ($config.database_tier.instance_type | default "t3.xlarge")
high_availability: ($config.database_tier.high_availability | default true)
backup_enabled: ($config.database_tier.backup_enabled | default true)
}
# Deploy database servers
if $db_config.high_availability {
deploy_ha_database $db_config
} else {
deploy_single_database $db_config
}
return {
name: $db_config.name
type: $db_config.type
high_availability: $db_config.high_availability
status: "deployed"
endpoint: $"($config.app_name)-db.local:5432"
}
}
def deploy_app_tier [config: record] -&gt; record {
print "Deploying application tier..."
let replicas = ($config.app_tier.replicas | default 5)
# Deploy app servers
mut servers = []
for i in 1..$replicas {
let server_config = {
name: $"($config.app_name)-app-($i | fill --width 2 --char '0')"
instance_type: ($config.app_tier.instance_type | default "t3.large")
subnet: "private"
}
let server = (deploy_app_server $server_config)
$servers = ($servers | append $server)
}
return {
tier: "application"
servers: $servers
replicas: $replicas
status: "deployed"
}
}
def calculate_cluster_cost [config: record] -&gt; float {
let web_cost = ($config.web_tier.replicas | default 3) * 0.10
let app_cost = ($config.app_tier.replicas | default 5) * 0.20
let db_cost = if ($config.database_tier.high_availability | default true) { 0.80 } else { 0.40 }
let lb_cost = 0.05
return ($web_cost + $app_cost + $db_cost + $lb_cost)
}
</code></pre>
<h2 id="extension-testing"><a class="header" href="#extension-testing">Extension Testing</a></h2>
<h3 id="test-structure"><a class="header" href="#test-structure">Test Structure</a></h3>
<pre><code>tests/
├── unit/ # Unit tests
│ ├── provider_test.nu # Provider unit tests
│ ├── taskserv_test.nu # Task service unit tests
│ └── cluster_test.nu # Cluster unit tests
├── integration/ # Integration tests
│ ├── provider_integration_test.nu
│ ├── taskserv_integration_test.nu
│ └── cluster_integration_test.nu
├── e2e/ # End-to-end tests
│ └── full_stack_test.nu
└── fixtures/ # Test data
├── configs/
└── mocks/
</code></pre>
<h3 id="example-unit-test"><a class="header" href="#example-unit-test">Example Unit Test</a></h3>
<p><code>tests/unit/provider_test.nu</code>:</p>
<pre><code class="language-nushell"># Unit tests for custom cloud provider
use std testing
export def test_provider_validation [] {
# Test valid configuration
let valid_config = {
api_key: "test-key"
region: "us-west-1"
project_id: "test-project"
}
let result = (validate_custom_cloud_config $valid_config)
assert equal $result.valid true
# Test invalid configuration
let invalid_config = {
region: "us-west-1"
# Missing api_key
}
let result2 = (validate_custom_cloud_config $invalid_config)
assert equal $result2.valid false
assert str contains $result2.error "api_key"
}
export def test_cost_calculation [] {
let server_config = {
machine_type: "medium"
disk_size: 50
}
let cost = (calculate_server_cost $server_config)
assert equal $cost 0.15 # 0.10 (medium) + 0.05 (50GB storage)
}
export def test_api_call_formatting [] {
let config = {
name: "test-server"
machine_type: "small"
zone: "us-west-1a"
}
let api_payload = (format_create_server_request $config)
assert str contains ($api_payload | to json) "test-server"
assert equal $api_payload.machine_type "small"
assert equal $api_payload.zone "us-west-1a"
}
</code></pre>
<h3 id="integration-test"><a class="header" href="#integration-test">Integration Test</a></h3>
<p><code>tests/integration/provider_integration_test.nu</code>:</p>
<pre><code class="language-nushell"># Integration tests for custom cloud provider
use std testing
export def test_server_lifecycle [] {
# Set up test environment
$env.CUSTOM_CLOUD_API_KEY = "test-api-key"
$env.CUSTOM_CLOUD_API_URL = "https://api.test.custom-cloud.com/v1"
let server_config = {
name: "test-integration-server"
machine_type: "micro"
zone: "us-west-1a"
}
# Test server creation
let create_result = (custom_cloud_create_server $server_config --check true)
assert equal $create_result.status "planned"
# Note: Actual creation would require valid API credentials
# In integration tests, you might use a test/sandbox environment
}
export def test_server_listing [] {
# Mock API response for testing
with-env [CUSTOM_CLOUD_API_KEY "test-key"] {
# This would test against a real API in integration environment
let servers = (custom_cloud_list_servers)
assert ($servers | is-not-empty)
}
}
</code></pre>
<h2 id="publishing-extensions"><a class="header" href="#publishing-extensions">Publishing Extensions</a></h2>
<h3 id="extension-package-structure"><a class="header" href="#extension-package-structure">Extension Package Structure</a></h3>
<pre><code>my-extension-package/
├── extension.toml # Extension metadata
├── README.md # Documentation
├── LICENSE # License file
├── CHANGELOG.md # Version history
├── examples/ # Usage examples
├── src/ # Source code
│ ├── kcl/
│ ├── nulib/
│ └── templates/
└── tests/ # Test files
</code></pre>
<h3 id="publishing-configuration"><a class="header" href="#publishing-configuration">Publishing Configuration</a></h3>
<p><code>extension.toml</code>:</p>
<pre><code class="language-toml">[extension]
name = "my-custom-provider"
version = "1.0.0"
description = "Custom cloud provider integration"
author = "Your Name &lt;you@example.com&gt;"
license = "MIT"
homepage = "https://github.com/username/my-custom-provider"
repository = "https://github.com/username/my-custom-provider"
keywords = ["cloud", "provider", "infrastructure"]
categories = ["providers"]
[compatibility]
provisioning_version = "&gt;=1.0.0"
kcl_version = "&gt;=0.11.2"
[provides]
providers = ["custom-cloud"]
taskservs = []
clusters = []
[dependencies]
system_packages = ["curl", "jq"]
extensions = []
[build]
include = ["src/**", "examples/**", "README.md", "LICENSE"]
exclude = ["tests/**", ".git/**", "*.tmp"]
</code></pre>
<h3 id="publishing-process"><a class="header" href="#publishing-process">Publishing Process</a></h3>
<pre><code class="language-bash"># 1. Validate extension
provisioning extension validate .
# 2. Run tests
provisioning extension test .
# 3. Build package
provisioning extension build .
# 4. Publish to registry
provisioning extension publish ./dist/my-custom-provider-1.0.0.tar.gz
</code></pre>
<h2 id="best-practices"><a class="header" href="#best-practices">Best Practices</a></h2>
<h3 id="1-code-organization"><a class="header" href="#1-code-organization">1. Code Organization</a></h3>
<pre><code># Follow standard structure
extension/
├── kcl/ # Schemas and models
├── nulib/ # Implementation
├── templates/ # Configuration templates
├── tests/ # Comprehensive tests
└── docs/ # Documentation
</code></pre>
<h3 id="2-error-handling"><a class="header" href="#2-error-handling">2. Error Handling</a></h3>
<pre><code class="language-nushell"># Always provide meaningful error messages
if ($api_response | get -o status | default "" | str contains "error") {
error make {
msg: $"API Error: ($api_response.message)"
label: {
text: "Custom Cloud API failure"
span: (metadata $api_response | get span)
}
help: "Check your API key and network connectivity"
}
}
</code></pre>
<h3 id="3-configuration-validation"><a class="header" href="#3-configuration-validation">3. Configuration Validation</a></h3>
<pre><code class="language-kcl"># Use KCL's validation features
schema CustomConfig:
name: str
size: int
check:
len(name) &gt; 0, "name cannot be empty"
size &gt; 0, "size must be positive"
size &lt;= 1000, "size cannot exceed 1000"
</code></pre>
<h3 id="4-testing"><a class="header" href="#4-testing">4. Testing</a></h3>
<ul>
<li>Write comprehensive unit tests</li>
<li>Include integration tests</li>
<li>Test error conditions</li>
<li>Use fixtures for consistent test data</li>
<li>Mock external dependencies</li>
</ul>
<h3 id="5-documentation"><a class="header" href="#5-documentation">5. Documentation</a></h3>
<ul>
<li>Include README with examples</li>
<li>Document all configuration options</li>
<li>Provide troubleshooting guide</li>
<li>Include architecture diagrams</li>
<li>Write API documentation</li>
</ul>
<h2 id="next-steps"><a class="header" href="#next-steps">Next Steps</a></h2>
<p>Now that you understand extension development:</p>
<ol>
<li><strong>Study existing extensions</strong> in the <code>providers/</code> and <code>taskservs/</code> directories</li>
<li><strong>Practice with simple extensions</strong> before building complex ones</li>
<li><strong>Join the community</strong> to share and collaborate on extensions</li>
<li><strong>Contribute to the core system</strong> by improving extension APIs</li>
<li><strong>Build a library</strong> of reusable templates and patterns</li>
</ol>
<p>Youre now equipped to extend provisioning for any custom requirements!</p>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="../user/RUSTYVAULT_KMS_GUIDE.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../user/NUSHELL_PLUGINS_GUIDE.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="../user/RUSTYVAULT_KMS_GUIDE.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../user/NUSHELL_PLUGINS_GUIDE.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<!-- Livereload script (if served using the cli tool) -->
<script>
const wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
const wsAddress = wsProtocol + "//" + location.host + "/" + "__livereload";
const socket = new WebSocket(wsAddress);
socket.onmessage = function (event) {
if (event.data === "reload") {
socket.close();
location.reload();
}
};
window.onbeforeunload = function() {
socket.close();
}
</script>
<script>
window.playground_copyable = true;
</script>
<script src="../elasticlunr.min.js"></script>
<script src="../mark.min.js"></script>
<script src="../searcher.js"></script>
<script src="../clipboard.min.js"></script>
<script src="../highlight.js"></script>
<script src="../book.js"></script>
<!-- Custom JS scripts -->
</div>
</body>
</html>