provisioning/docs/book/development/PROVIDER_AGNOSTIC_ARCHITECTURE.html
Jesús Pérez 6a59d34bb1
chore: update provisioning configuration and documentation
Update configuration files, templates, and internal documentation
for the provisioning repository system.

Configuration Updates:
- KMS configuration modernization
- Plugin system settings
- Service port mappings
- Test cluster topologies
- Installation configuration examples
- VM configuration defaults
- Cedar authorization policies

Documentation Updates:
- Library module documentation
- Extension API guides
- AI system documentation
- Service management guides
- Test environment setup
- Plugin usage guides
- Validator configuration documentation

All changes are backward compatible.
2025-12-11 21:50:42 +00:00

531 lines
24 KiB
HTML

<!DOCTYPE HTML>
<html lang="en" class="ayu sidebar-visible" dir="ltr">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Provider Agnostic Architecture - Provisioning Platform Documentation</title>
<!-- Custom HTML head -->
<meta name="description" content="Complete documentation for the Provisioning Platform - Infrastructure automation with Nushell, KCL, and Rust">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff">
<link rel="icon" href="../favicon.svg">
<link rel="shortcut icon" href="../favicon.png">
<link rel="stylesheet" href="../css/variables.css">
<link rel="stylesheet" href="../css/general.css">
<link rel="stylesheet" href="../css/chrome.css">
<link rel="stylesheet" href="../css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="../fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
<!-- Custom theme stylesheets -->
<!-- Provide site root and default themes to javascript -->
<script>
const path_to_root = "../";
const default_light_theme = "ayu";
const default_dark_theme = "navy";
</script>
<!-- Start loading toc.js asap -->
<script src="../toc.js"></script>
</head>
<body>
<div id="mdbook-help-container">
<div id="mdbook-help-popup">
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
<div>
<p>Press <kbd></kbd> or <kbd></kbd> to navigate between chapters</p>
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
<p>Press <kbd>?</kbd> to show this help</p>
<p>Press <kbd>Esc</kbd> to hide this help</p>
</div>
</div>
</div>
<div id="body-container">
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script>
try {
let theme = localStorage.getItem('mdbook-theme');
let sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script>
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
let theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
const html = document.documentElement;
html.classList.remove('ayu')
html.classList.add(theme);
html.classList.add("js");
</script>
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
<!-- Hide / unhide sidebar before it is displayed -->
<script>
let sidebar = null;
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
} else {
sidebar = 'hidden';
}
sidebar_toggle.checked = sidebar === 'visible';
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<!-- populated by js -->
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
<noscript>
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
</noscript>
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
<div class="sidebar-resize-indicator"></div>
</div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky">
<div class="left-buttons">
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</label>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">Provisioning Platform Documentation</h1>
<div class="right-buttons">
<a href="../print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
<a href="https://github.com/provisioning/provisioning-platform" title="Git repository" aria-label="Git repository">
<i id="git-repository-button" class="fa fa-github"></i>
</a>
<a href="https://github.com/provisioning/provisioning-platform/edit/main/provisioning/docs/src/development/PROVIDER_AGNOSTIC_ARCHITECTURE.md" title="Suggest an edit" aria-label="Suggest an edit">
<i id="git-edit-button" class="fa fa-edit"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script>
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="provider-agnostic-architecture-documentation"><a class="header" href="#provider-agnostic-architecture-documentation">Provider-Agnostic Architecture Documentation</a></h1>
<h2 id="overview"><a class="header" href="#overview">Overview</a></h2>
<p>The new provider-agnostic architecture eliminates hardcoded provider dependencies and enables true multi-provider infrastructure deployments. This addresses two critical limitations of the previous middleware:</p>
<ol>
<li><strong>Hardcoded provider dependencies</strong> - No longer requires importing specific provider modules</li>
<li><strong>Single-provider limitation</strong> - Now supports mixing multiple providers in the same deployment (e.g., AWS compute + Cloudflare DNS + UpCloud backup)</li>
</ol>
<h2 id="architecture-components"><a class="header" href="#architecture-components">Architecture Components</a></h2>
<h3 id="1-provider-interface-interfacenu"><a class="header" href="#1-provider-interface-interfacenu">1. Provider Interface (<code>interface.nu</code>)</a></h3>
<p>Defines the contract that all providers must implement:</p>
<pre><code class="language-nushell"># Standard interface functions
- query_servers
- server_info
- server_exists
- create_server
- delete_server
- server_state
- get_ip
# ... and 20+ other functions
</code></pre>
<p><strong>Key Features:</strong></p>
<ul>
<li>Type-safe function signatures</li>
<li>Comprehensive validation</li>
<li>Provider capability flags</li>
<li>Interface versioning</li>
</ul>
<h3 id="2-provider-registry-registrynu"><a class="header" href="#2-provider-registry-registrynu">2. Provider Registry (<code>registry.nu</code>)</a></h3>
<p>Manages provider discovery and registration:</p>
<pre><code class="language-nushell"># Initialize registry
init-provider-registry
# List available providers
list-providers --available-only
# Check provider availability
is-provider-available "aws"
</code></pre>
<p><strong>Features:</strong></p>
<ul>
<li>Automatic provider discovery</li>
<li>Core and extension provider support</li>
<li>Caching for performance</li>
<li>Provider capability tracking</li>
</ul>
<h3 id="3-provider-loader-loadernu"><a class="header" href="#3-provider-loader-loadernu">3. Provider Loader (<code>loader.nu</code>)</a></h3>
<p>Handles dynamic provider loading and validation:</p>
<pre><code class="language-nushell"># Load provider dynamically
load-provider "aws"
# Get provider with auto-loading
get-provider "upcloud"
# Call provider function
call-provider-function "aws" "query_servers" $find $cols
</code></pre>
<p><strong>Features:</strong></p>
<ul>
<li>Lazy loading (load only when needed)</li>
<li>Interface compliance validation</li>
<li>Error handling and recovery</li>
<li>Provider health checking</li>
</ul>
<h3 id="4-provider-adapters"><a class="header" href="#4-provider-adapters">4. Provider Adapters</a></h3>
<p>Each provider implements a standard adapter:</p>
<pre><code>provisioning/extensions/providers/
├── aws/provider.nu # AWS adapter
├── upcloud/provider.nu # UpCloud adapter
├── local/provider.nu # Local adapter
└── {custom}/provider.nu # Custom providers
</code></pre>
<p><strong>Adapter Structure:</strong></p>
<pre><code class="language-nushell"># AWS Provider Adapter
export def query_servers [find?: string, cols?: string] {
aws_query_servers $find $cols
}
export def create_server [settings: record, server: record, check: bool, wait: bool] {
# AWS-specific implementation
}
</code></pre>
<h3 id="5-provider-agnostic-middleware-middleware_provider_agnosticnu"><a class="header" href="#5-provider-agnostic-middleware-middleware_provider_agnosticnu">5. Provider-Agnostic Middleware (<code>middleware_provider_agnostic.nu</code>)</a></h3>
<p>The new middleware that uses dynamic dispatch:</p>
<pre><code class="language-nushell"># No hardcoded imports!
export def mw_query_servers [settings: record, find?: string, cols?: string] {
$settings.data.servers | each { |server|
# Dynamic provider loading and dispatch
dispatch_provider_function $server.provider "query_servers" $find $cols
}
}
</code></pre>
<h2 id="multi-provider-support"><a class="header" href="#multi-provider-support">Multi-Provider Support</a></h2>
<h3 id="example-mixed-provider-infrastructure"><a class="header" href="#example-mixed-provider-infrastructure">Example: Mixed Provider Infrastructure</a></h3>
<pre><code class="language-kcl">servers = [
aws.Server {
hostname = "compute-01"
provider = "aws"
# AWS-specific config
}
upcloud.Server {
hostname = "backup-01"
provider = "upcloud"
# UpCloud-specific config
}
cloudflare.DNS {
hostname = "api.example.com"
provider = "cloudflare"
# DNS-specific config
}
]
</code></pre>
<h3 id="multi-provider-deployment"><a class="header" href="#multi-provider-deployment">Multi-Provider Deployment</a></h3>
<pre><code class="language-nushell"># Deploy across multiple providers automatically
mw_deploy_multi_provider_infra $settings $deployment_plan
# Get deployment strategy recommendations
mw_suggest_deployment_strategy {
regions: ["us-east-1", "eu-west-1"]
high_availability: true
cost_optimization: true
}
</code></pre>
<h2 id="provider-capabilities"><a class="header" href="#provider-capabilities">Provider Capabilities</a></h2>
<p>Providers declare their capabilities:</p>
<pre><code class="language-nushell">capabilities: {
server_management: true
network_management: true
auto_scaling: true # AWS: yes, Local: no
multi_region: true # AWS: yes, Local: no
serverless: true # AWS: yes, UpCloud: no
compliance_certifications: ["SOC2", "HIPAA"]
}
</code></pre>
<h2 id="migration-guide"><a class="header" href="#migration-guide">Migration Guide</a></h2>
<h3 id="from-old-middleware"><a class="header" href="#from-old-middleware">From Old Middleware</a></h3>
<p><strong>Before (hardcoded):</strong></p>
<pre><code class="language-nushell"># middleware.nu
use ../aws/nulib/aws/servers.nu *
use ../upcloud/nulib/upcloud/servers.nu *
match $server.provider {
"aws" =&gt; { aws_query_servers $find $cols }
"upcloud" =&gt; { upcloud_query_servers $find $cols }
}
</code></pre>
<p><strong>After (provider-agnostic):</strong></p>
<pre><code class="language-nushell"># middleware_provider_agnostic.nu
# No hardcoded imports!
# Dynamic dispatch
dispatch_provider_function $server.provider "query_servers" $find $cols
</code></pre>
<h3 id="migration-steps"><a class="header" href="#migration-steps">Migration Steps</a></h3>
<ol>
<li>
<p><strong>Replace middleware file:</strong></p>
<pre><code class="language-bash">cp provisioning/extensions/providers/prov_lib/middleware.nu \
provisioning/extensions/providers/prov_lib/middleware_legacy.backup
cp provisioning/extensions/providers/prov_lib/middleware_provider_agnostic.nu \
provisioning/extensions/providers/prov_lib/middleware.nu
</code></pre>
</li>
<li>
<p><strong>Test with existing infrastructure:</strong></p>
<pre><code class="language-nushell">./provisioning/tools/test-provider-agnostic.nu run-all-tests
</code></pre>
</li>
<li>
<p><strong>Update any custom code</strong> that directly imported provider modules</p>
</li>
</ol>
<h2 id="adding-new-providers"><a class="header" href="#adding-new-providers">Adding New Providers</a></h2>
<h3 id="1-create-provider-adapter"><a class="header" href="#1-create-provider-adapter">1. Create Provider Adapter</a></h3>
<p>Create <code>provisioning/extensions/providers/{name}/provider.nu</code>:</p>
<pre><code class="language-nushell"># Digital Ocean Provider Example
export def get-provider-metadata [] {
{
name: "digitalocean"
version: "1.0.0"
capabilities: {
server_management: true
# ... other capabilities
}
}
}
# Implement required interface functions
export def query_servers [find?: string, cols?: string] {
# DigitalOcean-specific implementation
}
export def create_server [settings: record, server: record, check: bool, wait: bool] {
# DigitalOcean-specific implementation
}
# ... implement all required functions
</code></pre>
<h3 id="2-provider-discovery"><a class="header" href="#2-provider-discovery">2. Provider Discovery</a></h3>
<p>The registry will automatically discover the new provider on next initialization.</p>
<h3 id="3-test-new-provider"><a class="header" href="#3-test-new-provider">3. Test New Provider</a></h3>
<pre><code class="language-nushell"># Check if discovered
is-provider-available "digitalocean"
# Load and test
load-provider "digitalocean"
check-provider-health "digitalocean"
</code></pre>
<h2 id="best-practices"><a class="header" href="#best-practices">Best Practices</a></h2>
<h3 id="provider-development"><a class="header" href="#provider-development">Provider Development</a></h3>
<ol>
<li><strong>Implement full interface</strong> - All functions must be implemented</li>
<li><strong>Handle errors gracefully</strong> - Return appropriate error values</li>
<li><strong>Follow naming conventions</strong> - Use consistent function naming</li>
<li><strong>Document capabilities</strong> - Accurately declare what your provider supports</li>
<li><strong>Test thoroughly</strong> - Validate against the interface specification</li>
</ol>
<h3 id="multi-provider-deployments"><a class="header" href="#multi-provider-deployments">Multi-Provider Deployments</a></h3>
<ol>
<li><strong>Use capability-based selection</strong> - Choose providers based on required features</li>
<li><strong>Handle provider failures</strong> - Design for provider unavailability</li>
<li><strong>Optimize for cost/performance</strong> - Mix providers strategically</li>
<li><strong>Monitor cross-provider dependencies</strong> - Understand inter-provider communication</li>
</ol>
<h3 id="profile-based-security"><a class="header" href="#profile-based-security">Profile-Based Security</a></h3>
<pre><code class="language-nushell"># Environment profiles can restrict providers
PROVISIONING_PROFILE=production # Only allows certified providers
PROVISIONING_PROFILE=development # Allows all providers including local
</code></pre>
<h2 id="troubleshooting"><a class="header" href="#troubleshooting">Troubleshooting</a></h2>
<h3 id="common-issues"><a class="header" href="#common-issues">Common Issues</a></h3>
<ol>
<li>
<p><strong>Provider not found</strong></p>
<ul>
<li>Check provider is in correct directory</li>
<li>Verify provider.nu exists and implements interface</li>
<li>Run <code>init-provider-registry</code> to refresh</li>
</ul>
</li>
<li>
<p><strong>Interface validation failed</strong></p>
<ul>
<li>Use <code>validate-provider-interface</code> to check compliance</li>
<li>Ensure all required functions are implemented</li>
<li>Check function signatures match interface</li>
</ul>
</li>
<li>
<p><strong>Provider loading errors</strong></p>
<ul>
<li>Check Nushell module syntax</li>
<li>Verify import paths are correct</li>
<li>Use <code>check-provider-health</code> for diagnostics</li>
</ul>
</li>
</ol>
<h3 id="debug-commands"><a class="header" href="#debug-commands">Debug Commands</a></h3>
<pre><code class="language-nushell"># Registry diagnostics
get-provider-stats
list-providers --verbose
# Provider diagnostics
check-provider-health "aws"
check-all-providers-health
# Loader diagnostics
get-loader-stats
</code></pre>
<h2 id="performance-benefits"><a class="header" href="#performance-benefits">Performance Benefits</a></h2>
<ol>
<li><strong>Lazy Loading</strong> - Providers loaded only when needed</li>
<li><strong>Caching</strong> - Provider registry cached to disk</li>
<li><strong>Reduced Memory</strong> - No hardcoded imports reducing memory usage</li>
<li><strong>Parallel Operations</strong> - Multi-provider operations can run in parallel</li>
</ol>
<h2 id="future-enhancements"><a class="header" href="#future-enhancements">Future Enhancements</a></h2>
<ol>
<li><strong>Provider Plugins</strong> - Support for external provider plugins</li>
<li><strong>Provider Versioning</strong> - Multiple versions of same provider</li>
<li><strong>Provider Composition</strong> - Compose providers for complex scenarios</li>
<li><strong>Provider Marketplace</strong> - Community provider sharing</li>
</ol>
<h2 id="api-reference"><a class="header" href="#api-reference">API Reference</a></h2>
<p>See the interface specification for complete function documentation:</p>
<pre><code class="language-nushell">get-provider-interface-docs | table
</code></pre>
<p>This returns the complete API with signatures and descriptions for all provider interface functions.</p>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="../development/extensions.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../development/QUICK_PROVIDER_GUIDE.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="../development/extensions.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../development/QUICK_PROVIDER_GUIDE.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<!-- Livereload script (if served using the cli tool) -->
<script>
const wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
const wsAddress = wsProtocol + "//" + location.host + "/" + "__livereload";
const socket = new WebSocket(wsAddress);
socket.onmessage = function (event) {
if (event.data === "reload") {
socket.close();
location.reload();
}
};
window.onbeforeunload = function() {
socket.close();
}
</script>
<script>
window.playground_copyable = true;
</script>
<script src="../elasticlunr.min.js"></script>
<script src="../mark.min.js"></script>
<script src="../searcher.js"></script>
<script src="../clipboard.min.js"></script>
<script src="../highlight.js"></script>
<script src="../book.js"></script>
<!-- Custom JS scripts -->
</div>
</body>
</html>