2025-12-11 21:50:42 +00:00
<!DOCTYPE HTML>
< html lang = "en" class = "ayu sidebar-visible" dir = "ltr" >
< head >
<!-- Book generated using mdBook -->
< meta charset = "UTF-8" >
< title > Update Infrastructure - Provisioning Platform Documentation< / title >
<!-- Custom HTML head -->
2026-01-14 01:56:30 +00:00
< meta name = "description" content = "Complete documentation for the Provisioning Platform - Infrastructure automation with Nushell, Nickel, and Rust" >
2025-12-11 21:50:42 +00:00
< meta name = "viewport" content = "width=device-width, initial-scale=1" >
< meta name = "theme-color" content = "#ffffff" >
< link rel = "icon" href = "../favicon.svg" >
< link rel = "shortcut icon" href = "../favicon.png" >
< link rel = "stylesheet" href = "../css/variables.css" >
< link rel = "stylesheet" href = "../css/general.css" >
< link rel = "stylesheet" href = "../css/chrome.css" >
< link rel = "stylesheet" href = "../css/print.css" media = "print" >
<!-- Fonts -->
< link rel = "stylesheet" href = "../FontAwesome/css/font-awesome.css" >
< link rel = "stylesheet" href = "../fonts/fonts.css" >
<!-- Highlight.js Stylesheets -->
< link rel = "stylesheet" id = "highlight-css" href = "../highlight.css" >
< link rel = "stylesheet" id = "tomorrow-night-css" href = "../tomorrow-night.css" >
< link rel = "stylesheet" id = "ayu-highlight-css" href = "../ayu-highlight.css" >
<!-- Custom theme stylesheets -->
<!-- Provide site root and default themes to javascript -->
< script >
const path_to_root = "../";
const default_light_theme = "ayu";
const default_dark_theme = "navy";
< / script >
<!-- Start loading toc.js asap -->
< script src = "../toc.js" > < / script >
< / head >
< body >
< div id = "mdbook-help-container" >
< div id = "mdbook-help-popup" >
< h2 class = "mdbook-help-title" > Keyboard shortcuts< / h2 >
< div >
< p > Press < kbd > ←< / kbd > or < kbd > →< / kbd > to navigate between chapters< / p >
< p > Press < kbd > S< / kbd > or < kbd > /< / kbd > to search in the book< / p >
< p > Press < kbd > ?< / kbd > to show this help< / p >
< p > Press < kbd > Esc< / kbd > to hide this help< / p >
< / div >
< / div >
< / div >
< div id = "body-container" >
<!-- Work around some values being stored in localStorage wrapped in quotes -->
< script >
try {
let theme = localStorage.getItem('mdbook-theme');
let sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') & & theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') & & sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
< / script >
<!-- Set the theme before any content is loaded, prevents flash -->
< script >
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
let theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
const html = document.documentElement;
html.classList.remove('ayu')
html.classList.add(theme);
html.classList.add("js");
< / script >
< input type = "checkbox" id = "sidebar-toggle-anchor" class = "hidden" >
<!-- Hide / unhide sidebar before it is displayed -->
< script >
let sidebar = null;
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
} else {
sidebar = 'hidden';
}
sidebar_toggle.checked = sidebar === 'visible';
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
< / script >
< nav id = "sidebar" class = "sidebar" aria-label = "Table of contents" >
<!-- populated by js -->
< mdbook-sidebar-scrollbox class = "sidebar-scrollbox" > < / mdbook-sidebar-scrollbox >
< noscript >
< iframe class = "sidebar-iframe-outer" src = "../toc.html" > < / iframe >
< / noscript >
< div id = "sidebar-resize-handle" class = "sidebar-resize-handle" >
< div class = "sidebar-resize-indicator" > < / div >
< / div >
< / nav >
< div id = "page-wrapper" class = "page-wrapper" >
< div class = "page" >
< div id = "menu-bar-hover-placeholder" > < / div >
< div id = "menu-bar" class = "menu-bar sticky" >
< div class = "left-buttons" >
< label id = "sidebar-toggle" class = "icon-button" for = "sidebar-toggle-anchor" title = "Toggle Table of Contents" aria-label = "Toggle Table of Contents" aria-controls = "sidebar" >
< i class = "fa fa-bars" > < / i >
< / label >
< button id = "theme-toggle" class = "icon-button" type = "button" title = "Change theme" aria-label = "Change theme" aria-haspopup = "true" aria-expanded = "false" aria-controls = "theme-list" >
< i class = "fa fa-paint-brush" > < / i >
< / button >
< ul id = "theme-list" class = "theme-popup" aria-label = "Themes" role = "menu" >
< li role = "none" > < button role = "menuitem" class = "theme" id = "default_theme" > Auto< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "light" > Light< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "rust" > Rust< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "coal" > Coal< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "navy" > Navy< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "ayu" > Ayu< / button > < / li >
< / ul >
< button id = "search-toggle" class = "icon-button" type = "button" title = "Search (`/`)" aria-label = "Toggle Searchbar" aria-expanded = "false" aria-keyshortcuts = "/ s" aria-controls = "searchbar" >
< i class = "fa fa-search" > < / i >
< / button >
< / div >
< h1 class = "menu-title" > Provisioning Platform Documentation< / h1 >
< div class = "right-buttons" >
< a href = "../print.html" title = "Print this book" aria-label = "Print this book" >
< i id = "print-button" class = "fa fa-print" > < / i >
< / a >
< a href = "https://github.com/provisioning/provisioning-platform" title = "Git repository" aria-label = "Git repository" >
< i id = "git-repository-button" class = "fa fa-github" > < / i >
< / a >
< a href = "https://github.com/provisioning/provisioning-platform/edit/main/provisioning/docs/src/guides/update-infrastructure.md" title = "Suggest an edit" aria-label = "Suggest an edit" >
< i id = "git-edit-button" class = "fa fa-edit" > < / i >
< / a >
< / div >
< / div >
< div id = "search-wrapper" class = "hidden" >
< form id = "searchbar-outer" class = "searchbar-outer" >
< input type = "search" id = "searchbar" name = "searchbar" placeholder = "Search this book ..." aria-controls = "searchresults-outer" aria-describedby = "searchresults-header" >
< / form >
< div id = "searchresults-outer" class = "searchresults-outer hidden" >
< div id = "searchresults-header" class = "searchresults-header" > < / div >
< ul id = "searchresults" >
< / ul >
< / div >
< / div >
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
< script >
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
< / script >
< div id = "content" class = "content" >
< main >
2026-01-08 09:55:37 +00:00
< h1 id = "update-existing-infrastructure" > < a class = "header" href = "#update-existing-infrastructure" > Update Existing Infrastructure< / a > < / h1 >
< p > < strong > Goal< / strong > : Safely update running infrastructure with minimal downtime
< strong > Time< / strong > : 15-30 minutes
< strong > Difficulty< / strong > : Intermediate< / p >
2025-12-11 21:50:42 +00:00
< h2 id = "overview" > < a class = "header" href = "#overview" > Overview< / a > < / h2 >
2026-01-08 09:55:37 +00:00
< p > This guide covers:< / p >
< ol >
< li > Checking for updates< / li >
< li > Planning update strategies< / li >
< li > Updating task services< / li >
< li > Rolling updates< / li >
< li > Rollback procedures< / li >
< li > Verification< / li >
< / ol >
2025-12-11 21:50:42 +00:00
< h2 id = "update-strategies" > < a class = "header" href = "#update-strategies" > Update Strategies< / a > < / h2 >
2026-01-08 09:55:37 +00:00
< h3 id = "strategy-1-in-place-updates-fastest" > < a class = "header" href = "#strategy-1-in-place-updates-fastest" > Strategy 1: In-Place Updates (Fastest)< / a > < / h3 >
< p > < strong > Best for< / strong > : Non-critical environments, development, staging< / p >
< pre > < code class = "language-bash" > # Direct update without downtime consideration
provisioning t create < taskserv> --infra < project>
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "strategy-2-rolling-updates-recommended" > < a class = "header" href = "#strategy-2-rolling-updates-recommended" > Strategy 2: Rolling Updates (Recommended)< / a > < / h3 >
< p > < strong > Best for< / strong > : Production environments, high availability< / p >
< pre > < code class = "language-bash" > # Update servers one by one
2026-01-08 09:55:37 +00:00
provisioning s update --infra < project> --rolling
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "strategy-3-blue-green-deployment-safest" > < a class = "header" href = "#strategy-3-blue-green-deployment-safest" > Strategy 3: Blue-Green Deployment (Safest)< / a > < / h3 >
< p > < strong > Best for< / strong > : Critical production, zero-downtime requirements< / p >
< pre > < code class = "language-bash" > # Create new infrastructure, switch traffic, remove old
2026-01-08 09:55:37 +00:00
provisioning ws init < project> -green
# ... configure and deploy
# ... switch traffic
provisioning ws delete < project> -blue
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-1-check-for-updates" > < a class = "header" href = "#step-1-check-for-updates" > Step 1: Check for Updates< / a > < / h2 >
< h3 id = "11-check-all-task-services" > < a class = "header" href = "#11-check-all-task-services" > 1.1 Check All Task Services< / a > < / h3 >
< pre > < code class = "language-bash" > # Check all taskservs for updates
2026-01-08 09:55:37 +00:00
provisioning t check-updates
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 📦 Task Service Update Check:
2026-01-08 09:55:37 +00:00
NAME CURRENT LATEST STATUS
kubernetes 1.29.0 1.30.0 ⬆️ update available
containerd 1.7.13 1.7.13 ✅ up-to-date
cilium 1.14.5 1.15.0 ⬆️ update available
postgres 15.5 16.1 ⬆️ update available
redis 7.2.3 7.2.3 ✅ up-to-date
Updates available: 3
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "12-check-specific-task-service" > < a class = "header" href = "#12-check-specific-task-service" > 1.2 Check Specific Task Service< / a > < / h3 >
< pre > < code class = "language-bash" > # Check specific taskserv
2026-01-08 09:55:37 +00:00
provisioning t check-updates kubernetes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 📦 Kubernetes Update Check:
2026-01-08 09:55:37 +00:00
Current: 1.29.0
Latest: 1.30.0
Status: ⬆️ Update available
Changelog:
• Enhanced security features
• Performance improvements
• Bug fixes in kube-apiserver
• New workload resource types
Breaking Changes:
• None
Recommended: ✅ Safe to update
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "13-check-version-status" > < a class = "header" href = "#13-check-version-status" > 1.3 Check Version Status< / a > < / h3 >
< pre > < code class = "language-bash" > # Show detailed version information
2026-01-08 09:55:37 +00:00
provisioning version show
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 📋 Component Versions:
2026-01-08 09:55:37 +00:00
COMPONENT CURRENT LATEST DAYS OLD STATUS
kubernetes 1.29.0 1.30.0 45 ⬆️ update
containerd 1.7.13 1.7.13 0 ✅ current
cilium 1.14.5 1.15.0 30 ⬆️ update
postgres 15.5 16.1 60 ⬆️ update (major)
redis 7.2.3 7.2.3 0 ✅ current
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "14-check-for-security-updates" > < a class = "header" href = "#14-check-for-security-updates" > 1.4 Check for Security Updates< / a > < / h3 >
< pre > < code class = "language-bash" > # Check for security-related updates
2026-01-08 09:55:37 +00:00
provisioning version updates --security-only
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-2-plan-your-update" > < a class = "header" href = "#step-2-plan-your-update" > Step 2: Plan Your Update< / a > < / h2 >
< h3 id = "21-review-current-configuration" > < a class = "header" href = "#21-review-current-configuration" > 2.1 Review Current Configuration< / a > < / h3 >
< pre > < code class = "language-bash" > # Show current infrastructure
2026-01-08 09:55:37 +00:00
provisioning show settings --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "22-backup-configuration" > < a class = "header" href = "#22-backup-configuration" > 2.2 Backup Configuration< / a > < / h3 >
< pre > < code class = "language-bash" > # Create configuration backup
2026-01-08 09:55:37 +00:00
cp -r workspace/infra/my-production workspace/infra/my-production.backup-$(date +%Y%m%d)
# Or use built-in backup
provisioning ws backup my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > ✅ Backup created: workspace/backups/my-production-20250930.tar.gz
< / code > < / pre >
< h3 id = "23-create-update-plan" > < a class = "header" href = "#23-create-update-plan" > 2.3 Create Update Plan< / a > < / h3 >
< pre > < code class = "language-bash" > # Generate update plan
2026-01-08 09:55:37 +00:00
provisioning plan update --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 📝 Update Plan for my-production:
2026-01-08 09:55:37 +00:00
Phase 1: Minor Updates (Low Risk)
• containerd: No update needed
• redis: No update needed
Phase 2: Patch Updates (Medium Risk)
• cilium: 1.14.5 → 1.15.0 (estimated 5 minutes)
Phase 3: Major Updates (High Risk - Requires Testing)
• kubernetes: 1.29.0 → 1.30.0 (estimated 15 minutes)
• postgres: 15.5 → 16.1 (estimated 10 minutes, may require data migration)
Recommended Order:
1. Update cilium (low risk)
2. Update kubernetes (test in staging first)
3. Update postgres (requires maintenance window)
Total Estimated Time: 30 minutes
Recommended: Test in staging environment first
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-3-update-task-services" > < a class = "header" href = "#step-3-update-task-services" > Step 3: Update Task Services< / a > < / h2 >
< h3 id = "31-update-non-critical-service-cilium-example" > < a class = "header" href = "#31-update-non-critical-service-cilium-example" > 3.1 Update Non-Critical Service (Cilium Example)< / a > < / h3 >
< h4 id = "dry-run-update" > < a class = "header" href = "#dry-run-update" > Dry-Run Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Test update without applying
2026-01-08 09:55:37 +00:00
provisioning t create cilium --infra my-production --check
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🔍 CHECK MODE: Simulating Cilium update
2026-01-08 09:55:37 +00:00
Current: 1.14.5
Target: 1.15.0
Would perform:
1. Download Cilium 1.15.0
2. Update configuration
3. Rolling restart of Cilium pods
4. Verify connectivity
Estimated downtime: < 1 minute per node
No errors detected. Ready to update.
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "generate-updated-configuration" > < a class = "header" href = "#generate-updated-configuration" > Generate Updated Configuration< / a > < / h4 >
< pre > < code class = "language-bash" > # Generate new configuration
2026-01-08 09:55:37 +00:00
provisioning t generate cilium --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > ✅ Generated Cilium configuration (version 1.15.0)
Saved to: workspace/infra/my-production/taskservs/cilium.ncl
< / code > < / pre >
< h4 id = "apply-update" > < a class = "header" href = "#apply-update" > Apply Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Apply update
2026-01-08 09:55:37 +00:00
provisioning t create cilium --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating Cilium on my-production...
2025-12-11 21:50:42 +00:00
2026-01-08 09:55:37 +00:00
Downloading Cilium 1.15.0... ⏳
✅ Downloaded
2025-12-11 21:50:42 +00:00
2026-01-08 09:55:37 +00:00
Updating configuration... ⏳
✅ Configuration updated
Rolling restart: web-01... ⏳
✅ web-01 updated (Cilium 1.15.0)
Rolling restart: web-02... ⏳
✅ web-02 updated (Cilium 1.15.0)
Verifying connectivity... ⏳
✅ All nodes connected
🎉 Cilium update complete!
Version: 1.14.5 → 1.15.0
Downtime: 0 minutes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "verify-update" > < a class = "header" href = "#verify-update" > Verify Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Verify updated version
2026-01-08 09:55:37 +00:00
provisioning version taskserv cilium
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 📦 Cilium Version Info:
2026-01-08 09:55:37 +00:00
Installed: 1.15.0
Latest: 1.15.0
Status: ✅ Up-to-date
Nodes:
✅ web-01: 1.15.0 (running)
✅ web-02: 1.15.0 (running)
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "32-update-critical-service-kubernetes-example" > < a class = "header" href = "#32-update-critical-service-kubernetes-example" > 3.2 Update Critical Service (Kubernetes Example)< / a > < / h3 >
< h4 id = "test-in-staging-first" > < a class = "header" href = "#test-in-staging-first" > Test in Staging First< / a > < / h4 >
< pre > < code class = "language-bash" > # If you have staging environment
2026-01-08 09:55:37 +00:00
provisioning t create kubernetes --infra my-staging --check
provisioning t create kubernetes --infra my-staging
# Run integration tests
provisioning test kubernetes --infra my-staging
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "backup-current-state" > < a class = "header" href = "#backup-current-state" > Backup Current State< / a > < / h4 >
< pre > < code class = "language-bash" > # Backup Kubernetes state
2026-01-08 09:55:37 +00:00
kubectl get all -A -o yaml > k8s-backup-$(date +%Y%m%d).yaml
# Backup etcd (if using external etcd)
provisioning t backup kubernetes --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "schedule-maintenance-window" > < a class = "header" href = "#schedule-maintenance-window" > Schedule Maintenance Window< / a > < / h4 >
< pre > < code class = "language-bash" > # Set maintenance mode (optional, if supported)
2026-01-08 09:55:37 +00:00
provisioning maintenance enable --infra my-production --duration 30m
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "update-kubernetes" > < a class = "header" href = "#update-kubernetes" > Update Kubernetes< / a > < / h4 >
< pre > < code class = "language-bash" > # Update control plane first
2026-01-08 09:55:37 +00:00
provisioning t create kubernetes --infra my-production --control-plane-only
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating Kubernetes control plane on my-production...
2026-01-08 09:55:37 +00:00
Draining control plane: web-01... ⏳
✅ web-01 drained
Updating control plane: web-01... ⏳
✅ web-01 updated (Kubernetes 1.30.0)
Uncordoning: web-01... ⏳
✅ web-01 ready
Verifying control plane... ⏳
✅ Control plane healthy
🎉 Control plane update complete!
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< pre > < code class = "language-bash" > # Update worker nodes one by one
2026-01-08 09:55:37 +00:00
provisioning t create kubernetes --infra my-production --workers-only --rolling
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating Kubernetes workers on my-production...
2026-01-08 09:55:37 +00:00
Rolling update: web-02...
Draining... ⏳
✅ Drained (pods rescheduled)
Updating... ⏳
✅ Updated (Kubernetes 1.30.0)
Uncordoning... ⏳
✅ Ready
Waiting for pods to stabilize... ⏳
✅ All pods running
🎉 Worker update complete!
Updated: web-02
Version: 1.30.0
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "verify-update-1" > < a class = "header" href = "#verify-update-1" > Verify Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Verify Kubernetes cluster
2026-01-08 09:55:37 +00:00
kubectl get nodes
provisioning version taskserv kubernetes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > NAME STATUS ROLES AGE VERSION
2026-01-08 09:55:37 +00:00
web-01 Ready control-plane 30d v1.30.0
web-02 Ready < none> 30d v1.30.0
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< pre > < code class = "language-bash" > # Run smoke tests
2026-01-08 09:55:37 +00:00
provisioning test kubernetes --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "33-update-database-postgresql-example" > < a class = "header" href = "#33-update-database-postgresql-example" > 3.3 Update Database (PostgreSQL Example)< / a > < / h3 >
< p > ⚠️ < strong > WARNING< / strong > : Database updates may require data migration. Always backup first!< / p >
< h4 id = "backup-database" > < a class = "header" href = "#backup-database" > Backup Database< / a > < / h4 >
< pre > < code class = "language-bash" > # Backup PostgreSQL database
2026-01-08 09:55:37 +00:00
provisioning t backup postgres --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🗄️ Backing up PostgreSQL...
2026-01-08 09:55:37 +00:00
Creating dump: my-production-postgres-20250930.sql... ⏳
✅ Dump created (2.3 GB)
Compressing... ⏳
✅ Compressed (450 MB)
Saved to: workspace/backups/postgres/my-production-20250930.sql.gz
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "check-compatibility" > < a class = "header" href = "#check-compatibility" > Check Compatibility< / a > < / h4 >
< pre > < code class = "language-bash" > # Check if data migration is needed
2026-01-08 09:55:37 +00:00
provisioning t check-migration postgres --from 15.5 --to 16.1
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🔍 PostgreSQL Migration Check:
2026-01-08 09:55:37 +00:00
From: 15.5
To: 16.1
Migration Required: ✅ Yes (major version change)
Steps Required:
1. Dump database with pg_dump
2. Stop PostgreSQL 15.5
3. Install PostgreSQL 16.1
4. Initialize new data directory
5. Restore from dump
Estimated Time: 15-30 minutes (depending on data size)
Estimated Downtime: 15-30 minutes
Recommended: Use streaming replication for zero-downtime upgrade
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "perform-update" > < a class = "header" href = "#perform-update" > Perform Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Update PostgreSQL (with automatic migration)
2026-01-08 09:55:37 +00:00
provisioning t create postgres --infra my-production --migrate
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating PostgreSQL on my-production...
2026-01-08 09:55:37 +00:00
⚠️ Major version upgrade detected (15.5 → 16.1)
Automatic migration will be performed
Dumping database... ⏳
✅ Database dumped (2.3 GB)
Stopping PostgreSQL 15.5... ⏳
✅ Stopped
Installing PostgreSQL 16.1... ⏳
✅ Installed
Initializing new data directory... ⏳
✅ Initialized
Restoring database... ⏳
✅ Restored (2.3 GB)
Starting PostgreSQL 16.1... ⏳
✅ Started
Verifying data integrity... ⏳
✅ All tables verified
🎉 PostgreSQL update complete!
Version: 15.5 → 16.1
Downtime: 18 minutes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h4 id = "verify-update-2" > < a class = "header" href = "#verify-update-2" > Verify Update< / a > < / h4 >
< pre > < code class = "language-bash" > # Verify PostgreSQL
2026-01-08 09:55:37 +00:00
provisioning version taskserv postgres
ssh db-01 "psql --version"
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-4-update-multiple-services" > < a class = "header" href = "#step-4-update-multiple-services" > Step 4: Update Multiple Services< / a > < / h2 >
< h3 id = "41-batch-update-sequentially" > < a class = "header" href = "#41-batch-update-sequentially" > 4.1 Batch Update (Sequentially)< / a > < / h3 >
< pre > < code class = "language-bash" > # Update multiple taskservs one by one
2026-01-08 09:55:37 +00:00
provisioning t update --infra my-production --taskservs cilium,containerd,redis
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating 3 taskservs on my-production...
2026-01-08 09:55:37 +00:00
[1/3] Updating cilium... ⏳
✅ cilium updated (1.15.0)
[2/3] Updating containerd... ⏳
✅ containerd updated (1.7.14)
[3/3] Updating redis... ⏳
✅ redis updated (7.2.4)
🎉 All updates complete!
Updated: 3 taskservs
Total time: 8 minutes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "42-parallel-update-non-dependent-services" > < a class = "header" href = "#42-parallel-update-non-dependent-services" > 4.2 Parallel Update (Non-Dependent Services)< / a > < / h3 >
< pre > < code class = "language-bash" > # Update taskservs in parallel (if they don't depend on each other)
2026-01-08 09:55:37 +00:00
provisioning t update --infra my-production --taskservs redis,postgres --parallel
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating 2 taskservs in parallel on my-production...
2026-01-08 09:55:37 +00:00
redis: Updating... ⏳
postgres: Updating... ⏳
redis: ✅ Updated (7.2.4)
postgres: ✅ Updated (16.1)
🎉 All updates complete!
Updated: 2 taskservs
Total time: 3 minutes (parallel)
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-5-update-server-configuration" > < a class = "header" href = "#step-5-update-server-configuration" > Step 5: Update Server Configuration< / a > < / h2 >
< h3 id = "51-update-server-resources" > < a class = "header" href = "#51-update-server-resources" > 5.1 Update Server Resources< / a > < / h3 >
< pre > < code class = "language-bash" > # Edit server configuration
provisioning sops workspace/infra/my-production/servers.ncl
< / code > < / pre >
< p > < strong > Example: Upgrade server plan< / strong > < / p >
< pre > < code class = "language-kcl" > # Before
2026-01-08 09:55:37 +00:00
{
name = "web-01"
2026-01-12 04:42:18 +00:00
plan = "1xCPU-2 GB" # Old plan
2026-01-08 09:55:37 +00:00
}
# After
{
name = "web-01"
2026-01-12 04:42:18 +00:00
plan = "2xCPU-4 GB" # New plan
2026-01-08 09:55:37 +00:00
}
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< pre > < code class = "language-bash" > # Apply server update
2026-01-08 09:55:37 +00:00
provisioning s update --infra my-production --check
provisioning s update --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "52-update-server-os" > < a class = "header" href = "#52-update-server-os" > 5.2 Update Server OS< / a > < / h3 >
< pre > < code class = "language-bash" > # Update operating system packages
2026-01-08 09:55:37 +00:00
provisioning s update --infra my-production --os-update
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🚀 Updating OS packages on my-production servers...
2026-01-08 09:55:37 +00:00
web-01: Updating packages... ⏳
✅ web-01: 24 packages updated
web-02: Updating packages... ⏳
✅ web-02: 24 packages updated
db-01: Updating packages... ⏳
✅ db-01: 24 packages updated
🎉 OS updates complete!
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-6-rollback-procedures" > < a class = "header" href = "#step-6-rollback-procedures" > Step 6: Rollback Procedures< / a > < / h2 >
< h3 id = "61-rollback-task-service" > < a class = "header" href = "#61-rollback-task-service" > 6.1 Rollback Task Service< / a > < / h3 >
< p > If update fails or causes issues:< / p >
< pre > < code class = "language-bash" > # Rollback to previous version
2026-01-08 09:55:37 +00:00
provisioning t rollback cilium --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🔄 Rolling back Cilium on my-production...
2026-01-08 09:55:37 +00:00
Current: 1.15.0
Target: 1.14.5 (previous version)
Rolling back: web-01... ⏳
✅ web-01 rolled back
Rolling back: web-02... ⏳
✅ web-02 rolled back
Verifying connectivity... ⏳
✅ All nodes connected
🎉 Rollback complete!
Version: 1.15.0 → 1.14.5
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "62-rollback-from-backup" > < a class = "header" href = "#62-rollback-from-backup" > 6.2 Rollback from Backup< / a > < / h3 >
< pre > < code class = "language-bash" > # Restore configuration from backup
2026-01-08 09:55:37 +00:00
provisioning ws restore my-production --from workspace/backups/my-production-20250930.tar.gz
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "63-emergency-rollback" > < a class = "header" href = "#63-emergency-rollback" > 6.3 Emergency Rollback< / a > < / h3 >
< pre > < code class = "language-bash" > # Complete infrastructure rollback
2026-01-08 09:55:37 +00:00
provisioning rollback --infra my-production --to-snapshot < snapshot-id>
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "step-7-post-update-verification" > < a class = "header" href = "#step-7-post-update-verification" > Step 7: Post-Update Verification< / a > < / h2 >
< h3 id = "71-verify-all-components" > < a class = "header" href = "#71-verify-all-components" > 7.1 Verify All Components< / a > < / h3 >
< pre > < code class = "language-bash" > # Check overall health
2026-01-08 09:55:37 +00:00
provisioning health --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🏥 Health Check: my-production
2026-01-08 09:55:37 +00:00
Servers:
✅ web-01: Healthy
✅ web-02: Healthy
✅ db-01: Healthy
Task Services:
✅ kubernetes: 1.30.0 (healthy)
✅ containerd: 1.7.13 (healthy)
✅ cilium: 1.15.0 (healthy)
✅ postgres: 16.1 (healthy)
Clusters:
✅ buildkit: 2/2 replicas (healthy)
Overall Status: ✅ All systems healthy
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "72-verify-version-updates" > < a class = "header" href = "#72-verify-version-updates" > 7.2 Verify Version Updates< / a > < / h3 >
< pre > < code class = "language-bash" > # Verify all versions are updated
2026-01-08 09:55:37 +00:00
provisioning version show
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "73-run-integration-tests" > < a class = "header" href = "#73-run-integration-tests" > 7.3 Run Integration Tests< / a > < / h3 >
< pre > < code class = "language-bash" > # Run comprehensive tests
2026-01-08 09:55:37 +00:00
provisioning test all --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< p > < strong > Expected Output:< / strong > < / p >
< pre > < code class = "language-plaintext" > 🧪 Running Integration Tests...
2026-01-08 09:55:37 +00:00
[1/5] Server connectivity... ⏳
✅ All servers reachable
[2/5] Kubernetes health... ⏳
✅ All nodes ready, all pods running
[3/5] Network connectivity... ⏳
✅ All services reachable
[4/5] Database connectivity... ⏳
✅ PostgreSQL responsive
[5/5] Application health... ⏳
✅ All applications healthy
🎉 All tests passed!
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "74-monitor-for-issues" > < a class = "header" href = "#74-monitor-for-issues" > 7.4 Monitor for Issues< / a > < / h3 >
< pre > < code class = "language-bash" > # Monitor logs for errors
2026-01-08 09:55:37 +00:00
provisioning logs --infra my-production --follow --level error
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "update-checklist" > < a class = "header" href = "#update-checklist" > Update Checklist< / a > < / h2 >
< p > Use this checklist for production updates:< / p >
< ul >
< li > < input disabled = "" type = "checkbox" / >
Check for available updates< / li >
< li > < input disabled = "" type = "checkbox" / >
Review changelog and breaking changes< / li >
< li > < input disabled = "" type = "checkbox" / >
Create configuration backup< / li >
< li > < input disabled = "" type = "checkbox" / >
Test update in staging environment< / li >
< li > < input disabled = "" type = "checkbox" / >
Schedule maintenance window< / li >
< li > < input disabled = "" type = "checkbox" / >
Notify team/users of maintenance< / li >
< li > < input disabled = "" type = "checkbox" / >
Update non-critical services first< / li >
< li > < input disabled = "" type = "checkbox" / >
Verify each update before proceeding< / li >
< li > < input disabled = "" type = "checkbox" / >
Update critical services with rolling updates< / li >
< li > < input disabled = "" type = "checkbox" / >
Backup database before major updates< / li >
< li > < input disabled = "" type = "checkbox" / >
Verify all components after update< / li >
< li > < input disabled = "" type = "checkbox" / >
Run integration tests< / li >
< li > < input disabled = "" type = "checkbox" / >
Monitor for issues (30 minutes minimum)< / li >
< li > < input disabled = "" type = "checkbox" / >
Document any issues encountered< / li >
< li > < input disabled = "" type = "checkbox" / >
Close maintenance window< / li >
< / ul >
< h2 id = "common-update-scenarios" > < a class = "header" href = "#common-update-scenarios" > Common Update Scenarios< / a > < / h2 >
< h3 id = "scenario-1-minor-security-patch" > < a class = "header" href = "#scenario-1-minor-security-patch" > Scenario 1: Minor Security Patch< / a > < / h3 >
< pre > < code class = "language-bash" > # Quick security update
2026-01-08 09:55:37 +00:00
provisioning t check-updates --security-only
provisioning t update --infra my-production --security-patches --yes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "scenario-2-major-version-upgrade" > < a class = "header" href = "#scenario-2-major-version-upgrade" > Scenario 2: Major Version Upgrade< / a > < / h3 >
< pre > < code class = "language-bash" > # Careful major version update
2026-01-08 09:55:37 +00:00
provisioning ws backup my-production
provisioning t check-migration < service> --from X.Y --to X+1.Y
provisioning t create < service> --infra my-production --migrate
provisioning test all --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "scenario-3-emergency-hotfix" > < a class = "header" href = "#scenario-3-emergency-hotfix" > Scenario 3: Emergency Hotfix< / a > < / h3 >
< pre > < code class = "language-bash" > # Apply critical hotfix immediately
2026-01-08 09:55:37 +00:00
provisioning t create < service> --infra my-production --hotfix --yes
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "troubleshooting-updates" > < a class = "header" href = "#troubleshooting-updates" > Troubleshooting Updates< / a > < / h2 >
< h3 id = "issue-update-fails-mid-process" > < a class = "header" href = "#issue-update-fails-mid-process" > Issue: Update fails mid-process< / a > < / h3 >
< p > < strong > Solution:< / strong > < / p >
< pre > < code class = "language-bash" > # Check update status
2026-01-08 09:55:37 +00:00
provisioning t status < taskserv> --infra my-production
2025-12-11 21:50:42 +00:00
# Resume failed update
2026-01-08 09:55:37 +00:00
provisioning t update < taskserv> --infra my-production --resume
2025-12-11 21:50:42 +00:00
# Or rollback
2026-01-08 09:55:37 +00:00
provisioning t rollback < taskserv> --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "issue-service-not-starting-after-update" > < a class = "header" href = "#issue-service-not-starting-after-update" > Issue: Service not starting after update< / a > < / h3 >
< p > < strong > Solution:< / strong > < / p >
< pre > < code class = "language-bash" > # Check logs
2026-01-08 09:55:37 +00:00
provisioning logs < taskserv> --infra my-production
# Verify configuration
provisioning t validate < taskserv> --infra my-production
# Rollback if necessary
provisioning t rollback < taskserv> --infra my-production
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h3 id = "issue-data-migration-fails" > < a class = "header" href = "#issue-data-migration-fails" > Issue: Data migration fails< / a > < / h3 >
< p > < strong > Solution:< / strong > < / p >
< pre > < code class = "language-bash" > # Check migration logs
2026-01-08 09:55:37 +00:00
provisioning t migration-logs < taskserv> --infra my-production
# Restore from backup
provisioning t restore < taskserv> --infra my-production --from < backup-file>
2026-01-12 04:42:18 +00:00
< / code > < / pre >
< h2 id = "best-practices" > < a class = "header" href = "#best-practices" > Best Practices< / a > < / h2 >
< ol >
< li > < strong > Always Test First< / strong > : Test updates in staging before production< / li >
< li > < strong > Backup Everything< / strong > : Create backups before any update< / li >
< li > < strong > Update Gradually< / strong > : Update one service at a time< / li >
< li > < strong > Monitor Closely< / strong > : Watch for errors after each update< / li >
< li > < strong > Have Rollback Plan< / strong > : Always have a rollback strategy< / li >
< li > < strong > Document Changes< / strong > : Keep update logs for reference< / li >
< li > < strong > Schedule Wisely< / strong > : Update during low-traffic periods< / li >
< li > < strong > Verify Thoroughly< / strong > : Run tests after each update< / li >
< / ol >
< h2 id = "next-steps" > < a class = "header" href = "#next-steps" > Next Steps< / a > < / h2 >
< ul >
< li > < strong > < a href = "customize-infrastructure.html" > Customize Guide< / a > < / strong > - Customize your infrastructure< / li >
< li > < strong > < a href = "from-scratch.html" > From Scratch Guide< / a > < / strong > - Deploy new infrastructure< / li >
< li > < strong > < a href = "../development/workflow.html" > Workflow Guide< / a > < / strong > - Automate with workflows< / li >
< / ul >
< h2 id = "quick-reference" > < a class = "header" href = "#quick-reference" > Quick Reference< / a > < / h2 >
< pre > < code class = "language-bash" > # Update workflow
2026-01-08 09:55:37 +00:00
provisioning t check-updates
provisioning ws backup my-production
provisioning t create < taskserv> --infra my-production --check
provisioning t create < taskserv> --infra my-production
provisioning version taskserv < taskserv>
provisioning health --infra my-production
provisioning test all --infra my-production
2025-12-11 21:50:42 +00:00
< / code > < / pre >
2026-01-12 04:42:18 +00:00
< hr / >
< p > < em > This guide is part of the provisioning project documentation. Last updated: 2025-09-30< / em > < / p >
2025-12-11 21:50:42 +00:00
< / main >
< nav class = "nav-wrapper" aria-label = "Page navigation" >
<!-- Mobile navigation buttons -->
< a rel = "prev" href = "../guides/from-scratch.html" class = "mobile-nav-chapters previous" title = "Previous chapter" aria-label = "Previous chapter" aria-keyshortcuts = "Left" >
< i class = "fa fa-angle-left" > < / i >
< / a >
< a rel = "next prefetch" href = "../guides/customize-infrastructure.html" class = "mobile-nav-chapters next" title = "Next chapter" aria-label = "Next chapter" aria-keyshortcuts = "Right" >
< i class = "fa fa-angle-right" > < / i >
< / a >
< div style = "clear: both" > < / div >
< / nav >
< / div >
< / div >
< nav class = "nav-wide-wrapper" aria-label = "Page navigation" >
< a rel = "prev" href = "../guides/from-scratch.html" class = "nav-chapters previous" title = "Previous chapter" aria-label = "Previous chapter" aria-keyshortcuts = "Left" >
< i class = "fa fa-angle-left" > < / i >
< / a >
< a rel = "next prefetch" href = "../guides/customize-infrastructure.html" class = "nav-chapters next" title = "Next chapter" aria-label = "Next chapter" aria-keyshortcuts = "Right" >
< i class = "fa fa-angle-right" > < / i >
< / a >
< / nav >
< / div >
< script >
window.playground_copyable = true;
< / script >
< script src = "../elasticlunr.min.js" > < / script >
< script src = "../mark.min.js" > < / script >
< script src = "../searcher.js" > < / script >
< script src = "../clipboard.min.js" > < / script >
< script src = "../highlight.js" > < / script >
< script src = "../book.js" > < / script >
<!-- Custom JS scripts -->
< / div >
< / body >
< / html >
