863 lines
37 KiB
HTML
863 lines
37 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="ayu sidebar-visible" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>Update Infrastructure - Provisioning Platform Documentation</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="Complete documentation for the Provisioning Platform - Infrastructure automation with Nushell, Nickel, and Rust">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
|
|
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
|
|
|
|
<!-- Provide site root and default themes to javascript -->
|
|
<script>
|
|
const path_to_root = "../";
|
|
const default_light_theme = "ayu";
|
|
const default_dark_theme = "navy";
|
|
</script>
|
|
<!-- Start loading toc.js asap -->
|
|
<script src="../toc.js"></script>
|
|
</head>
|
|
<body>
|
|
<div id="mdbook-help-container">
|
|
<div id="mdbook-help-popup">
|
|
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
|
|
<div>
|
|
<p>Press <kbd>←</kbd> or <kbd>→</kbd> to navigate between chapters</p>
|
|
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
|
|
<p>Press <kbd>?</kbd> to show this help</p>
|
|
<p>Press <kbd>Esc</kbd> to hide this help</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div id="body-container">
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
let theme = localStorage.getItem('mdbook-theme');
|
|
let sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
|
let theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
const html = document.documentElement;
|
|
html.classList.remove('ayu')
|
|
html.classList.add(theme);
|
|
html.classList.add("js");
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
let sidebar = null;
|
|
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<!-- populated by js -->
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
|
<noscript>
|
|
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
|
|
</noscript>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
|
<div class="sidebar-resize-indicator"></div>
|
|
</div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Provisioning Platform Documentation</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
<a href="https://github.com/provisioning/provisioning-platform" title="Git repository" aria-label="Git repository">
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
|
</a>
|
|
<a href="https://github.com/provisioning/provisioning-platform/edit/main/provisioning/docs/src/guides/update-infrastructure.md" title="Suggest an edit" aria-label="Suggest an edit">
|
|
<i id="git-edit-button" class="fa fa-edit"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<h1 id="update-existing-infrastructure"><a class="header" href="#update-existing-infrastructure">Update Existing Infrastructure</a></h1>
|
|
<p><strong>Goal</strong>: Safely update running infrastructure with minimal downtime
|
|
<strong>Time</strong>: 15-30 minutes
|
|
<strong>Difficulty</strong>: Intermediate</p>
|
|
<h2 id="overview"><a class="header" href="#overview">Overview</a></h2>
|
|
<p>This guide covers:</p>
|
|
<ol>
|
|
<li>Checking for updates</li>
|
|
<li>Planning update strategies</li>
|
|
<li>Updating task services</li>
|
|
<li>Rolling updates</li>
|
|
<li>Rollback procedures</li>
|
|
<li>Verification</li>
|
|
</ol>
|
|
<h2 id="update-strategies"><a class="header" href="#update-strategies">Update Strategies</a></h2>
|
|
<h3 id="strategy-1-in-place-updates-fastest"><a class="header" href="#strategy-1-in-place-updates-fastest">Strategy 1: In-Place Updates (Fastest)</a></h3>
|
|
<p><strong>Best for</strong>: Non-critical environments, development, staging</p>
|
|
<pre><code class="language-bash"># Direct update without downtime consideration
|
|
provisioning t create <taskserv> --infra <project>
|
|
</code></pre>
|
|
<h3 id="strategy-2-rolling-updates-recommended"><a class="header" href="#strategy-2-rolling-updates-recommended">Strategy 2: Rolling Updates (Recommended)</a></h3>
|
|
<p><strong>Best for</strong>: Production environments, high availability</p>
|
|
<pre><code class="language-bash"># Update servers one by one
|
|
provisioning s update --infra <project> --rolling
|
|
</code></pre>
|
|
<h3 id="strategy-3-blue-green-deployment-safest"><a class="header" href="#strategy-3-blue-green-deployment-safest">Strategy 3: Blue-Green Deployment (Safest)</a></h3>
|
|
<p><strong>Best for</strong>: Critical production, zero-downtime requirements</p>
|
|
<pre><code class="language-bash"># Create new infrastructure, switch traffic, remove old
|
|
provisioning ws init <project>-green
|
|
# ... configure and deploy
|
|
# ... switch traffic
|
|
provisioning ws delete <project>-blue
|
|
</code></pre>
|
|
<h2 id="step-1-check-for-updates"><a class="header" href="#step-1-check-for-updates">Step 1: Check for Updates</a></h2>
|
|
<h3 id="11-check-all-task-services"><a class="header" href="#11-check-all-task-services">1.1 Check All Task Services</a></h3>
|
|
<pre><code class="language-bash"># Check all taskservs for updates
|
|
provisioning t check-updates
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">📦 Task Service Update Check:
|
|
|
|
NAME CURRENT LATEST STATUS
|
|
kubernetes 1.29.0 1.30.0 ⬆️ update available
|
|
containerd 1.7.13 1.7.13 ✅ up-to-date
|
|
cilium 1.14.5 1.15.0 ⬆️ update available
|
|
postgres 15.5 16.1 ⬆️ update available
|
|
redis 7.2.3 7.2.3 ✅ up-to-date
|
|
|
|
Updates available: 3
|
|
</code></pre>
|
|
<h3 id="12-check-specific-task-service"><a class="header" href="#12-check-specific-task-service">1.2 Check Specific Task Service</a></h3>
|
|
<pre><code class="language-bash"># Check specific taskserv
|
|
provisioning t check-updates kubernetes
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">📦 Kubernetes Update Check:
|
|
|
|
Current: 1.29.0
|
|
Latest: 1.30.0
|
|
Status: ⬆️ Update available
|
|
|
|
Changelog:
|
|
• Enhanced security features
|
|
• Performance improvements
|
|
• Bug fixes in kube-apiserver
|
|
• New workload resource types
|
|
|
|
Breaking Changes:
|
|
• None
|
|
|
|
Recommended: ✅ Safe to update
|
|
</code></pre>
|
|
<h3 id="13-check-version-status"><a class="header" href="#13-check-version-status">1.3 Check Version Status</a></h3>
|
|
<pre><code class="language-bash"># Show detailed version information
|
|
provisioning version show
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">📋 Component Versions:
|
|
|
|
COMPONENT CURRENT LATEST DAYS OLD STATUS
|
|
kubernetes 1.29.0 1.30.0 45 ⬆️ update
|
|
containerd 1.7.13 1.7.13 0 ✅ current
|
|
cilium 1.14.5 1.15.0 30 ⬆️ update
|
|
postgres 15.5 16.1 60 ⬆️ update (major)
|
|
redis 7.2.3 7.2.3 0 ✅ current
|
|
</code></pre>
|
|
<h3 id="14-check-for-security-updates"><a class="header" href="#14-check-for-security-updates">1.4 Check for Security Updates</a></h3>
|
|
<pre><code class="language-bash"># Check for security-related updates
|
|
provisioning version updates --security-only
|
|
</code></pre>
|
|
<h2 id="step-2-plan-your-update"><a class="header" href="#step-2-plan-your-update">Step 2: Plan Your Update</a></h2>
|
|
<h3 id="21-review-current-configuration"><a class="header" href="#21-review-current-configuration">2.1 Review Current Configuration</a></h3>
|
|
<pre><code class="language-bash"># Show current infrastructure
|
|
provisioning show settings --infra my-production
|
|
</code></pre>
|
|
<h3 id="22-backup-configuration"><a class="header" href="#22-backup-configuration">2.2 Backup Configuration</a></h3>
|
|
<pre><code class="language-bash"># Create configuration backup
|
|
cp -r workspace/infra/my-production workspace/infra/my-production.backup-$(date +%Y%m%d)
|
|
|
|
# Or use built-in backup
|
|
provisioning ws backup my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">✅ Backup created: workspace/backups/my-production-20250930.tar.gz
|
|
</code></pre>
|
|
<h3 id="23-create-update-plan"><a class="header" href="#23-create-update-plan">2.3 Create Update Plan</a></h3>
|
|
<pre><code class="language-bash"># Generate update plan
|
|
provisioning plan update --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">📝 Update Plan for my-production:
|
|
|
|
Phase 1: Minor Updates (Low Risk)
|
|
• containerd: No update needed
|
|
• redis: No update needed
|
|
|
|
Phase 2: Patch Updates (Medium Risk)
|
|
• cilium: 1.14.5 → 1.15.0 (estimated 5 minutes)
|
|
|
|
Phase 3: Major Updates (High Risk - Requires Testing)
|
|
• kubernetes: 1.29.0 → 1.30.0 (estimated 15 minutes)
|
|
• postgres: 15.5 → 16.1 (estimated 10 minutes, may require data migration)
|
|
|
|
Recommended Order:
|
|
1. Update cilium (low risk)
|
|
2. Update kubernetes (test in staging first)
|
|
3. Update postgres (requires maintenance window)
|
|
|
|
Total Estimated Time: 30 minutes
|
|
Recommended: Test in staging environment first
|
|
</code></pre>
|
|
<h2 id="step-3-update-task-services"><a class="header" href="#step-3-update-task-services">Step 3: Update Task Services</a></h2>
|
|
<h3 id="31-update-non-critical-service-cilium-example"><a class="header" href="#31-update-non-critical-service-cilium-example">3.1 Update Non-Critical Service (Cilium Example)</a></h3>
|
|
<h4 id="dry-run-update"><a class="header" href="#dry-run-update">Dry-Run Update</a></h4>
|
|
<pre><code class="language-bash"># Test update without applying
|
|
provisioning t create cilium --infra my-production --check
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🔍 CHECK MODE: Simulating Cilium update
|
|
|
|
Current: 1.14.5
|
|
Target: 1.15.0
|
|
|
|
Would perform:
|
|
1. Download Cilium 1.15.0
|
|
2. Update configuration
|
|
3. Rolling restart of Cilium pods
|
|
4. Verify connectivity
|
|
|
|
Estimated downtime: <1 minute per node
|
|
No errors detected. Ready to update.
|
|
</code></pre>
|
|
<h4 id="generate-updated-configuration"><a class="header" href="#generate-updated-configuration">Generate Updated Configuration</a></h4>
|
|
<pre><code class="language-bash"># Generate new configuration
|
|
provisioning t generate cilium --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">✅ Generated Cilium configuration (version 1.15.0)
|
|
Saved to: workspace/infra/my-production/taskservs/cilium.ncl
|
|
</code></pre>
|
|
<h4 id="apply-update"><a class="header" href="#apply-update">Apply Update</a></h4>
|
|
<pre><code class="language-bash"># Apply update
|
|
provisioning t create cilium --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating Cilium on my-production...
|
|
|
|
Downloading Cilium 1.15.0... ⏳
|
|
✅ Downloaded
|
|
|
|
Updating configuration... ⏳
|
|
✅ Configuration updated
|
|
|
|
Rolling restart: web-01... ⏳
|
|
✅ web-01 updated (Cilium 1.15.0)
|
|
|
|
Rolling restart: web-02... ⏳
|
|
✅ web-02 updated (Cilium 1.15.0)
|
|
|
|
Verifying connectivity... ⏳
|
|
✅ All nodes connected
|
|
|
|
🎉 Cilium update complete!
|
|
Version: 1.14.5 → 1.15.0
|
|
Downtime: 0 minutes
|
|
</code></pre>
|
|
<h4 id="verify-update"><a class="header" href="#verify-update">Verify Update</a></h4>
|
|
<pre><code class="language-bash"># Verify updated version
|
|
provisioning version taskserv cilium
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">📦 Cilium Version Info:
|
|
|
|
Installed: 1.15.0
|
|
Latest: 1.15.0
|
|
Status: ✅ Up-to-date
|
|
|
|
Nodes:
|
|
✅ web-01: 1.15.0 (running)
|
|
✅ web-02: 1.15.0 (running)
|
|
</code></pre>
|
|
<h3 id="32-update-critical-service-kubernetes-example"><a class="header" href="#32-update-critical-service-kubernetes-example">3.2 Update Critical Service (Kubernetes Example)</a></h3>
|
|
<h4 id="test-in-staging-first"><a class="header" href="#test-in-staging-first">Test in Staging First</a></h4>
|
|
<pre><code class="language-bash"># If you have staging environment
|
|
provisioning t create kubernetes --infra my-staging --check
|
|
provisioning t create kubernetes --infra my-staging
|
|
|
|
# Run integration tests
|
|
provisioning test kubernetes --infra my-staging
|
|
</code></pre>
|
|
<h4 id="backup-current-state"><a class="header" href="#backup-current-state">Backup Current State</a></h4>
|
|
<pre><code class="language-bash"># Backup Kubernetes state
|
|
kubectl get all -A -o yaml > k8s-backup-$(date +%Y%m%d).yaml
|
|
|
|
# Backup etcd (if using external etcd)
|
|
provisioning t backup kubernetes --infra my-production
|
|
</code></pre>
|
|
<h4 id="schedule-maintenance-window"><a class="header" href="#schedule-maintenance-window">Schedule Maintenance Window</a></h4>
|
|
<pre><code class="language-bash"># Set maintenance mode (optional, if supported)
|
|
provisioning maintenance enable --infra my-production --duration 30m
|
|
</code></pre>
|
|
<h4 id="update-kubernetes"><a class="header" href="#update-kubernetes">Update Kubernetes</a></h4>
|
|
<pre><code class="language-bash"># Update control plane first
|
|
provisioning t create kubernetes --infra my-production --control-plane-only
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating Kubernetes control plane on my-production...
|
|
|
|
Draining control plane: web-01... ⏳
|
|
✅ web-01 drained
|
|
|
|
Updating control plane: web-01... ⏳
|
|
✅ web-01 updated (Kubernetes 1.30.0)
|
|
|
|
Uncordoning: web-01... ⏳
|
|
✅ web-01 ready
|
|
|
|
Verifying control plane... ⏳
|
|
✅ Control plane healthy
|
|
|
|
🎉 Control plane update complete!
|
|
</code></pre>
|
|
<pre><code class="language-bash"># Update worker nodes one by one
|
|
provisioning t create kubernetes --infra my-production --workers-only --rolling
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating Kubernetes workers on my-production...
|
|
|
|
Rolling update: web-02...
|
|
Draining... ⏳
|
|
✅ Drained (pods rescheduled)
|
|
|
|
Updating... ⏳
|
|
✅ Updated (Kubernetes 1.30.0)
|
|
|
|
Uncordoning... ⏳
|
|
✅ Ready
|
|
|
|
Waiting for pods to stabilize... ⏳
|
|
✅ All pods running
|
|
|
|
🎉 Worker update complete!
|
|
Updated: web-02
|
|
Version: 1.30.0
|
|
</code></pre>
|
|
<h4 id="verify-update-1"><a class="header" href="#verify-update-1">Verify Update</a></h4>
|
|
<pre><code class="language-bash"># Verify Kubernetes cluster
|
|
kubectl get nodes
|
|
provisioning version taskserv kubernetes
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">NAME STATUS ROLES AGE VERSION
|
|
web-01 Ready control-plane 30d v1.30.0
|
|
web-02 Ready <none> 30d v1.30.0
|
|
</code></pre>
|
|
<pre><code class="language-bash"># Run smoke tests
|
|
provisioning test kubernetes --infra my-production
|
|
</code></pre>
|
|
<h3 id="33-update-database-postgresql-example"><a class="header" href="#33-update-database-postgresql-example">3.3 Update Database (PostgreSQL Example)</a></h3>
|
|
<p>⚠️ <strong>WARNING</strong>: Database updates may require data migration. Always backup first!</p>
|
|
<h4 id="backup-database"><a class="header" href="#backup-database">Backup Database</a></h4>
|
|
<pre><code class="language-bash"># Backup PostgreSQL database
|
|
provisioning t backup postgres --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🗄️ Backing up PostgreSQL...
|
|
|
|
Creating dump: my-production-postgres-20250930.sql... ⏳
|
|
✅ Dump created (2.3 GB)
|
|
|
|
Compressing... ⏳
|
|
✅ Compressed (450 MB)
|
|
|
|
Saved to: workspace/backups/postgres/my-production-20250930.sql.gz
|
|
</code></pre>
|
|
<h4 id="check-compatibility"><a class="header" href="#check-compatibility">Check Compatibility</a></h4>
|
|
<pre><code class="language-bash"># Check if data migration is needed
|
|
provisioning t check-migration postgres --from 15.5 --to 16.1
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🔍 PostgreSQL Migration Check:
|
|
|
|
From: 15.5
|
|
To: 16.1
|
|
|
|
Migration Required: ✅ Yes (major version change)
|
|
|
|
Steps Required:
|
|
1. Dump database with pg_dump
|
|
2. Stop PostgreSQL 15.5
|
|
3. Install PostgreSQL 16.1
|
|
4. Initialize new data directory
|
|
5. Restore from dump
|
|
|
|
Estimated Time: 15-30 minutes (depending on data size)
|
|
Estimated Downtime: 15-30 minutes
|
|
|
|
Recommended: Use streaming replication for zero-downtime upgrade
|
|
</code></pre>
|
|
<h4 id="perform-update"><a class="header" href="#perform-update">Perform Update</a></h4>
|
|
<pre><code class="language-bash"># Update PostgreSQL (with automatic migration)
|
|
provisioning t create postgres --infra my-production --migrate
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating PostgreSQL on my-production...
|
|
|
|
⚠️ Major version upgrade detected (15.5 → 16.1)
|
|
Automatic migration will be performed
|
|
|
|
Dumping database... ⏳
|
|
✅ Database dumped (2.3 GB)
|
|
|
|
Stopping PostgreSQL 15.5... ⏳
|
|
✅ Stopped
|
|
|
|
Installing PostgreSQL 16.1... ⏳
|
|
✅ Installed
|
|
|
|
Initializing new data directory... ⏳
|
|
✅ Initialized
|
|
|
|
Restoring database... ⏳
|
|
✅ Restored (2.3 GB)
|
|
|
|
Starting PostgreSQL 16.1... ⏳
|
|
✅ Started
|
|
|
|
Verifying data integrity... ⏳
|
|
✅ All tables verified
|
|
|
|
🎉 PostgreSQL update complete!
|
|
Version: 15.5 → 16.1
|
|
Downtime: 18 minutes
|
|
</code></pre>
|
|
<h4 id="verify-update-2"><a class="header" href="#verify-update-2">Verify Update</a></h4>
|
|
<pre><code class="language-bash"># Verify PostgreSQL
|
|
provisioning version taskserv postgres
|
|
ssh db-01 "psql --version"
|
|
</code></pre>
|
|
<h2 id="step-4-update-multiple-services"><a class="header" href="#step-4-update-multiple-services">Step 4: Update Multiple Services</a></h2>
|
|
<h3 id="41-batch-update-sequentially"><a class="header" href="#41-batch-update-sequentially">4.1 Batch Update (Sequentially)</a></h3>
|
|
<pre><code class="language-bash"># Update multiple taskservs one by one
|
|
provisioning t update --infra my-production --taskservs cilium,containerd,redis
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating 3 taskservs on my-production...
|
|
|
|
[1/3] Updating cilium... ⏳
|
|
✅ cilium updated (1.15.0)
|
|
|
|
[2/3] Updating containerd... ⏳
|
|
✅ containerd updated (1.7.14)
|
|
|
|
[3/3] Updating redis... ⏳
|
|
✅ redis updated (7.2.4)
|
|
|
|
🎉 All updates complete!
|
|
Updated: 3 taskservs
|
|
Total time: 8 minutes
|
|
</code></pre>
|
|
<h3 id="42-parallel-update-non-dependent-services"><a class="header" href="#42-parallel-update-non-dependent-services">4.2 Parallel Update (Non-Dependent Services)</a></h3>
|
|
<pre><code class="language-bash"># Update taskservs in parallel (if they don't depend on each other)
|
|
provisioning t update --infra my-production --taskservs redis,postgres --parallel
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating 2 taskservs in parallel on my-production...
|
|
|
|
redis: Updating... ⏳
|
|
postgres: Updating... ⏳
|
|
|
|
redis: ✅ Updated (7.2.4)
|
|
postgres: ✅ Updated (16.1)
|
|
|
|
🎉 All updates complete!
|
|
Updated: 2 taskservs
|
|
Total time: 3 minutes (parallel)
|
|
</code></pre>
|
|
<h2 id="step-5-update-server-configuration"><a class="header" href="#step-5-update-server-configuration">Step 5: Update Server Configuration</a></h2>
|
|
<h3 id="51-update-server-resources"><a class="header" href="#51-update-server-resources">5.1 Update Server Resources</a></h3>
|
|
<pre><code class="language-bash"># Edit server configuration
|
|
provisioning sops workspace/infra/my-production/servers.ncl
|
|
</code></pre>
|
|
<p><strong>Example: Upgrade server plan</strong></p>
|
|
<pre><code class="language-kcl"># Before
|
|
{
|
|
name = "web-01"
|
|
plan = "1xCPU-2 GB" # Old plan
|
|
}
|
|
|
|
# After
|
|
{
|
|
name = "web-01"
|
|
plan = "2xCPU-4 GB" # New plan
|
|
}
|
|
</code></pre>
|
|
<pre><code class="language-bash"># Apply server update
|
|
provisioning s update --infra my-production --check
|
|
provisioning s update --infra my-production
|
|
</code></pre>
|
|
<h3 id="52-update-server-os"><a class="header" href="#52-update-server-os">5.2 Update Server OS</a></h3>
|
|
<pre><code class="language-bash"># Update operating system packages
|
|
provisioning s update --infra my-production --os-update
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🚀 Updating OS packages on my-production servers...
|
|
|
|
web-01: Updating packages... ⏳
|
|
✅ web-01: 24 packages updated
|
|
|
|
web-02: Updating packages... ⏳
|
|
✅ web-02: 24 packages updated
|
|
|
|
db-01: Updating packages... ⏳
|
|
✅ db-01: 24 packages updated
|
|
|
|
🎉 OS updates complete!
|
|
</code></pre>
|
|
<h2 id="step-6-rollback-procedures"><a class="header" href="#step-6-rollback-procedures">Step 6: Rollback Procedures</a></h2>
|
|
<h3 id="61-rollback-task-service"><a class="header" href="#61-rollback-task-service">6.1 Rollback Task Service</a></h3>
|
|
<p>If update fails or causes issues:</p>
|
|
<pre><code class="language-bash"># Rollback to previous version
|
|
provisioning t rollback cilium --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🔄 Rolling back Cilium on my-production...
|
|
|
|
Current: 1.15.0
|
|
Target: 1.14.5 (previous version)
|
|
|
|
Rolling back: web-01... ⏳
|
|
✅ web-01 rolled back
|
|
|
|
Rolling back: web-02... ⏳
|
|
✅ web-02 rolled back
|
|
|
|
Verifying connectivity... ⏳
|
|
✅ All nodes connected
|
|
|
|
🎉 Rollback complete!
|
|
Version: 1.15.0 → 1.14.5
|
|
</code></pre>
|
|
<h3 id="62-rollback-from-backup"><a class="header" href="#62-rollback-from-backup">6.2 Rollback from Backup</a></h3>
|
|
<pre><code class="language-bash"># Restore configuration from backup
|
|
provisioning ws restore my-production --from workspace/backups/my-production-20250930.tar.gz
|
|
</code></pre>
|
|
<h3 id="63-emergency-rollback"><a class="header" href="#63-emergency-rollback">6.3 Emergency Rollback</a></h3>
|
|
<pre><code class="language-bash"># Complete infrastructure rollback
|
|
provisioning rollback --infra my-production --to-snapshot <snapshot-id>
|
|
</code></pre>
|
|
<h2 id="step-7-post-update-verification"><a class="header" href="#step-7-post-update-verification">Step 7: Post-Update Verification</a></h2>
|
|
<h3 id="71-verify-all-components"><a class="header" href="#71-verify-all-components">7.1 Verify All Components</a></h3>
|
|
<pre><code class="language-bash"># Check overall health
|
|
provisioning health --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🏥 Health Check: my-production
|
|
|
|
Servers:
|
|
✅ web-01: Healthy
|
|
✅ web-02: Healthy
|
|
✅ db-01: Healthy
|
|
|
|
Task Services:
|
|
✅ kubernetes: 1.30.0 (healthy)
|
|
✅ containerd: 1.7.13 (healthy)
|
|
✅ cilium: 1.15.0 (healthy)
|
|
✅ postgres: 16.1 (healthy)
|
|
|
|
Clusters:
|
|
✅ buildkit: 2/2 replicas (healthy)
|
|
|
|
Overall Status: ✅ All systems healthy
|
|
</code></pre>
|
|
<h3 id="72-verify-version-updates"><a class="header" href="#72-verify-version-updates">7.2 Verify Version Updates</a></h3>
|
|
<pre><code class="language-bash"># Verify all versions are updated
|
|
provisioning version show
|
|
</code></pre>
|
|
<h3 id="73-run-integration-tests"><a class="header" href="#73-run-integration-tests">7.3 Run Integration Tests</a></h3>
|
|
<pre><code class="language-bash"># Run comprehensive tests
|
|
provisioning test all --infra my-production
|
|
</code></pre>
|
|
<p><strong>Expected Output:</strong></p>
|
|
<pre><code class="language-plaintext">🧪 Running Integration Tests...
|
|
|
|
[1/5] Server connectivity... ⏳
|
|
✅ All servers reachable
|
|
|
|
[2/5] Kubernetes health... ⏳
|
|
✅ All nodes ready, all pods running
|
|
|
|
[3/5] Network connectivity... ⏳
|
|
✅ All services reachable
|
|
|
|
[4/5] Database connectivity... ⏳
|
|
✅ PostgreSQL responsive
|
|
|
|
[5/5] Application health... ⏳
|
|
✅ All applications healthy
|
|
|
|
🎉 All tests passed!
|
|
</code></pre>
|
|
<h3 id="74-monitor-for-issues"><a class="header" href="#74-monitor-for-issues">7.4 Monitor for Issues</a></h3>
|
|
<pre><code class="language-bash"># Monitor logs for errors
|
|
provisioning logs --infra my-production --follow --level error
|
|
</code></pre>
|
|
<h2 id="update-checklist"><a class="header" href="#update-checklist">Update Checklist</a></h2>
|
|
<p>Use this checklist for production updates:</p>
|
|
<ul>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Check for available updates</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Review changelog and breaking changes</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Create configuration backup</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Test update in staging environment</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Schedule maintenance window</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Notify team/users of maintenance</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Update non-critical services first</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Verify each update before proceeding</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Update critical services with rolling updates</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Backup database before major updates</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Verify all components after update</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Run integration tests</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Monitor for issues (30 minutes minimum)</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Document any issues encountered</li>
|
|
<li><input disabled="" type="checkbox"/>
|
|
Close maintenance window</li>
|
|
</ul>
|
|
<h2 id="common-update-scenarios"><a class="header" href="#common-update-scenarios">Common Update Scenarios</a></h2>
|
|
<h3 id="scenario-1-minor-security-patch"><a class="header" href="#scenario-1-minor-security-patch">Scenario 1: Minor Security Patch</a></h3>
|
|
<pre><code class="language-bash"># Quick security update
|
|
provisioning t check-updates --security-only
|
|
provisioning t update --infra my-production --security-patches --yes
|
|
</code></pre>
|
|
<h3 id="scenario-2-major-version-upgrade"><a class="header" href="#scenario-2-major-version-upgrade">Scenario 2: Major Version Upgrade</a></h3>
|
|
<pre><code class="language-bash"># Careful major version update
|
|
provisioning ws backup my-production
|
|
provisioning t check-migration <service> --from X.Y --to X+1.Y
|
|
provisioning t create <service> --infra my-production --migrate
|
|
provisioning test all --infra my-production
|
|
</code></pre>
|
|
<h3 id="scenario-3-emergency-hotfix"><a class="header" href="#scenario-3-emergency-hotfix">Scenario 3: Emergency Hotfix</a></h3>
|
|
<pre><code class="language-bash"># Apply critical hotfix immediately
|
|
provisioning t create <service> --infra my-production --hotfix --yes
|
|
</code></pre>
|
|
<h2 id="troubleshooting-updates"><a class="header" href="#troubleshooting-updates">Troubleshooting Updates</a></h2>
|
|
<h3 id="issue-update-fails-mid-process"><a class="header" href="#issue-update-fails-mid-process">Issue: Update fails mid-process</a></h3>
|
|
<p><strong>Solution:</strong></p>
|
|
<pre><code class="language-bash"># Check update status
|
|
provisioning t status <taskserv> --infra my-production
|
|
|
|
# Resume failed update
|
|
provisioning t update <taskserv> --infra my-production --resume
|
|
|
|
# Or rollback
|
|
provisioning t rollback <taskserv> --infra my-production
|
|
</code></pre>
|
|
<h3 id="issue-service-not-starting-after-update"><a class="header" href="#issue-service-not-starting-after-update">Issue: Service not starting after update</a></h3>
|
|
<p><strong>Solution:</strong></p>
|
|
<pre><code class="language-bash"># Check logs
|
|
provisioning logs <taskserv> --infra my-production
|
|
|
|
# Verify configuration
|
|
provisioning t validate <taskserv> --infra my-production
|
|
|
|
# Rollback if necessary
|
|
provisioning t rollback <taskserv> --infra my-production
|
|
</code></pre>
|
|
<h3 id="issue-data-migration-fails"><a class="header" href="#issue-data-migration-fails">Issue: Data migration fails</a></h3>
|
|
<p><strong>Solution:</strong></p>
|
|
<pre><code class="language-bash"># Check migration logs
|
|
provisioning t migration-logs <taskserv> --infra my-production
|
|
|
|
# Restore from backup
|
|
provisioning t restore <taskserv> --infra my-production --from <backup-file>
|
|
</code></pre>
|
|
<h2 id="best-practices"><a class="header" href="#best-practices">Best Practices</a></h2>
|
|
<ol>
|
|
<li><strong>Always Test First</strong>: Test updates in staging before production</li>
|
|
<li><strong>Backup Everything</strong>: Create backups before any update</li>
|
|
<li><strong>Update Gradually</strong>: Update one service at a time</li>
|
|
<li><strong>Monitor Closely</strong>: Watch for errors after each update</li>
|
|
<li><strong>Have Rollback Plan</strong>: Always have a rollback strategy</li>
|
|
<li><strong>Document Changes</strong>: Keep update logs for reference</li>
|
|
<li><strong>Schedule Wisely</strong>: Update during low-traffic periods</li>
|
|
<li><strong>Verify Thoroughly</strong>: Run tests after each update</li>
|
|
</ol>
|
|
<h2 id="next-steps"><a class="header" href="#next-steps">Next Steps</a></h2>
|
|
<ul>
|
|
<li><strong><a href="customize-infrastructure.html">Customize Guide</a></strong> - Customize your infrastructure</li>
|
|
<li><strong><a href="from-scratch.html">From Scratch Guide</a></strong> - Deploy new infrastructure</li>
|
|
<li><strong><a href="../development/workflow.html">Workflow Guide</a></strong> - Automate with workflows</li>
|
|
</ul>
|
|
<h2 id="quick-reference"><a class="header" href="#quick-reference">Quick Reference</a></h2>
|
|
<pre><code class="language-bash"># Update workflow
|
|
provisioning t check-updates
|
|
provisioning ws backup my-production
|
|
provisioning t create <taskserv> --infra my-production --check
|
|
provisioning t create <taskserv> --infra my-production
|
|
provisioning version taskserv <taskserv>
|
|
provisioning health --infra my-production
|
|
provisioning test all --infra my-production
|
|
</code></pre>
|
|
<hr />
|
|
<p><em>This guide is part of the provisioning project documentation. Last updated: 2025-09-30</em></p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../guides/from-scratch.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../guides/customize-infrastructure.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../guides/from-scratch.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../guides/customize-infrastructure.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js"></script>
|
|
<script src="../mark.min.js"></script>
|
|
<script src="../searcher.js"></script>
|
|
|
|
<script src="../clipboard.min.js"></script>
|
|
<script src="../highlight.js"></script>
|
|
<script src="../book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|