697 lines
31 KiB
HTML
697 lines
31 KiB
HTML
|
<!DOCTYPE HTML>
|
||
|
|
<html lang="en" class="light sidebar-visible" dir="ltr">
|
||
|
|
<head>
|
||
|
|
<!-- Book generated using mdBook -->
|
||
|
|
<meta charset="UTF-8">
|
||
|
|
<title>Backup & Recovery Automation - VAPORA Platform Documentation</title>
|
||
|
|
|
||
|
|
|
||
|
|
<!-- Custom HTML head -->
|
||
|
|
|
||
|
|
<meta name="description" content="Comprehensive documentation for VAPORA, an intelligent development orchestration platform built entirely in Rust.">
|
||
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
|
|
<meta name="theme-color" content="#ffffff">
|
||
|
|
|
||
|
|
<link rel="icon" href="../favicon.svg">
|
||
|
|
<link rel="shortcut icon" href="../favicon.png">
|
||
|
|
<link rel="stylesheet" href="../css/variables.css">
|
||
|
|
<link rel="stylesheet" href="../css/general.css">
|
||
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
||
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
||
|
|
|
||
|
|
<!-- Fonts -->
|
||
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
||
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
||
|
|
|
||
|
|
<!-- Highlight.js Stylesheets -->
|
||
|
|
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
|
||
|
|
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
|
||
|
|
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
|
||
|
|
|
||
|
|
<!-- Custom theme stylesheets -->
|
||
|
|
|
||
|
|
|
||
|
|
<!-- Provide site root and default themes to javascript -->
|
||
|
|
<script>
|
||
|
|
const path_to_root = "../";
|
||
|
|
const default_light_theme = "light";
|
||
|
|
const default_dark_theme = "dark";
|
||
|
|
</script>
|
||
|
|
<!-- Start loading toc.js asap -->
|
||
|
|
<script src="../toc.js"></script>
|
||
|
|
</head>
|
||
|
|
<body>
|
||
|
|
<div id="mdbook-help-container">
|
||
|
|
<div id="mdbook-help-popup">
|
||
|
|
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
|
||
|
|
<div>
|
||
|
|
<p>Press <kbd>←</kbd> or <kbd>→</kbd> to navigate between chapters</p>
|
||
|
|
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
|
||
|
|
<p>Press <kbd>?</kbd> to show this help</p>
|
||
|
|
<p>Press <kbd>Esc</kbd> to hide this help</p>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
<div id="body-container">
|
||
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
||
|
|
<script>
|
||
|
|
try {
|
||
|
|
let theme = localStorage.getItem('mdbook-theme');
|
||
|
|
let sidebar = localStorage.getItem('mdbook-sidebar');
|
||
|
|
|
||
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
||
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
||
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
||
|
|
}
|
||
|
|
} catch (e) { }
|
||
|
|
</script>
|
||
|
|
|
||
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
||
|
|
<script>
|
||
|
|
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
||
|
|
let theme;
|
||
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
||
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
||
|
|
const html = document.documentElement;
|
||
|
|
html.classList.remove('light')
|
||
|
|
html.classList.add(theme);
|
||
|
|
html.classList.add("js");
|
||
|
|
</script>
|
||
|
|
|
||
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
||
|
|
|
||
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
||
|
|
<script>
|
||
|
|
let sidebar = null;
|
||
|
|
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
||
|
|
if (document.body.clientWidth >= 1080) {
|
||
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
||
|
|
sidebar = sidebar || 'visible';
|
||
|
|
} else {
|
||
|
|
sidebar = 'hidden';
|
||
|
|
}
|
||
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
||
|
|
html.classList.remove('sidebar-visible');
|
||
|
|
html.classList.add("sidebar-" + sidebar);
|
||
|
|
</script>
|
||
|
|
|
||
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
||
|
|
<!-- populated by js -->
|
||
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
||
|
|
<noscript>
|
||
|
|
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
|
||
|
|
</noscript>
|
||
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
||
|
|
<div class="sidebar-resize-indicator"></div>
|
||
|
|
</div>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
<div id="page-wrapper" class="page-wrapper">
|
||
|
|
|
||
|
|
<div class="page">
|
||
|
|
<div id="menu-bar-hover-placeholder"></div>
|
||
|
|
<div id="menu-bar" class="menu-bar sticky">
|
||
|
|
<div class="left-buttons">
|
||
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
||
|
|
<i class="fa fa-bars"></i>
|
||
|
|
</label>
|
||
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
||
|
|
<i class="fa fa-paint-brush"></i>
|
||
|
|
</button>
|
||
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
||
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
||
|
|
</ul>
|
||
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
|
||
|
|
<i class="fa fa-search"></i>
|
||
|
|
</button>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<h1 class="menu-title">VAPORA Platform Documentation</h1>
|
||
|
|
|
||
|
|
<div class="right-buttons">
|
||
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
||
|
|
<i id="print-button" class="fa fa-print"></i>
|
||
|
|
</a>
|
||
|
|
<a href="https://github.com/vapora-platform/vapora" title="Git repository" aria-label="Git repository">
|
||
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
||
|
|
</a>
|
||
|
|
<a href="https://github.com/vapora-platform/vapora/edit/main/docs/src/../operations/backup-recovery-automation.md" title="Suggest an edit" aria-label="Suggest an edit">
|
||
|
|
<i id="git-edit-button" class="fa fa-edit"></i>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<div id="search-wrapper" class="hidden">
|
||
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
||
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
||
|
|
</form>
|
||
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
||
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
||
|
|
<ul id="searchresults">
|
||
|
|
</ul>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
||
|
|
<script>
|
||
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
||
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
||
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
||
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
||
|
|
});
|
||
|
|
</script>
|
||
|
|
|
||
|
|
<div id="content" class="content">
|
||
|
|
<main>
|
||
|
|
<h1 id="vapora-automated-backup--recovery-automation"><a class="header" href="#vapora-automated-backup--recovery-automation">VAPORA Automated Backup & Recovery Automation</a></h1>
|
||
|
|
<p>Automated backup and recovery procedures using Nushell scripts and Kubernetes CronJobs. Supports both direct S3 backups and Restic-based incremental backups.</p>
|
||
|
|
<hr />
|
||
|
|
<h2 id="overview"><a class="header" href="#overview">Overview</a></h2>
|
||
|
|
<p><strong>Backup Strategy</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>Hourly: Database export + Restic backup (1-hour RPO)</li>
|
||
|
|
<li>Daily: Kubernetes config backup + Restic backup</li>
|
||
|
|
<li>Monthly: Cleanup old snapshots and archive</li>
|
||
|
|
</ul>
|
||
|
|
<p><strong>Dual Backup Approach</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li><strong>S3 Direct</strong>: Simple file upload for quick recovery</li>
|
||
|
|
<li><strong>Restic</strong>: Incremental, deduplicated backups with integrated encryption</li>
|
||
|
|
</ul>
|
||
|
|
<p><strong>Recovery Procedures</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>One-command restore from S3 or Restic</li>
|
||
|
|
<li>Verification before committing to production</li>
|
||
|
|
<li>Automated database readiness checks</li>
|
||
|
|
</ul>
|
||
|
|
<hr />
|
||
|
|
<h2 id="files-and-components"><a class="header" href="#files-and-components">Files and Components</a></h2>
|
||
|
|
<h3 id="backup-scripts"><a class="header" href="#backup-scripts">Backup Scripts</a></h3>
|
||
|
|
<p>All scripts follow NUSHELL_GUIDELINES.md (0.109.0+) strictly.</p>
|
||
|
|
<h4 id="scriptsbackupdatabase-backupnu"><a class="header" href="#scriptsbackupdatabase-backupnu"><code>scripts/backup/database-backup.nu</code></a></h4>
|
||
|
|
<p>Direct S3 backup of SurrealDB with encryption.</p>
|
||
|
|
<pre><code class="language-bash">nu scripts/backup/database-backup.nu \
|
||
|
|
--surreal-url "ws://localhost:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS" \
|
||
|
|
--s3-bucket "vapora-backups" \
|
||
|
|
--s3-prefix "backups/database" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Process</strong>:</p>
|
||
|
|
<ol>
|
||
|
|
<li>Export SurrealDB to SQL</li>
|
||
|
|
<li>Compress with gzip</li>
|
||
|
|
<li>Encrypt with AES-256</li>
|
||
|
|
<li>Upload to S3 with metadata</li>
|
||
|
|
<li>Verify upload completed</li>
|
||
|
|
</ol>
|
||
|
|
<p><strong>Output</strong>: <code>s3://vapora-backups/backups/database/database-YYYYMMDD-HHMMSS.sql.gz.enc</code></p>
|
||
|
|
<h4 id="scriptsbackupconfig-backupnu"><a class="header" href="#scriptsbackupconfig-backupnu"><code>scripts/backup/config-backup.nu</code></a></h4>
|
||
|
|
<p>Backup Kubernetes resources (ConfigMaps, Secrets, Deployments).</p>
|
||
|
|
<pre><code class="language-bash">nu scripts/backup/config-backup.nu \
|
||
|
|
--namespace "vapora" \
|
||
|
|
--s3-bucket "vapora-backups" \
|
||
|
|
--s3-prefix "backups/config"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Process</strong>:</p>
|
||
|
|
<ol>
|
||
|
|
<li>Export ConfigMaps from namespace</li>
|
||
|
|
<li>Export Secrets</li>
|
||
|
|
<li>Export Deployments, Services, Ingress</li>
|
||
|
|
<li>Compress all to tar.gz</li>
|
||
|
|
<li>Upload to S3</li>
|
||
|
|
</ol>
|
||
|
|
<p><strong>Output</strong>: <code>s3://vapora-backups/backups/config/configs-YYYYMMDD-HHMMSS.tar.gz</code></p>
|
||
|
|
<h4 id="scriptsbackuprestic-backupnu"><a class="header" href="#scriptsbackuprestic-backupnu"><code>scripts/backup/restic-backup.nu</code></a></h4>
|
||
|
|
<p>Incremental, deduplicated backup using Restic.</p>
|
||
|
|
<pre><code class="language-bash">nu scripts/backup/restic-backup.nu \
|
||
|
|
--repo "s3:s3.amazonaws.com/vapora-backups/restic" \
|
||
|
|
--password "$RESTIC_PASSWORD" \
|
||
|
|
--database-dir "/tmp/vapora-db-backup" \
|
||
|
|
--k8s-dir "/tmp/vapora-k8s-backup" \
|
||
|
|
--iac-dir "provisioning" \
|
||
|
|
--backup-db \
|
||
|
|
--backup-k8s \
|
||
|
|
--backup-iac \
|
||
|
|
--verify \
|
||
|
|
--cleanup \
|
||
|
|
--keep-daily 7 \
|
||
|
|
--keep-weekly 4 \
|
||
|
|
--keep-monthly 12
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Features</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>Incremental backups (only changed data stored)</li>
|
||
|
|
<li>Deduplication across snapshots</li>
|
||
|
|
<li>Built-in compression and encryption</li>
|
||
|
|
<li>Automatic retention policies</li>
|
||
|
|
<li>Repository health verification</li>
|
||
|
|
</ul>
|
||
|
|
<p><strong>Output</strong>: Tagged snapshots in Restic repository with metadata</p>
|
||
|
|
<h4 id="scriptsorchestrate-backup-recoverynu"><a class="header" href="#scriptsorchestrate-backup-recoverynu"><code>scripts/orchestrate-backup-recovery.nu</code></a></h4>
|
||
|
|
<p>Coordinates all backup types (S3 + Restic).</p>
|
||
|
|
<pre><code class="language-bash"># Full backup cycle
|
||
|
|
nu scripts/orchestrate-backup-recovery.nu \
|
||
|
|
--operation backup \
|
||
|
|
--mode full \
|
||
|
|
--surreal-url "ws://localhost:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS" \
|
||
|
|
--namespace "vapora" \
|
||
|
|
--s3-bucket "vapora-backups" \
|
||
|
|
--s3-prefix "backups/database" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE" \
|
||
|
|
--restic-repo "s3:s3.amazonaws.com/vapora-backups/restic" \
|
||
|
|
--restic-password "$RESTIC_PASSWORD" \
|
||
|
|
--iac-dir "provisioning"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Modes</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li><code>full</code>: Database export → S3 + Restic</li>
|
||
|
|
<li><code>database-only</code>: Database export only</li>
|
||
|
|
<li><code>config-only</code>: Kubernetes config only</li>
|
||
|
|
</ul>
|
||
|
|
<h3 id="recovery-scripts"><a class="header" href="#recovery-scripts">Recovery Scripts</a></h3>
|
||
|
|
<h4 id="scriptsrecoverydatabase-recoverynu"><a class="header" href="#scriptsrecoverydatabase-recoverynu"><code>scripts/recovery/database-recovery.nu</code></a></h4>
|
||
|
|
<p>Restore SurrealDB from S3 backup (with decryption).</p>
|
||
|
|
<pre><code class="language-bash">nu scripts/recovery/database-recovery.nu \
|
||
|
|
--s3-location "s3://vapora-backups/backups/database/database-20260112-010000.sql.gz.enc" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE" \
|
||
|
|
--surreal-url "ws://localhost:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS" \
|
||
|
|
--namespace "vapora" \
|
||
|
|
--statefulset "surrealdb" \
|
||
|
|
--pvc "surrealdb-data-surrealdb-0" \
|
||
|
|
--verify
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Process</strong>:</p>
|
||
|
|
<ol>
|
||
|
|
<li>Download encrypted backup from S3</li>
|
||
|
|
<li>Decrypt backup file</li>
|
||
|
|
<li>Decompress backup</li>
|
||
|
|
<li>Scale down StatefulSet (for PVC replacement)</li>
|
||
|
|
<li>Delete current PVC</li>
|
||
|
|
<li>Scale up StatefulSet (creates new PVC)</li>
|
||
|
|
<li>Wait for pod readiness</li>
|
||
|
|
<li>Import backup to database</li>
|
||
|
|
<li>Verify data integrity</li>
|
||
|
|
</ol>
|
||
|
|
<p><strong>Output</strong>: Restored database at specified SurrealDB URL</p>
|
||
|
|
<h4 id="scriptsorchestrate-backup-recoverynu-recovery-mode"><a class="header" href="#scriptsorchestrate-backup-recoverynu-recovery-mode"><code>scripts/orchestrate-backup-recovery.nu</code> (Recovery Mode)</a></h4>
|
||
|
|
<p>One-command recovery from backup.</p>
|
||
|
|
<pre><code class="language-bash">nu scripts/orchestrate-backup-recovery.nu \
|
||
|
|
--operation recovery \
|
||
|
|
--s3-location "s3://vapora-backups/backups/database/database-20260112-010000.sql.gz.enc" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE" \
|
||
|
|
--surreal-url "ws://localhost:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS"
|
||
|
|
</code></pre>
|
||
|
|
<h3 id="verification-scripts"><a class="header" href="#verification-scripts">Verification Scripts</a></h3>
|
||
|
|
<h4 id="scriptsverify-backup-healthnu"><a class="header" href="#scriptsverify-backup-healthnu"><code>scripts/verify-backup-health.nu</code></a></h4>
|
||
|
|
<p>Health check for backup infrastructure.</p>
|
||
|
|
<pre><code class="language-bash"># Basic health check
|
||
|
|
nu scripts/verify-backup-health.nu \
|
||
|
|
--s3-bucket "vapora-backups" \
|
||
|
|
--s3-prefix "backups/database" \
|
||
|
|
--restic-repo "s3:s3.amazonaws.com/vapora-backups/restic" \
|
||
|
|
--restic-password "$RESTIC_PASSWORD" \
|
||
|
|
--surreal-url "ws://localhost:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS" \
|
||
|
|
--max-age-hours 25
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Checks Performed</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>✓ S3 backups exist and have content</li>
|
||
|
|
<li>✓ Restic repository accessible and has snapshots</li>
|
||
|
|
<li>✓ Database connectivity verified</li>
|
||
|
|
<li>✓ Backup freshness (< 25 hours old)</li>
|
||
|
|
<li>✓ Backup rotation policy (daily, weekly, monthly)</li>
|
||
|
|
<li>✓ Restore test (if <code>--full-test</code> specified)</li>
|
||
|
|
</ul>
|
||
|
|
<p><strong>Output</strong>: Pass/fail for each check with detailed status</p>
|
||
|
|
<hr />
|
||
|
|
<h2 id="kubernetes-automation"><a class="header" href="#kubernetes-automation">Kubernetes Automation</a></h2>
|
||
|
|
<h3 id="cronjob-configuration"><a class="header" href="#cronjob-configuration">CronJob Configuration</a></h3>
|
||
|
|
<p>File: <code>kubernetes/09-backup-cronjobs.yaml</code></p>
|
||
|
|
<p>Defines four automated CronJobs:</p>
|
||
|
|
<h4 id="1-hourly-database-backup"><a class="header" href="#1-hourly-database-backup">1. Hourly Database Backup</a></h4>
|
||
|
|
<pre><code class="language-yaml">schedule: "0 * * * *" # Every hour
|
||
|
|
timeout: 1800 seconds # 30 minutes
|
||
|
|
</code></pre>
|
||
|
|
<p>Runs <code>orchestrate-backup-recovery.nu --operation backup --mode full</code></p>
|
||
|
|
<p><strong>Backups</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>SurrealDB to S3 (encrypted)</li>
|
||
|
|
<li>SurrealDB to Restic (incremental)</li>
|
||
|
|
<li>IaC to Restic</li>
|
||
|
|
</ul>
|
||
|
|
<h4 id="2-daily-configuration-backup"><a class="header" href="#2-daily-configuration-backup">2. Daily Configuration Backup</a></h4>
|
||
|
|
<pre><code class="language-yaml">schedule: "0 2 * * *" # 02:00 UTC daily
|
||
|
|
timeout: 3600 seconds # 60 minutes
|
||
|
|
</code></pre>
|
||
|
|
<p>Runs <code>config-backup.nu</code> for Kubernetes resources.</p>
|
||
|
|
<h4 id="3-daily-health-verification"><a class="header" href="#3-daily-health-verification">3. Daily Health Verification</a></h4>
|
||
|
|
<pre><code class="language-yaml">schedule: "0 3 * * *" # 03:00 UTC daily
|
||
|
|
timeout: 900 seconds # 15 minutes
|
||
|
|
</code></pre>
|
||
|
|
<p>Runs <code>verify-backup-health.nu</code> to verify backup infrastructure.</p>
|
||
|
|
<p><strong>Alerts if</strong>:</p>
|
||
|
|
<ul>
|
||
|
|
<li>No S3 backups found</li>
|
||
|
|
<li>Restic repository inaccessible</li>
|
||
|
|
<li>Database unreachable</li>
|
||
|
|
<li>Backups older than 25 hours</li>
|
||
|
|
<li>Rotation policy violated</li>
|
||
|
|
</ul>
|
||
|
|
<h4 id="4-monthly-backup-rotation"><a class="header" href="#4-monthly-backup-rotation">4. Monthly Backup Rotation</a></h4>
|
||
|
|
<pre><code class="language-yaml">schedule: "0 4 1 * *" # First day of month, 04:00 UTC
|
||
|
|
timeout: 3600 seconds
|
||
|
|
</code></pre>
|
||
|
|
<p>Cleans up old Restic snapshots per retention policy:</p>
|
||
|
|
<ul>
|
||
|
|
<li>Keep: 7 daily, 4 weekly, 12 monthly</li>
|
||
|
|
<li>Prune: Remove unreferenced data</li>
|
||
|
|
</ul>
|
||
|
|
<h3 id="environment-configuration"><a class="header" href="#environment-configuration">Environment Configuration</a></h3>
|
||
|
|
<p>CronJobs require these secrets and ConfigMaps:</p>
|
||
|
|
<p><strong>ConfigMap: <code>vapora-config</code></strong></p>
|
||
|
|
<pre><code class="language-yaml">backup_s3_bucket: "vapora-backups"
|
||
|
|
restic_repo: "s3:s3.amazonaws.com/vapora-backups/restic"
|
||
|
|
aws_region: "us-east-1"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Secret: <code>vapora-secrets</code></strong></p>
|
||
|
|
<pre><code class="language-yaml">surreal_password: "<database-password>"
|
||
|
|
restic_password: "<restic-encryption-password>"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Secret: <code>vapora-aws-credentials</code></strong></p>
|
||
|
|
<pre><code class="language-yaml">access_key_id: "<aws-access-key>"
|
||
|
|
secret_access_key: "<aws-secret-key>"
|
||
|
|
</code></pre>
|
||
|
|
<p><strong>Secret: <code>vapora-encryption-key</code></strong></p>
|
||
|
|
<pre><code class="language-yaml"># File containing AES-256 encryption key
|
||
|
|
encryption.key: "<binary-key-data>"
|
||
|
|
</code></pre>
|
||
|
|
<h3 id="deployment"><a class="header" href="#deployment">Deployment</a></h3>
|
||
|
|
<ol>
|
||
|
|
<li><strong>Create secrets</strong> (if not existing):</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">kubectl create secret generic vapora-secrets \
|
||
|
|
--from-literal=surreal_password="$SURREAL_PASS" \
|
||
|
|
--from-literal=restic_password="$RESTIC_PASSWORD" \
|
||
|
|
-n vapora
|
||
|
|
|
||
|
|
kubectl create secret generic vapora-aws-credentials \
|
||
|
|
--from-literal=access_key_id="$AWS_ACCESS_KEY_ID" \
|
||
|
|
--from-literal=secret_access_key="$AWS_SECRET_ACCESS_KEY" \
|
||
|
|
-n vapora
|
||
|
|
|
||
|
|
kubectl create secret generic vapora-encryption-key \
|
||
|
|
--from-file=encryption.key=/path/to/encryption.key \
|
||
|
|
-n vapora
|
||
|
|
</code></pre>
|
||
|
|
<ol start="2">
|
||
|
|
<li><strong>Deploy CronJobs</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">kubectl apply -f kubernetes/09-backup-cronjobs.yaml
|
||
|
|
</code></pre>
|
||
|
|
<ol start="3">
|
||
|
|
<li><strong>Verify CronJobs</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">kubectl get cronjobs -n vapora
|
||
|
|
kubectl describe cronjob vapora-backup-database-hourly -n vapora
|
||
|
|
</code></pre>
|
||
|
|
<ol start="4">
|
||
|
|
<li><strong>Monitor scheduled runs</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash"># Watch CronJob executions
|
||
|
|
kubectl get jobs -n vapora -l job-type=backup --watch
|
||
|
|
|
||
|
|
# View logs from backup job
|
||
|
|
kubectl logs -n vapora -l backup-type=database --tail=100 -f
|
||
|
|
</code></pre>
|
||
|
|
<hr />
|
||
|
|
<h2 id="setup-instructions"><a class="header" href="#setup-instructions">Setup Instructions</a></h2>
|
||
|
|
<h3 id="prerequisites"><a class="header" href="#prerequisites">Prerequisites</a></h3>
|
||
|
|
<ul>
|
||
|
|
<li>Kubernetes 1.18+ with CronJob support</li>
|
||
|
|
<li>Nushell 0.109.0+</li>
|
||
|
|
<li>AWS CLI v2+</li>
|
||
|
|
<li>Restic installed (or container image with restic)</li>
|
||
|
|
<li>SurrealDB CLI (<code>surreal</code> command)</li>
|
||
|
|
<li><code>kubectl</code> with cluster access</li>
|
||
|
|
</ul>
|
||
|
|
<h3 id="local-testing"><a class="header" href="#local-testing">Local Testing</a></h3>
|
||
|
|
<ol>
|
||
|
|
<li><strong>Setup environment variables</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">export SURREAL_URL="ws://localhost:8000"
|
||
|
|
export SURREAL_USER="root"
|
||
|
|
export SURREAL_PASS="password"
|
||
|
|
export S3_BUCKET="vapora-backups"
|
||
|
|
export ENCRYPTION_KEY_FILE="/path/to/encryption.key"
|
||
|
|
export RESTIC_REPO="s3:s3.amazonaws.com/vapora-backups/restic"
|
||
|
|
export RESTIC_PASSWORD="restic-password"
|
||
|
|
export AWS_REGION="us-east-1"
|
||
|
|
export AWS_ACCESS_KEY_ID="your-key"
|
||
|
|
export AWS_SECRET_ACCESS_KEY="your-secret"
|
||
|
|
</code></pre>
|
||
|
|
<ol start="2">
|
||
|
|
<li><strong>Run backup</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">nu scripts/orchestrate-backup-recovery.nu \
|
||
|
|
--operation backup \
|
||
|
|
--mode full \
|
||
|
|
--surreal-url "$SURREAL_URL" \
|
||
|
|
--surreal-user "$SURREAL_USER" \
|
||
|
|
--surreal-pass "$SURREAL_PASS" \
|
||
|
|
--s3-bucket "$S3_BUCKET" \
|
||
|
|
--s3-prefix "backups/database" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE" \
|
||
|
|
--restic-repo "$RESTIC_REPO" \
|
||
|
|
--restic-password "$RESTIC_PASSWORD" \
|
||
|
|
--iac-dir "provisioning"
|
||
|
|
</code></pre>
|
||
|
|
<ol start="3">
|
||
|
|
<li><strong>Verify backup</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">nu scripts/verify-backup-health.nu \
|
||
|
|
--s3-bucket "$S3_BUCKET" \
|
||
|
|
--s3-prefix "backups/database" \
|
||
|
|
--restic-repo "$RESTIC_REPO" \
|
||
|
|
--restic-password "$RESTIC_PASSWORD" \
|
||
|
|
--surreal-url "$SURREAL_URL" \
|
||
|
|
--surreal-user "$SURREAL_USER" \
|
||
|
|
--surreal-pass "$SURREAL_PASS"
|
||
|
|
</code></pre>
|
||
|
|
<ol start="4">
|
||
|
|
<li><strong>Test recovery</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash"># First, list available backups
|
||
|
|
aws s3 ls s3://$S3_BUCKET/backups/database/
|
||
|
|
|
||
|
|
# Then recover from latest backup
|
||
|
|
nu scripts/orchestrate-backup-recovery.nu \
|
||
|
|
--operation recovery \
|
||
|
|
--s3-location "s3://$S3_BUCKET/backups/database/database-20260112-010000.sql.gz.enc" \
|
||
|
|
--encryption-key "$ENCRYPTION_KEY_FILE" \
|
||
|
|
--surreal-url "$SURREAL_URL" \
|
||
|
|
--surreal-user "$SURREAL_USER" \
|
||
|
|
--surreal-pass "$SURREAL_PASS"
|
||
|
|
</code></pre>
|
||
|
|
<h3 id="production-deployment"><a class="header" href="#production-deployment">Production Deployment</a></h3>
|
||
|
|
<ol>
|
||
|
|
<li><strong>Create S3 bucket</strong> for backups:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">aws s3 mb s3://vapora-backups --region us-east-1
|
||
|
|
</code></pre>
|
||
|
|
<ol start="2">
|
||
|
|
<li><strong>Enable bucket versioning</strong> for protection:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">aws s3api put-bucket-versioning \
|
||
|
|
--bucket vapora-backups \
|
||
|
|
--versioning-configuration Status=Enabled
|
||
|
|
</code></pre>
|
||
|
|
<ol start="3">
|
||
|
|
<li><strong>Set lifecycle policy</strong> for Glacier archival (optional):</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash"># 30 days to standard-IA, 90 days to Glacier
|
||
|
|
aws s3api put-bucket-lifecycle-configuration \
|
||
|
|
--bucket vapora-backups \
|
||
|
|
--lifecycle-configuration file://s3-lifecycle-policy.json
|
||
|
|
</code></pre>
|
||
|
|
<ol start="4">
|
||
|
|
<li><strong>Create Restic repository</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">export RESTIC_REPO="s3:s3.amazonaws.com/vapora-backups/restic"
|
||
|
|
export RESTIC_PASSWORD="your-restic-password"
|
||
|
|
|
||
|
|
restic init
|
||
|
|
</code></pre>
|
||
|
|
<ol start="5">
|
||
|
|
<li><strong>Deploy to Kubernetes</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash"># 1. Create namespace
|
||
|
|
kubectl create namespace vapora
|
||
|
|
|
||
|
|
# 2. Create secrets
|
||
|
|
kubectl create secret generic vapora-secrets \
|
||
|
|
--from-literal=surreal_password="$SURREAL_PASS" \
|
||
|
|
--from-literal=restic_password="$RESTIC_PASSWORD" \
|
||
|
|
-n vapora
|
||
|
|
|
||
|
|
# 3. Create ConfigMap
|
||
|
|
kubectl create configmap vapora-config \
|
||
|
|
--from-literal=backup_s3_bucket="vapora-backups" \
|
||
|
|
--from-literal=restic_repo="s3:s3.amazonaws.com/vapora-backups/restic" \
|
||
|
|
--from-literal=aws_region="us-east-1" \
|
||
|
|
-n vapora
|
||
|
|
|
||
|
|
# 4. Deploy CronJobs
|
||
|
|
kubectl apply -f kubernetes/09-backup-cronjobs.yaml
|
||
|
|
</code></pre>
|
||
|
|
<ol start="6">
|
||
|
|
<li><strong>Monitor</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash"># Watch CronJobs
|
||
|
|
kubectl get cronjobs -n vapora --watch
|
||
|
|
|
||
|
|
# View backup logs
|
||
|
|
kubectl logs -n vapora -l backup-type=database -f
|
||
|
|
|
||
|
|
# Check health status
|
||
|
|
kubectl get jobs -n vapora -l job-type=health-check -o wide
|
||
|
|
</code></pre>
|
||
|
|
<hr />
|
||
|
|
<h2 id="emergency-recovery"><a class="header" href="#emergency-recovery">Emergency Recovery</a></h2>
|
||
|
|
<h3 id="complete-database-loss"><a class="header" href="#complete-database-loss">Complete Database Loss</a></h3>
|
||
|
|
<p>If production database is lost, restore from backup:</p>
|
||
|
|
<pre><code class="language-bash"># 1. Scale down StatefulSet
|
||
|
|
kubectl scale statefulset surrealdb --replicas=0 -n vapora
|
||
|
|
|
||
|
|
# 2. Delete current PVC
|
||
|
|
kubectl delete pvc surrealdb-data-surrealdb-0 -n vapora
|
||
|
|
|
||
|
|
# 3. Run recovery
|
||
|
|
nu scripts/orchestrate-backup-recovery.nu \
|
||
|
|
--operation recovery \
|
||
|
|
--s3-location "s3://vapora-backups/backups/database/database-LATEST.sql.gz.enc" \
|
||
|
|
--encryption-key "/path/to/encryption.key" \
|
||
|
|
--surreal-url "ws://surrealdb:8000" \
|
||
|
|
--surreal-user "root" \
|
||
|
|
--surreal-pass "$SURREAL_PASS"
|
||
|
|
|
||
|
|
# 4. Verify database restored
|
||
|
|
kubectl exec -n vapora surrealdb-0 -- \
|
||
|
|
surreal query \
|
||
|
|
--conn ws://localhost:8000 \
|
||
|
|
--user root \
|
||
|
|
--pass "$SURREAL_PASS" \
|
||
|
|
"SELECT COUNT() FROM projects"
|
||
|
|
</code></pre>
|
||
|
|
<h3 id="backup-verification-failed"><a class="header" href="#backup-verification-failed">Backup Verification Failed</a></h3>
|
||
|
|
<p>If health check fails:</p>
|
||
|
|
<ol>
|
||
|
|
<li><strong>Check Restic repository</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">export RESTIC_PASSWORD="$RESTIC_PASSWORD"
|
||
|
|
restic -r "s3:s3.amazonaws.com/vapora-backups/restic" check
|
||
|
|
</code></pre>
|
||
|
|
<ol start="2">
|
||
|
|
<li><strong>Force full verification</strong> (slow):</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">restic -r "s3:s3.amazonaws.com/vapora-backups/restic" check --read-data
|
||
|
|
</code></pre>
|
||
|
|
<ol start="3">
|
||
|
|
<li><strong>List recent snapshots</strong>:</li>
|
||
|
|
</ol>
|
||
|
|
<pre><code class="language-bash">restic -r "s3:s3.amazonaws.com/vapora-backups/restic" snapshots --max 10
|
||
|
|
</code></pre>
|
||
|
|
<hr />
|
||
|
|
<h2 id="troubleshooting"><a class="header" href="#troubleshooting">Troubleshooting</a></h2>
|
||
|
|
<div class="table-wrapper"><table><thead><tr><th>Issue</th><th>Cause</th><th>Solution</th></tr></thead><tbody>
|
||
|
|
<tr><td><strong>CronJob not running</strong></td><td>Schedule incorrect</td><td>Check <code>kubectl get cronjobs</code> and verify schedule format</td></tr>
|
||
|
|
<tr><td><strong>Backup file too large</strong></td><td>Database growing</td><td>Check for old data that can be cleaned up</td></tr>
|
||
|
|
<tr><td><strong>S3 upload fails</strong></td><td>Credentials invalid</td><td>Verify <code>AWS_ACCESS_KEY_ID</code>, <code>AWS_SECRET_ACCESS_KEY</code></td></tr>
|
||
|
|
<tr><td><strong>Restic backup slow</strong></td><td>First backup or network latency</td><td>Expected on first run; use <code>--keep-*</code> flags to limit retention</td></tr>
|
||
|
|
<tr><td><strong>Recovery fails</strong></td><td>Database already running</td><td>Scale down StatefulSet before recovery</td></tr>
|
||
|
|
<tr><td><strong>Encryption key missing</strong></td><td>Secret not created</td><td>Create <code>vapora-encryption-key</code> secret in namespace</td></tr>
|
||
|
|
</tbody></table>
|
||
|
|
</div>
|
||
|
|
<hr />
|
||
|
|
<h2 id="related-documentation"><a class="header" href="#related-documentation">Related Documentation</a></h2>
|
||
|
|
<ul>
|
||
|
|
<li><strong>Disaster Recovery Procedures</strong>: <code>docs/disaster-recovery/README.md</code></li>
|
||
|
|
<li><strong>Backup Strategy</strong>: <code>docs/disaster-recovery/backup-strategy.md</code></li>
|
||
|
|
<li><strong>Database Recovery</strong>: <code>docs/disaster-recovery/database-recovery-procedures.md</code></li>
|
||
|
|
<li><strong>Operations Guide</strong>: <code>docs/operations/README.md</code></li>
|
||
|
|
</ul>
|
||
|
|
<hr />
|
||
|
|
<p><strong>Last Updated</strong>: January 12, 2026
|
||
|
|
<strong>Status</strong>: Production-Ready
|
||
|
|
<strong>Automation</strong>: Full CronJob automation with health checks</p>
|
||
|
|
|
||
|
|
</main>
|
||
|
|
|
||
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
||
|
|
<!-- Mobile navigation buttons -->
|
||
|
|
<a rel="prev" href="../../operations/rollback-runbook.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
|
|
<i class="fa fa-angle-left"></i>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
<a rel="next prefetch" href="../../disaster-recovery/index.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
|
|
<i class="fa fa-angle-right"></i>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
<div style="clear: both"></div>
|
||
|
|
</nav>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
||
|
|
<a rel="prev" href="../../operations/rollback-runbook.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
|
|
<i class="fa fa-angle-left"></i>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
<a rel="next prefetch" href="../../disaster-recovery/index.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
|
|
<i class="fa fa-angle-right"></i>
|
||
|
|
</a>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<script>
|
||
|
|
window.playground_copyable = true;
|
||
|
|
</script>
|
||
|
|
|
||
|
|
|
||
|
|
<script src="../elasticlunr.min.js"></script>
|
||
|
|
<script src="../mark.min.js"></script>
|
||
|
|
<script src="../searcher.js"></script>
|
||
|
|
|
||
|
|
<script src="../clipboard.min.js"></script>
|
||
|
|
<script src="../highlight.js"></script>
|
||
|
|
<script src="../book.js"></script>
|
||
|
|
|
||
|
|
<!-- Custom JS scripts -->
|
||
|
|
|
||
|
|
|
||
|
|
</div>
|
||
|
|
</body>
|
||
|
|
</html>
|