354 lines
9.4 KiB
Text
354 lines
9.4 KiB
Text
# VM Cleanup Scheduler (Phase 2)
|
|
#
|
|
# Manages automatic cleanup of expired temporary VMs.
|
|
# Rule 1: Single purpose, Rule 4: Pure functions, Rule 5: Atomic operations
|
|
|
|
# Selective imports (ADR-025 Phase 3 Layer 2).
|
|
# vm/lifecycle.nu star-import was dead — dropped.
|
|
# Note: line ~211 embeds a string template with `use platform/vm/cleanup_scheduler.nu *`
|
|
# (scheduler script written to disk at runtime) — NOT an actual import.
|
|
use platform/vm/vm_persistence.nu [
|
|
cleanup-expired-vms find-expired-vms get-vm-persistence-info
|
|
get-vm-time-to-cleanup
|
|
]
|
|
|
|
export def "start-cleanup-scheduler" [
|
|
--check-interval-minutes: int = 60
|
|
--background = true
|
|
]: record {
|
|
"""
|
|
Start the VM cleanup scheduler.
|
|
|
|
Periodically checks for and cleans up expired temporary VMs.
|
|
Can run in background or foreground.
|
|
"""
|
|
|
|
if $background {
|
|
# Start in background
|
|
start-scheduler-background $check_interval_minutes
|
|
} else {
|
|
# Run foreground
|
|
run-scheduler-loop $check_interval_minutes
|
|
}
|
|
}
|
|
|
|
def start-scheduler-background [interval_minutes: int]: record {
|
|
"""Start scheduler in background"""
|
|
|
|
let scheduler_file = (get-scheduler-pid-file)
|
|
let scheduler_script = (get-scheduler-script-path)
|
|
|
|
# Create scheduler script
|
|
create-scheduler-script $interval_minutes $scheduler_script
|
|
|
|
# Start in background (no try-catch)
|
|
let start_result = (do { bash -c $"nohup nu ($scheduler_script) > /tmp/vm-cleanup-scheduler.log 2>&1 &" } | complete)
|
|
if $start_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to start scheduler: ($start_result.stderr)"
|
|
}
|
|
}
|
|
|
|
let pid_result = (do { bash -c "echo $!" } | complete)
|
|
if $pid_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to get scheduler PID: ($pid_result.stderr)"
|
|
}
|
|
}
|
|
|
|
let pid = ($pid_result.stdout | str trim)
|
|
|
|
# Save PID (no try-catch)
|
|
let save_pid_result = (do { bash -c $"echo ($pid) > ($scheduler_file)" } | complete)
|
|
if $save_pid_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to save scheduler PID: ($save_pid_result.stderr)"
|
|
}
|
|
}
|
|
|
|
{
|
|
success: true
|
|
pid: $pid
|
|
message: "Cleanup scheduler started in background"
|
|
}
|
|
}
|
|
|
|
export def "stop-cleanup-scheduler" []: record {
|
|
"""Stop the VM cleanup scheduler"""
|
|
|
|
let scheduler_file = (get-scheduler-pid-file)
|
|
|
|
if not ($scheduler_file | path exists) {
|
|
return {
|
|
success: true
|
|
message: "Scheduler not running"
|
|
}
|
|
}
|
|
|
|
# Load scheduler PID (no try-catch)
|
|
let pid_result = (do { open $scheduler_file | str trim } | complete)
|
|
if $pid_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to read scheduler PID: ($pid_result.stderr)"
|
|
}
|
|
}
|
|
|
|
let pid = ($pid_result.stdout)
|
|
|
|
# Kill scheduler process (no try-catch)
|
|
let kill_result = (do { bash -c $"kill ($pid) 2>/dev/null || true" } | complete)
|
|
if $kill_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to kill scheduler: ($kill_result.stderr)"
|
|
}
|
|
}
|
|
|
|
# Remove PID file (no try-catch)
|
|
let rm_result = (do { bash -c $"rm -f ($scheduler_file)" } | complete)
|
|
if $rm_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to remove PID file: ($rm_result.stderr)"
|
|
}
|
|
}
|
|
|
|
{
|
|
success: true
|
|
pid: $pid
|
|
message: "Scheduler stopped"
|
|
}
|
|
}
|
|
|
|
export def "get-cleanup-scheduler-status" []: record {
|
|
"""Get status of cleanup scheduler"""
|
|
|
|
let scheduler_file = (get-scheduler-pid-file)
|
|
let log_file = "/tmp/vm-cleanup-scheduler.log"
|
|
|
|
if not ($scheduler_file | path exists) {
|
|
return {
|
|
running: false
|
|
message: "Scheduler not running"
|
|
}
|
|
}
|
|
|
|
# Load scheduler PID (no try-catch)
|
|
let pid_result = (do { open $scheduler_file | str trim } | complete)
|
|
if $pid_result.exit_code != 0 {
|
|
return {
|
|
running: false
|
|
error: $"Failed to read scheduler PID: ($pid_result.stderr)"
|
|
}
|
|
}
|
|
|
|
let pid = ($pid_result.stdout)
|
|
|
|
# Check if process exists (no try-catch)
|
|
let check_result = (do { bash -c $"kill -0 ($pid) 2>/dev/null && echo 'true' || echo 'false'" } | complete)
|
|
let is_running = (
|
|
if $check_result.exit_code == 0 {
|
|
($check_result.stdout | str trim)
|
|
} else {
|
|
"false"
|
|
}
|
|
)
|
|
|
|
let log_exists = ($log_file | path exists)
|
|
|
|
# Read log file if it exists (no try-catch)
|
|
let last_log_lines = (
|
|
if $log_exists {
|
|
let log_result = (do { bash -c $"tail -5 ($log_file)" } | complete)
|
|
if $log_result.exit_code == 0 {
|
|
($log_result.stdout | lines)
|
|
} else {
|
|
[]
|
|
}
|
|
} else {
|
|
[]
|
|
}
|
|
)
|
|
|
|
{
|
|
running: ($is_running == "true")
|
|
pid: $pid
|
|
log_file: $log_file
|
|
recent_logs: $last_log_lines
|
|
}
|
|
}
|
|
|
|
def run-scheduler-loop [interval_minutes: int] {
|
|
"""Run cleanup scheduler loop"""
|
|
|
|
print "VM Cleanup Scheduler starting..."
|
|
print $"Check interval: ($interval_minutes) minutes"
|
|
print "Press Ctrl+C to stop scheduler"
|
|
|
|
mut iteration = 0
|
|
let max_iterations = 1_000_000 # Safety limit: ~2 years at 1 min intervals
|
|
|
|
while { $iteration < $max_iterations } {
|
|
# Run cleanup
|
|
let result = (cleanup-expired-vms)
|
|
|
|
if $result.cleaned_up > 0 {
|
|
print $"[$(date now)] Cleaned up ($result.cleaned_up) VMs"
|
|
}
|
|
|
|
# Wait for next check
|
|
print $"[$(date now)] Next check in ($interval_minutes) minutes"
|
|
sleep ($interval_minutes)m
|
|
|
|
$iteration += 1
|
|
}
|
|
|
|
print "Scheduler reached iteration limit - exiting"
|
|
}
|
|
|
|
def create-scheduler-script [interval: int, script_path: string] {
|
|
"""Create background scheduler script"""
|
|
|
|
let script_content = $'
|
|
use platform/vm/cleanup_scheduler.nu *
|
|
|
|
mut iteration = 0
|
|
let max_iterations = 1_000_000
|
|
|
|
while \{ $iteration < $max_iterations \} \{
|
|
cleanup-expired-vms
|
|
sleep ($interval)m
|
|
$iteration += 1
|
|
\}
|
|
'
|
|
|
|
bash -c $"cat > ($script_path) << 'SCRIPT'\n($script_content)\nSCRIPT"
|
|
}
|
|
|
|
export def "schedule-vm-cleanup" [
|
|
vm_name: string # VM name to cleanup
|
|
delay_minutes: int = 0 # Minutes to delay before cleanup
|
|
]: record {
|
|
"""Schedule cleanup for a specific VM"""
|
|
|
|
let persist_info = (get-vm-persistence-info $vm_name)
|
|
|
|
if ($persist_info.mode // "") != "temporary" {
|
|
return {
|
|
success: false
|
|
error: "VM is not temporary"
|
|
}
|
|
}
|
|
|
|
let cleanup_time = (
|
|
if $delay_minutes > 0 {
|
|
(date now) + ($delay_minutes)m
|
|
} else {
|
|
date now
|
|
}
|
|
)
|
|
|
|
let updated = (
|
|
$persist_info
|
|
| upsert scheduled_cleanup_at $cleanup_time
|
|
| upsert cleanup_status "pending"
|
|
)
|
|
|
|
let persist_file = (get-persistence-file $vm_name)
|
|
|
|
# Schedule cleanup (no try-catch)
|
|
let schedule_result = (do { bash -c $"cat > ($persist_file) << 'EOF'\n($updated | to json)\nEOF" } | complete)
|
|
if $schedule_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to schedule cleanup: ($schedule_result.stderr)"
|
|
}
|
|
}
|
|
|
|
{
|
|
success: true
|
|
vm_name: $vm_name
|
|
scheduled_cleanup_at: $cleanup_time
|
|
message: $"Cleanup scheduled for ($vm_name)"
|
|
}
|
|
}
|
|
|
|
export def "cancel-vm-cleanup" [
|
|
vm_name: string # VM name
|
|
]: record {
|
|
"""Cancel scheduled cleanup for a VM"""
|
|
|
|
let persist_info = (get-vm-persistence-info $vm_name)
|
|
|
|
if ($persist_info.mode // "") != "temporary" {
|
|
return {
|
|
success: false
|
|
error: "VM is not temporary"
|
|
}
|
|
}
|
|
|
|
# Set cleanup to far future (essentially disabled)
|
|
let far_future = (
|
|
(date now) + 8760h # 1 year from now
|
|
)
|
|
|
|
let updated = (
|
|
$persist_info
|
|
| upsert scheduled_cleanup_at $far_future
|
|
| upsert cleanup_status "cancelled"
|
|
)
|
|
|
|
let persist_file = (get-persistence-file $vm_name)
|
|
|
|
# Cancel cleanup (no try-catch)
|
|
let cancel_result = (do { bash -c $"cat > ($persist_file) << 'EOF'\n($updated | to json)\nEOF" } | complete)
|
|
if $cancel_result.exit_code != 0 {
|
|
return {
|
|
success: false
|
|
error: $"Failed to cancel cleanup: ($cancel_result.stderr)"
|
|
}
|
|
}
|
|
|
|
{
|
|
success: true
|
|
vm_name: $vm_name
|
|
message: "Cleanup cancelled for VM"
|
|
}
|
|
}
|
|
|
|
export def "get-cleanup-queue" []: table {
|
|
"""Get list of VMs pending cleanup"""
|
|
|
|
find-expired-vms
|
|
| map {|vm|
|
|
let time_info = (get-vm-time-to-cleanup $vm.vm_name)
|
|
|
|
{
|
|
vm_name: $vm.vm_name
|
|
created_at: $vm.created_at
|
|
ttl_hours: $vm.ttl_hours
|
|
status: $vm.cleanup_status
|
|
scheduled_cleanup_at: $vm.scheduled_cleanup_at
|
|
time_remaining: $time_info
|
|
}
|
|
}
|
|
}
|
|
|
|
def get-scheduler-pid-file []: string {
|
|
"""Get PID file path for scheduler"""
|
|
"/var/run/vm-cleanup-scheduler.pid"
|
|
}
|
|
|
|
def get-scheduler-script-path []: string {
|
|
"""Get scheduler script path"""
|
|
$"{{paths.workspace}}/vms/.cleanup-scheduler.nu"
|
|
}
|
|
|
|
def get-persistence-file [vm_name: string]: string {
|
|
"""Get persistence file for VM"""
|
|
$"{{paths.workspace}}/vms/persistence/($vm_name).json"
|
|
}
|