use anyhow::{Context, Result}; use std::path::Path; use tokio::process::Command; use tracing::{debug, info}; pub const OOM_EXIT_CODE: i32 = 137; pub struct BuildOutput { pub cpu_secs: f64, pub mem_peak_mb: f64, pub duration_secs: f64, } pub async fn rsync_context( ssh_host: &str, ssh_port: u16, context_path: &Path, ssh_key: &Path, ) -> Result<()> { info!(host = %ssh_host, port = ssh_port, "rsyncing build context"); let status = Command::new("rsync") .args([ "-az", "--delete", "-e", &format!( "ssh -p {} -i {} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", ssh_port, ssh_key.display() ), ]) .arg(format!("{}/", context_path.display())) .arg(format!("root@{}:/build/context/", ssh_host)) .status() .await .context("rsync failed to start")?; if !status.success() { anyhow::bail!("rsync exited {}", status); } Ok(()) } pub async fn run_buildctl( ssh_host: &str, ssh_port: u16, ssh_key: &Path, image_ref: &str, dockerfile: &str, cache_from: Option<&str>, cache_to: Option<&str>, ) -> Result { let mut buildctl_args = vec![ "build".to_string(), "--frontend".to_string(), "dockerfile.v0".to_string(), "--opt".to_string(), format!("filename={}", dockerfile), "--local".to_string(), "context=/build/context".to_string(), "--local".to_string(), "dockerfile=/build/context".to_string(), "--output".to_string(), format!("type=image,name={},push=true", image_ref), ]; if let Some(from) = cache_from { buildctl_args.push("--import-cache".to_string()); buildctl_args.push(from.to_string()); } if let Some(to) = cache_to { buildctl_args.push("--export-cache".to_string()); buildctl_args.push(to.to_string()); } let remote_cmd = format!("buildctl {}", buildctl_args.join(" ")); debug!(cmd = %remote_cmd, "running buildctl on runner"); let start = std::time::Instant::now(); let output = Command::new("ssh") .args([ "-p", &ssh_port.to_string(), "-i", &ssh_key.to_string_lossy(), "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", &format!("root@{}", ssh_host), &remote_cmd, ]) .output() .await .context("ssh buildctl failed to start")?; let duration_secs = start.elapsed().as_secs_f64(); if !output.status.success() { let exit_code = output.status.code().unwrap_or(-1); let stderr = String::from_utf8_lossy(&output.stderr); anyhow::bail!( "buildctl exit={}: {}", exit_code, stderr.lines().last().unwrap_or("(no stderr)") ); } // Parse /proc/self/status peak memory from buildkitd if available in stdout // Fallback: estimate from image size (not accurate; real metrics come from cgroups) let mem_peak_mb = parse_mem_peak_from_output(&String::from_utf8_lossy(&output.stdout)) .unwrap_or(0.0); // cpu_secs: rough estimate — not from cgroups; orchestrator records real P95 after multiple runs let cpu_secs = duration_secs; Ok(BuildOutput { cpu_secs, mem_peak_mb, duration_secs }) } fn parse_mem_peak_from_output(stdout: &str) -> Option { for line in stdout.lines() { if let Some(val) = line.strip_prefix("MEM_PEAK_MB=") { return val.parse().ok(); } } None } pub fn is_oom(err: &anyhow::Error) -> bool { let msg = err.to_string(); msg.contains(&format!("exit={}", OOM_EXIT_CODE)) || msg.contains("exit=137") || msg.contains("OOM") || msg.contains("Killed") }