137 lines
3.9 KiB
Rust
137 lines
3.9 KiB
Rust
use anyhow::{Context, Result};
|
|
use std::path::Path;
|
|
use tokio::process::Command;
|
|
use tracing::{debug, info};
|
|
|
|
pub const OOM_EXIT_CODE: i32 = 137;
|
|
|
|
pub struct BuildOutput {
|
|
pub cpu_secs: f64,
|
|
pub mem_peak_mb: f64,
|
|
pub duration_secs: f64,
|
|
}
|
|
|
|
pub async fn rsync_context(
|
|
ssh_host: &str,
|
|
ssh_port: u16,
|
|
context_path: &Path,
|
|
ssh_key: &Path,
|
|
) -> Result<()> {
|
|
info!(host = %ssh_host, port = ssh_port, "rsyncing build context");
|
|
|
|
let status = Command::new("rsync")
|
|
.args([
|
|
"-az",
|
|
"--delete",
|
|
"-e",
|
|
&format!(
|
|
"ssh -p {} -i {} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null",
|
|
ssh_port,
|
|
ssh_key.display()
|
|
),
|
|
])
|
|
.arg(format!("{}/", context_path.display()))
|
|
.arg(format!("root@{}:/build/context/", ssh_host))
|
|
.status()
|
|
.await
|
|
.context("rsync failed to start")?;
|
|
|
|
if !status.success() {
|
|
anyhow::bail!("rsync exited {}", status);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn run_buildctl(
|
|
ssh_host: &str,
|
|
ssh_port: u16,
|
|
ssh_key: &Path,
|
|
image_ref: &str,
|
|
dockerfile: &str,
|
|
cache_from: Option<&str>,
|
|
cache_to: Option<&str>,
|
|
) -> Result<BuildOutput> {
|
|
let mut buildctl_args = vec![
|
|
"build".to_string(),
|
|
"--frontend".to_string(),
|
|
"dockerfile.v0".to_string(),
|
|
"--opt".to_string(),
|
|
format!("filename={}", dockerfile),
|
|
"--local".to_string(),
|
|
"context=/build/context".to_string(),
|
|
"--local".to_string(),
|
|
"dockerfile=/build/context".to_string(),
|
|
"--output".to_string(),
|
|
format!("type=image,name={},push=true", image_ref),
|
|
];
|
|
|
|
if let Some(from) = cache_from {
|
|
buildctl_args.push("--import-cache".to_string());
|
|
buildctl_args.push(from.to_string());
|
|
}
|
|
if let Some(to) = cache_to {
|
|
buildctl_args.push("--export-cache".to_string());
|
|
buildctl_args.push(to.to_string());
|
|
}
|
|
|
|
let remote_cmd = format!("buildctl {}", buildctl_args.join(" "));
|
|
debug!(cmd = %remote_cmd, "running buildctl on runner");
|
|
|
|
let start = std::time::Instant::now();
|
|
|
|
let output = Command::new("ssh")
|
|
.args([
|
|
"-p",
|
|
&ssh_port.to_string(),
|
|
"-i",
|
|
&ssh_key.to_string_lossy(),
|
|
"-o",
|
|
"StrictHostKeyChecking=no",
|
|
"-o",
|
|
"UserKnownHostsFile=/dev/null",
|
|
&format!("root@{}", ssh_host),
|
|
&remote_cmd,
|
|
])
|
|
.output()
|
|
.await
|
|
.context("ssh buildctl failed to start")?;
|
|
|
|
let duration_secs = start.elapsed().as_secs_f64();
|
|
|
|
if !output.status.success() {
|
|
let exit_code = output.status.code().unwrap_or(-1);
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
anyhow::bail!(
|
|
"buildctl exit={}: {}",
|
|
exit_code,
|
|
stderr.lines().last().unwrap_or("(no stderr)")
|
|
);
|
|
}
|
|
|
|
// Parse /proc/self/status peak memory from buildkitd if available in stdout
|
|
// Fallback: estimate from image size (not accurate; real metrics come from cgroups)
|
|
let mem_peak_mb = parse_mem_peak_from_output(&String::from_utf8_lossy(&output.stdout))
|
|
.unwrap_or(0.0);
|
|
|
|
// cpu_secs: rough estimate — not from cgroups; orchestrator records real P95 after multiple runs
|
|
let cpu_secs = duration_secs;
|
|
|
|
Ok(BuildOutput { cpu_secs, mem_peak_mb, duration_secs })
|
|
}
|
|
|
|
fn parse_mem_peak_from_output(stdout: &str) -> Option<f64> {
|
|
for line in stdout.lines() {
|
|
if let Some(val) = line.strip_prefix("MEM_PEAK_MB=") {
|
|
return val.parse().ok();
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
pub fn is_oom(err: &anyhow::Error) -> bool {
|
|
let msg = err.to_string();
|
|
msg.contains(&format!("exit={}", OOM_EXIT_CODE))
|
|
|| msg.contains("exit=137")
|
|
|| msg.contains("OOM")
|
|
|| msg.contains("Killed")
|
|
}
|