1032 lines
37 KiB
Rust
1032 lines
37 KiB
Rust
use serde::{Deserialize, Serialize};
|
|
use std::error::Error;
|
|
use std::fmt;
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
use std::collections::hash_map::DefaultHasher;
|
|
use std::hash::{Hash, Hasher};
|
|
|
|
use sal_os;
|
|
use sal_process;
|
|
use crate::qcow2;
|
|
use crate::cloudhv::net::{NetworkingProfileSpec, DefaultNatOptions};
|
|
|
|
pub mod builder;
|
|
pub mod net;
|
|
|
|
/// Error type for Cloud Hypervisor operations
|
|
#[derive(Debug)]
|
|
pub enum CloudHvError {
|
|
CommandFailed(String),
|
|
IoError(String),
|
|
JsonError(String),
|
|
DependencyMissing(String),
|
|
InvalidSpec(String),
|
|
NotFound(String),
|
|
}
|
|
|
|
impl fmt::Display for CloudHvError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
CloudHvError::CommandFailed(e) => write!(f, "{}", e),
|
|
CloudHvError::IoError(e) => write!(f, "IO error: {}", e),
|
|
CloudHvError::JsonError(e) => write!(f, "JSON error: {}", e),
|
|
CloudHvError::DependencyMissing(e) => write!(f, "Dependency missing: {}", e),
|
|
CloudHvError::InvalidSpec(e) => write!(f, "Invalid spec: {}", e),
|
|
CloudHvError::NotFound(e) => write!(f, "{}", e),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Error for CloudHvError {}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct VmSpec {
|
|
pub id: String,
|
|
/// Optional for firmware boot; required for direct kernel boot
|
|
pub kernel_path: Option<String>,
|
|
/// Optional initramfs when using direct kernel boot
|
|
pub initramfs_path: Option<String>,
|
|
/// Optional for direct kernel boot; required for firmware boot
|
|
pub firmware_path: Option<String>,
|
|
/// Disk image path (qcow2 or raw)
|
|
pub disk_path: String,
|
|
/// API socket path for ch-remote and management
|
|
pub api_socket: String,
|
|
/// vCPUs to boot with
|
|
pub vcpus: u32,
|
|
/// Memory in MB
|
|
pub memory_mb: u32,
|
|
/// Kernel cmdline (only used for direct kernel boot)
|
|
pub cmdline: Option<String>,
|
|
/// Extra args (raw) if you need to extend; keep minimal for Phase 2
|
|
pub extra_args: Option<Vec<String>>,
|
|
/// Optional networking profile; when None, behavior follows explicit --net/--no-default-net or defaults
|
|
#[serde(default)]
|
|
pub net_profile: Option<NetworkingProfileSpec>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct VmRuntime {
|
|
/// PID of cloud-hypervisor process if running
|
|
pub pid: Option<i64>,
|
|
/// Last known status: "stopped" | "running"
|
|
pub status: String,
|
|
/// Console log file path
|
|
pub log_file: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct VmRecord {
|
|
pub spec: VmSpec,
|
|
pub runtime: VmRuntime,
|
|
}
|
|
|
|
fn ensure_deps() -> Result<(), CloudHvError> {
|
|
if sal_process::which("cloud-hypervisor-static").is_none() {
|
|
return Err(CloudHvError::DependencyMissing(
|
|
"cloud-hypervisor-static not found on PATH. Install Cloud Hypervisor static binary.".into(),
|
|
));
|
|
}
|
|
if sal_process::which("ch-remote-static").is_none() {
|
|
return Err(CloudHvError::DependencyMissing(
|
|
"ch-remote-static not found on PATH. Install Cloud Hypervisor tools (static).".into(),
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn hero_vm_root() -> PathBuf {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());
|
|
Path::new(&home).join("hero/virt/vms")
|
|
}
|
|
|
|
fn vm_dir(id: &str) -> PathBuf {
|
|
hero_vm_root().join(id)
|
|
}
|
|
|
|
fn vm_json_path(id: &str) -> PathBuf {
|
|
vm_dir(id).join("vm.json")
|
|
}
|
|
|
|
fn vm_log_path(id: &str) -> PathBuf {
|
|
vm_dir(id).join("logs/console.log")
|
|
}
|
|
|
|
fn vm_pid_path(id: &str) -> PathBuf {
|
|
vm_dir(id).join("pid")
|
|
}
|
|
|
|
fn write_json(path: &Path, value: &serde_json::Value) -> Result<(), CloudHvError> {
|
|
if let Some(parent) = path.parent() {
|
|
fs::create_dir_all(parent).map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
}
|
|
let s = serde_json::to_string_pretty(value).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
fs::write(path, s).map_err(|e| CloudHvError::IoError(e.to_string()))
|
|
}
|
|
|
|
fn read_json(path: &Path) -> Result<serde_json::Value, CloudHvError> {
|
|
let content = fs::read_to_string(path).map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
serde_json::from_str(&content).map_err(|e| CloudHvError::JsonError(e.to_string()))
|
|
}
|
|
|
|
fn proc_exists(pid: i64) -> bool {
|
|
#[cfg(target_os = "linux")]
|
|
{
|
|
Path::new(&format!("/proc/{}", pid)).exists()
|
|
}
|
|
#[cfg(not(target_os = "linux"))]
|
|
{
|
|
// Minimal check for non-Linux; try a kill -0 style command
|
|
let res = sal_process::run(&format!("kill -0 {}", pid)).die(false).silent(true).execute();
|
|
res.map(|r| r.success).unwrap_or(false)
|
|
}
|
|
}
|
|
|
|
/// Create and persist a VM spec
|
|
pub fn vm_create(spec: &VmSpec) -> Result<String, CloudHvError> {
|
|
// Validate inputs minimally
|
|
if spec.id.trim().is_empty() {
|
|
return Err(CloudHvError::InvalidSpec("spec.id must not be empty".into()));
|
|
}
|
|
// Validate boot method: either firmware_path exists or kernel_path exists
|
|
let has_fw = spec
|
|
.firmware_path
|
|
.as_ref()
|
|
.map(|p| Path::new(p).exists())
|
|
.unwrap_or(false);
|
|
let has_kernel = spec
|
|
.kernel_path
|
|
.as_ref()
|
|
.map(|p| Path::new(p).exists())
|
|
.unwrap_or(false);
|
|
|
|
if !(has_fw || has_kernel) {
|
|
return Err(CloudHvError::InvalidSpec(
|
|
"either firmware_path or kernel_path must be set to an existing file".into(),
|
|
));
|
|
}
|
|
|
|
if !Path::new(&spec.disk_path).exists() {
|
|
return Err(CloudHvError::InvalidSpec(format!(
|
|
"disk_path not found: {}",
|
|
&spec.disk_path
|
|
)));
|
|
}
|
|
if spec.vcpus == 0 {
|
|
return Err(CloudHvError::InvalidSpec("vcpus must be >= 1".into()));
|
|
}
|
|
if spec.memory_mb == 0 {
|
|
return Err(CloudHvError::InvalidSpec("memory_mb must be >= 128".into()));
|
|
}
|
|
|
|
// Prepare directory layout
|
|
let dir = vm_dir(&spec.id);
|
|
sal_os::mkdir(
|
|
dir.to_str()
|
|
.unwrap_or_else(|| "/tmp/hero/virt/vms/__invalid__"),
|
|
)
|
|
.map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
let log_dir = dir.join("logs");
|
|
sal_os::mkdir(log_dir.to_str().unwrap()).map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
|
|
// Persist initial record
|
|
let rec = VmRecord {
|
|
spec: spec.clone(),
|
|
runtime: VmRuntime {
|
|
pid: None,
|
|
status: "stopped".into(),
|
|
log_file: vm_log_path(&spec.id).to_string_lossy().into_owned(),
|
|
},
|
|
};
|
|
let value = serde_json::to_value(&rec).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
write_json(&vm_json_path(&spec.id), &value)?;
|
|
|
|
Ok(spec.id.clone())
|
|
}
|
|
|
|
/// Start a VM using cloud-hypervisor
|
|
pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
|
|
ensure_deps()?;
|
|
|
|
// Load record
|
|
let p = vm_json_path(id);
|
|
if !p.exists() {
|
|
return Err(CloudHvError::NotFound(format!("VM '{}' not found", id)));
|
|
}
|
|
let value = read_json(&p)?;
|
|
let mut rec: VmRecord =
|
|
serde_json::from_value(value).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
|
|
// Prepare invocation
|
|
let api_socket = if rec.spec.api_socket.trim().is_empty() {
|
|
vm_dir(id).join("api.sock").to_string_lossy().into_owned()
|
|
} else {
|
|
rec.spec.api_socket.clone()
|
|
};
|
|
let log_file = vm_log_path(id).to_string_lossy().into_owned();
|
|
|
|
// Ensure API socket directory exists and remove any stale socket file
|
|
let api_path = Path::new(&api_socket);
|
|
if let Some(parent) = api_path.parent() {
|
|
fs::create_dir_all(parent).map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
}
|
|
// Best-effort removal of stale socket
|
|
let _ = fs::remove_file(&api_path);
|
|
|
|
// Preflight disk: if source is qcow2, convert to raw to avoid CH "Compressed blocks not supported"
|
|
// Robust conversion:
|
|
// - Remove any stale destination
|
|
// - Try direct convert to destination file
|
|
// - On failure (e.g., byte-range lock issues), fallback to piping stdout into dd
|
|
let mut disk_to_use = rec.spec.disk_path.clone();
|
|
if let Ok(info) = qcow2::info(&disk_to_use) {
|
|
if info.get("format").and_then(|v| v.as_str()) == Some("qcow2") {
|
|
let dest = vm_dir(id).join("disk.raw").to_string_lossy().into_owned();
|
|
// Best-effort remove stale target file to avoid locking errors
|
|
let _ = fs::remove_file(&dest);
|
|
|
|
// Attempt 1: normal qemu-img convert to dest file
|
|
let cmd1 = format!(
|
|
"qemu-img convert -O raw {} {}",
|
|
shell_escape(&disk_to_use),
|
|
shell_escape(&dest)
|
|
);
|
|
let attempt1 = sal_process::run(&cmd1).silent(true).die(false).execute();
|
|
|
|
let mut converted_ok = false;
|
|
let mut err1: Option<String> = None;
|
|
|
|
if let Ok(res) = attempt1 {
|
|
if res.success {
|
|
converted_ok = true;
|
|
} else {
|
|
err1 = Some(format!("{}{}", res.stdout, res.stderr));
|
|
}
|
|
} else if let Err(e) = attempt1 {
|
|
err1 = Some(e.to_string());
|
|
}
|
|
|
|
if !converted_ok {
|
|
// Attempt 2: pipe via stdout into dd (avoids qemu-img destination locking semantics on some FS)
|
|
let heredoc2 = format!(
|
|
"bash -e -s <<'EOF'\nset -euo pipefail\nqemu-img convert -O raw {} - | dd of={} bs=4M status=none\nEOF\n",
|
|
shell_escape(&disk_to_use),
|
|
shell_escape(&dest)
|
|
);
|
|
match sal_process::run(&heredoc2).silent(true).die(false).execute() {
|
|
Ok(res) if res.success => {
|
|
converted_ok = true;
|
|
}
|
|
Ok(res) => {
|
|
let mut msg = String::from("Failed converting qcow2 to raw.");
|
|
if let Some(e1) = err1 {
|
|
msg.push_str(&format!("\nFirst attempt error:\n{}", e1));
|
|
}
|
|
msg.push_str(&format!("\nSecond attempt error:\n{}{}", res.stdout, res.stderr));
|
|
return Err(CloudHvError::CommandFailed(msg));
|
|
}
|
|
Err(e) => {
|
|
let mut msg = String::from("Failed converting qcow2 to raw.");
|
|
if let Some(e1) = err1 {
|
|
msg.push_str(&format!("\nFirst attempt error:\n{}", e1));
|
|
}
|
|
msg.push_str(&format!("\nSecond attempt error:\n{}", e));
|
|
return Err(CloudHvError::CommandFailed(msg));
|
|
}
|
|
}
|
|
}
|
|
|
|
if converted_ok {
|
|
disk_to_use = dest;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Consolidate extra --disk occurrences from spec.extra_args into a single --disk (CH version requires variadic form)
|
|
// Collect disk value tokens provided by the user and strip them from extra args so we can render one '--disk' followed by multiple values.
|
|
let mut extra_disk_vals: Vec<String> = Vec::new();
|
|
let mut extra_args_sans_disks: Vec<String> = Vec::new();
|
|
if let Some(extra) = rec.spec.extra_args.clone() {
|
|
let mut i = 0usize;
|
|
while i < extra.len() {
|
|
let tok = extra[i].clone();
|
|
if tok == "--disk" {
|
|
if i + 1 < extra.len() {
|
|
extra_disk_vals.push(extra[i + 1].clone());
|
|
i += 2;
|
|
continue;
|
|
} else {
|
|
// dangling --disk without value; drop it
|
|
i += 1;
|
|
continue;
|
|
}
|
|
} else if tok == "--no-default-net" {
|
|
// sentinel: suppress default networking; do not pass to CH CLI
|
|
i += 1;
|
|
continue;
|
|
} else if let Some(rest) = tok.strip_prefix("--disk=") {
|
|
if !rest.is_empty() {
|
|
extra_disk_vals.push(rest.to_string());
|
|
}
|
|
i += 1;
|
|
continue;
|
|
}
|
|
// keep token
|
|
extra_args_sans_disks.push(tok);
|
|
i += 1;
|
|
}
|
|
}
|
|
|
|
// CH CLI flags (very common subset)
|
|
// --disk path=... uses virtio-blk by default
|
|
let mut parts: Vec<String> = vec![
|
|
"cloud-hypervisor-static".into(),
|
|
"--api-socket".into(),
|
|
api_socket.clone(),
|
|
];
|
|
|
|
if let Some(fw) = rec.spec.firmware_path.clone() {
|
|
// Firmware boot path
|
|
parts.push("--firmware".into());
|
|
parts.push(fw);
|
|
} else if let Some(kpath) = rec.spec.kernel_path.clone() {
|
|
// Direct kernel boot path
|
|
let cmdline = rec
|
|
.spec
|
|
.cmdline
|
|
.clone()
|
|
.unwrap_or_else(|| "console=ttyS0 reboot=k panic=1".to_string());
|
|
parts.push("--kernel".into());
|
|
parts.push(kpath);
|
|
if let Some(initrd) = rec.spec.initramfs_path.clone() {
|
|
if Path::new(&initrd).exists() {
|
|
parts.push("--initramfs".into());
|
|
parts.push(initrd);
|
|
}
|
|
}
|
|
parts.push("--cmdline".into());
|
|
parts.push(cmdline);
|
|
} else {
|
|
return Err(CloudHvError::InvalidSpec(
|
|
"neither firmware_path nor kernel_path set at start time".into(),
|
|
));
|
|
}
|
|
|
|
parts.push("--disk".into());
|
|
parts.push(format!("path={}", disk_to_use));
|
|
// Append any additional disk value tokens (from sanitized extra args) so CH sees a single '--disk' with multiple values
|
|
for dv in &extra_disk_vals {
|
|
parts.push(dv.clone());
|
|
}
|
|
parts.push("--cpus".into());
|
|
parts.push(format!("boot={}", rec.spec.vcpus));
|
|
parts.push("--memory".into());
|
|
parts.push(format!("size={}M", rec.spec.memory_mb));
|
|
parts.push("--serial".into());
|
|
parts.push("tty".into());
|
|
parts.push("--console".into());
|
|
parts.push("off".into());
|
|
|
|
// Determine if the user provided explicit network arguments (e.g. "--net", "tap=...,mac=...")
|
|
// If so, do NOT provision the default host networking or add a default NIC.
|
|
let has_user_net = rec
|
|
.spec
|
|
.extra_args
|
|
.as_ref()
|
|
.map(|v| v.iter().any(|tok| tok == "--net" || tok == "--no-default-net"))
|
|
.unwrap_or(false);
|
|
|
|
// Track chosen bridge/lease for later discovery
|
|
let mut bridge_for_disc: Option<String> = None;
|
|
let mut lease_for_disc: Option<String> = None;
|
|
|
|
// Determine effective networking profile
|
|
let profile_effective = if let Some(p) = rec.spec.net_profile.clone() {
|
|
Some(p)
|
|
} else if has_user_net {
|
|
// User provided explicit --net or --no-default-net; do not provision
|
|
None
|
|
} else {
|
|
// Default behavior: NAT profile
|
|
Some(NetworkingProfileSpec::DefaultNat(DefaultNatOptions::default()))
|
|
};
|
|
|
|
if let Some(profile) = profile_effective {
|
|
match profile {
|
|
NetworkingProfileSpec::DefaultNat(mut nat) => {
|
|
// IPv6 handling (auto via Mycelium unless disabled)
|
|
let mut ipv6_bridge_cidr: Option<String> = None;
|
|
if nat.ipv6_enable {
|
|
if let Ok(cidr) = std::env::var("HERO_VIRT_IPV6_BRIDGE_CIDR") {
|
|
// Validate mycelium iface presence if specified or default
|
|
let if_hint = nat.mycelium_if.clone().unwrap_or_else(|| "mycelium".into());
|
|
let _ = net::mycelium_ipv6_addr(&if_hint)?;
|
|
ipv6_bridge_cidr = Some(cidr);
|
|
} else {
|
|
let if_hint = nat.mycelium_if.clone().unwrap_or_else(|| "mycelium".into());
|
|
println!("auto-deriving mycelium address...");
|
|
let (_ifname, myc_addr) = net::mycelium_ipv6_addr(&if_hint)?;
|
|
let (_pfx, router_cidr) = net::derive_ipv6_prefix_from_mycelium(&myc_addr)?;
|
|
println!("derived router cidr for bridge: {}", router_cidr);
|
|
ipv6_bridge_cidr = Some(router_cidr);
|
|
}
|
|
}
|
|
|
|
// Ensure bridge, NAT, and DHCP
|
|
net::ensure_bridge(&nat.bridge_name, &nat.bridge_addr_cidr, ipv6_bridge_cidr.as_deref())?;
|
|
// Derive IPv6 subnet for NAT
|
|
let ipv6_subnet = ipv6_bridge_cidr.as_ref().map(|cidr| {
|
|
let parts: Vec<&str> = cidr.split('/').collect();
|
|
if parts.len() == 2 {
|
|
let addr = parts[0];
|
|
if let Ok(ip) = addr.parse::<std::net::Ipv6Addr>() {
|
|
let seg = ip.segments();
|
|
let pfx = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 0);
|
|
format!("{}/64", pfx)
|
|
} else {
|
|
"".to_string()
|
|
}
|
|
} else {
|
|
"".to_string()
|
|
}
|
|
});
|
|
net::ensure_nat(&nat.subnet_cidr, ipv6_subnet.as_deref())?;
|
|
let lease_used = net::ensure_dnsmasq(
|
|
&nat.bridge_name,
|
|
&nat.dhcp_start,
|
|
&nat.dhcp_end,
|
|
ipv6_bridge_cidr.as_deref(),
|
|
nat.lease_file.as_deref(),
|
|
)?;
|
|
|
|
bridge_for_disc = Some(nat.bridge_name.clone());
|
|
lease_for_disc = Some(lease_used.clone());
|
|
|
|
// TAP + NIC args
|
|
let tap_name = net::ensure_tap_for_vm(&nat.bridge_name, id)?;
|
|
println!("TAP device for vm called: {tap_name}");
|
|
let mac = net::stable_mac_from_id(id);
|
|
println!("MAC for vm: {mac}");
|
|
parts.push("--net".into());
|
|
parts.push(format!("tap={},mac={}", tap_name, mac));
|
|
}
|
|
NetworkingProfileSpec::BridgeOnly(opts) => {
|
|
let bridge_name = opts.bridge_name.clone();
|
|
// Use provided IPv4 if any, else env default
|
|
let bridge_addr_cidr = opts
|
|
.bridge_addr_cidr
|
|
.clone()
|
|
.unwrap_or_else(|| std::env::var("HERO_VIRT_BRIDGE_ADDR_CIDR").unwrap_or_else(|_| "172.30.0.1/24".into()));
|
|
// Ensure bridge (optional IPv6 from opts)
|
|
net::ensure_bridge(&bridge_name, &bridge_addr_cidr, opts.bridge_ipv6_cidr.as_deref())?;
|
|
// TAP + NIC only, no NAT/DHCP
|
|
let tap_name = net::ensure_tap_for_vm(&bridge_name, id)?;
|
|
println!("TAP device for vm called: {tap_name}");
|
|
let mac = net::stable_mac_from_id(id);
|
|
println!("MAC for vm: {mac}");
|
|
parts.push("--net".into());
|
|
parts.push(format!("tap={},mac={}", tap_name, mac));
|
|
|
|
// For discovery: we can attempt IPv6 neighbor; IPv4 lease not present
|
|
bridge_for_disc = Some(bridge_name);
|
|
lease_for_disc = None;
|
|
}
|
|
NetworkingProfileSpec::NoNet => {
|
|
// Do nothing
|
|
}
|
|
NetworkingProfileSpec::CustomCli(_args) => {
|
|
// Do not provision; user must add --net via extra_args
|
|
}
|
|
}
|
|
}
|
|
|
|
// Append any user-provided extra args, sans any '--disk' we already consolidated
|
|
for e in extra_args_sans_disks {
|
|
parts.push(e);
|
|
}
|
|
|
|
let args_str = shell_join(&parts);
|
|
// Execute via a bash heredoc to avoid any quoting pitfalls
|
|
let heredoc = format!(
|
|
"bash -e -s <<'EOF'\nnohup {} > '{}' 2>&1 &\necho $! > '{}'\nEOF\n",
|
|
args_str,
|
|
log_file,
|
|
vm_pid_path(id).to_string_lossy()
|
|
);
|
|
println!("executing command:\n{heredoc}");
|
|
// Execute command; this will background cloud-hypervisor and return
|
|
let result = sal_process::run(&heredoc).execute();
|
|
match result {
|
|
Ok(res) => {
|
|
if !res.success {
|
|
return Err(CloudHvError::CommandFailed(format!(
|
|
"Failed to start VM '{}': {}",
|
|
id, res.stderr
|
|
)));
|
|
}
|
|
}
|
|
Err(e) => {
|
|
return Err(CloudHvError::CommandFailed(format!(
|
|
"Failed to start VM '{}': {}",
|
|
id, e
|
|
)))
|
|
}
|
|
}
|
|
|
|
// Read PID back
|
|
let pid = match fs::read_to_string(vm_pid_path(id)) {
|
|
Ok(s) => s.trim().parse::<i64>().ok(),
|
|
Err(_) => None,
|
|
};
|
|
println!("reading PID back: {} - (if 0 == not found)", pid.unwrap_or(0));
|
|
|
|
// Quick health check: ensure process did not exit immediately due to CLI errors (e.g., duplicate flags)
|
|
if let Some(pid_num) = pid {
|
|
thread::sleep(Duration::from_millis(300));
|
|
if !proc_exists(pid_num) {
|
|
// Tail log to surface the error cause
|
|
let tail_cmd = format!("tail -n 200 {}", shell_escape(&log_file));
|
|
println!("executing tail_cmd command:\n{tail_cmd}");
|
|
let tail = sal_process::run(&tail_cmd).die(false).silent(true).execute();
|
|
let mut log_snip = String::new();
|
|
if let Ok(res) = tail {
|
|
if res.success {
|
|
log_snip = res.stdout;
|
|
} else {
|
|
log_snip = format!("{}{}", res.stdout, res.stderr);
|
|
}
|
|
}
|
|
return Err(CloudHvError::CommandFailed(format!(
|
|
"cloud-hypervisor exited immediately after start. Log tail:\n{}",
|
|
log_snip
|
|
)));
|
|
}
|
|
} else {
|
|
return Err(CloudHvError::CommandFailed(
|
|
"failed to obtain cloud-hypervisor PID (start script did not write pid)".into(),
|
|
));
|
|
}
|
|
|
|
// Update state
|
|
rec.runtime.pid = pid;
|
|
rec.runtime.status = if pid.is_some() { "running".into() } else { "stopped".into() };
|
|
rec.runtime.log_file = log_file;
|
|
rec.spec.api_socket = api_socket.clone();
|
|
|
|
let value = serde_json::to_value(&rec).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
write_json(&vm_json_path(id), &value)?;
|
|
println!("wrote JSON for VM");
|
|
|
|
// Best-effort: discover and print guest IPv4/IPv6 addresses (default-net path)
|
|
println!("waiting 5 secs for DHCP/ND");
|
|
thread::sleep(Duration::from_millis(5000));
|
|
let mac_lower = net::stable_mac_from_id(id).to_lowercase();
|
|
|
|
if let Some(bridge_name) = bridge_for_disc.clone() {
|
|
let lease_path = lease_for_disc.unwrap_or_else(|| {
|
|
std::env::var("HERO_VIRT_DHCP_LEASE_FILE")
|
|
.unwrap_or_else(|_| format!("/var/lib/misc/dnsmasq-hero-{}.leases", bridge_name))
|
|
});
|
|
let ipv4 = net::discover_ipv4_from_leases(&lease_path, &mac_lower, 12);
|
|
println!(
|
|
"Got IPv4 from dnsmasq lease ({}): {}",
|
|
lease_path,
|
|
ipv4.clone().unwrap_or("not found".to_string())
|
|
);
|
|
|
|
let ipv6 = net::discover_ipv6_on_bridge(&bridge_name, &mac_lower);
|
|
println!(
|
|
"Got IPv6 from neighbor table on bridge: {}",
|
|
ipv6.clone().unwrap_or("not found".to_string())
|
|
);
|
|
|
|
println!(
|
|
"[cloudhv] VM '{}' guest addresses: IPv4={}, IPv6={}",
|
|
id,
|
|
ipv4.as_deref().unwrap_or(""),
|
|
ipv6.as_deref().unwrap_or("")
|
|
);
|
|
} else {
|
|
println!(
|
|
"[cloudhv] VM '{}' guest addresses discovery skipped (no default bridge in use)",
|
|
id
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Return VM record info (spec + runtime) by id
|
|
pub fn vm_info(id: &str) -> Result<VmRecord, CloudHvError> {
|
|
let p = vm_json_path(id);
|
|
if !p.exists() {
|
|
return Err(CloudHvError::NotFound(format!("VM '{}' not found", id)));
|
|
}
|
|
let value = read_json(&p)?;
|
|
let rec: VmRecord = serde_json::from_value(value).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
Ok(rec)
|
|
}
|
|
|
|
/// Stop a VM via ch-remote (graceful), optionally force kill
|
|
pub fn vm_stop(id: &str, force: bool) -> Result<(), CloudHvError> {
|
|
ensure_deps().ok(); // best-effort; we might still force-kill
|
|
|
|
let p = vm_json_path(id);
|
|
if !p.exists() {
|
|
return Err(CloudHvError::NotFound(format!("VM '{}' not found", id)));
|
|
}
|
|
let value = read_json(&p)?;
|
|
let mut rec: VmRecord =
|
|
serde_json::from_value(value).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
|
|
// Attempt graceful shutdown if api socket known
|
|
if !rec.spec.api_socket.trim().is_empty() {
|
|
let cmd = format!("ch-remote-static --api-socket {} shutdown", rec.spec.api_socket);
|
|
let _ = sal_process::run(&cmd).die(false).silent(true).execute();
|
|
}
|
|
|
|
// Wait for process to exit (up to ~10s)
|
|
if let Some(pid) = rec.runtime.pid {
|
|
for _ in 0..50 {
|
|
if !proc_exists(pid) {
|
|
break;
|
|
}
|
|
thread::sleep(Duration::from_millis(200));
|
|
}
|
|
// If still alive and force, kill -9 and wait again (up to ~10s)
|
|
if proc_exists(pid) && force {
|
|
// Send SIGKILL without extra shell layers; suppress errors/noise
|
|
let _ = sal_process::run(&format!("kill -9 {}", pid))
|
|
.die(false)
|
|
.silent(true)
|
|
.execute();
|
|
for _ in 0..50 {
|
|
if !proc_exists(pid) {
|
|
break;
|
|
}
|
|
thread::sleep(Duration::from_millis(200));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update state
|
|
rec.runtime.status = "stopped".into();
|
|
rec.runtime.pid = None;
|
|
let value = serde_json::to_value(&rec).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
write_json(&vm_json_path(id), &value)?;
|
|
|
|
// Remove pid file
|
|
let _ = fs::remove_file(vm_pid_path(id));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Delete a VM definition; optionally delete disks.
|
|
pub fn vm_delete(id: &str, delete_disks: bool) -> Result<(), CloudHvError> {
|
|
let p = vm_json_path(id);
|
|
if !p.exists() {
|
|
return Err(CloudHvError::NotFound(format!("VM '{}' not found", id)));
|
|
}
|
|
let rec: VmRecord = serde_json::from_value(read_json(&p)?)
|
|
.map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
|
|
// If appears to be running, attempt a force stop first (best-effort)
|
|
if let Some(pid) = rec.runtime.pid {
|
|
if proc_exists(pid) {
|
|
let _ = vm_stop(id, true);
|
|
// Re-check original PID for liveness (up to ~5s)
|
|
for _ in 0..25 {
|
|
if !proc_exists(pid) {
|
|
break;
|
|
}
|
|
thread::sleep(Duration::from_millis(200));
|
|
}
|
|
if proc_exists(pid) {
|
|
return Err(CloudHvError::CommandFailed(
|
|
"VM appears to be running; stop it first".into(),
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
if delete_disks {
|
|
let _ = fs::remove_file(&rec.spec.disk_path);
|
|
}
|
|
|
|
let d = vm_dir(id);
|
|
fs::remove_dir_all(&d).map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
Ok(())
|
|
}
|
|
|
|
/// List all VMs
|
|
pub fn vm_list() -> Result<Vec<VmRecord>, CloudHvError> {
|
|
let root = hero_vm_root();
|
|
if !root.exists() {
|
|
return Ok(vec![]);
|
|
}
|
|
let mut out = vec![];
|
|
for entry in fs::read_dir(&root).map_err(|e| CloudHvError::IoError(e.to_string()))? {
|
|
let entry = entry.map_err(|e| CloudHvError::IoError(e.to_string()))?;
|
|
let p = entry.path();
|
|
if !p.is_dir() {
|
|
continue;
|
|
}
|
|
let vm_json = p.join("vm.json");
|
|
if !vm_json.exists() {
|
|
continue;
|
|
}
|
|
let rec: VmRecord = serde_json::from_value(read_json(&vm_json)?)
|
|
.map_err(|e| CloudHvError::JsonError(e.to_string()))?;
|
|
|
|
out.push(rec);
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
fn tap_name_for_id(id: &str) -> String {
|
|
// Linux IFNAMSIZ is typically 15; keep "tap-" + 10 hex = 14 chars
|
|
let mut h = DefaultHasher::new();
|
|
id.hash(&mut h);
|
|
let v = h.finish();
|
|
let hex = format!("{:016x}", v);
|
|
format!("tap-{}", &hex[..10])
|
|
}
|
|
|
|
fn ensure_tap_for_vm(bridge_name: &str, id: &str) -> Result<String, CloudHvError> {
|
|
let tap = tap_name_for_id(id);
|
|
|
|
let body = format!(
|
|
"BR={br}
|
|
TAP={tap}
|
|
UIDX=$(id -u)
|
|
GIDX=$(id -g)
|
|
|
|
# Create TAP if missing and assign to current user/group
|
|
ip link show \"$TAP\" >/dev/null 2>&1 || ip tuntap add dev \"$TAP\" mode tap user \"$UIDX\" group \"$GIDX\"
|
|
|
|
# Enslave to bridge and bring up (idempotent)
|
|
ip link set \"$TAP\" master \"$BR\" 2>/dev/null || true
|
|
ip link set \"$TAP\" up
|
|
",
|
|
br = shell_escape(bridge_name),
|
|
tap = shell_escape(&tap),
|
|
);
|
|
let heredoc_tap = format!("bash -e -s <<'EOF'\n{}\nEOF\n", body);
|
|
|
|
match sal_process::run(&heredoc_tap).silent(true).execute() {
|
|
Ok(res) if res.success => Ok(tap),
|
|
Ok(res) => Err(CloudHvError::CommandFailed(format!(
|
|
"Failed to ensure TAP '{}': {}",
|
|
tap, res.stderr
|
|
))),
|
|
Err(e) => Err(CloudHvError::CommandFailed(format!(
|
|
"Failed to ensure TAP '{}': {}",
|
|
tap, e
|
|
))),
|
|
}
|
|
}
|
|
|
|
fn stable_mac_from_id(id: &str) -> String {
|
|
let mut h = DefaultHasher::new();
|
|
id.hash(&mut h);
|
|
let v = h.finish();
|
|
let b0 = (((v >> 40) & 0xff) as u8 & 0xfe) | 0x02; // locally administered, unicast
|
|
let b1 = ((v >> 32) & 0xff) as u8;
|
|
let b2 = ((v >> 24) & 0xff) as u8;
|
|
let b3 = ((v >> 16) & 0xff) as u8;
|
|
let b4 = ((v >> 8) & 0xff) as u8;
|
|
let b5 = (v & 0xff) as u8;
|
|
format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5)
|
|
}
|
|
|
|
/// Discover the mycelium IPv6 address by inspecting the interface itself (no CLI dependency).
|
|
/// Returns (interface_name, first global IPv6 address found on the interface).
|
|
fn get_mycelium_ipv6_addr(iface_hint: &str) -> Result<(String, String), CloudHvError> {
|
|
let iface = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| iface_hint.to_string());
|
|
|
|
// Query IPv6 addresses on the interface
|
|
let cmd = format!("ip -6 addr show dev {}", shell_escape(&iface));
|
|
let res = sal_process::run(&cmd).silent(true).die(false).execute();
|
|
let out = match res {
|
|
Ok(r) if r.success => r.stdout,
|
|
_ => {
|
|
return Err(CloudHvError::DependencyMissing(format!(
|
|
"mycelium interface '{}' not found or no IPv6 configured",
|
|
iface
|
|
)))
|
|
}
|
|
};
|
|
|
|
// Extract the first global IPv6 address present on the interface.
|
|
for line in out.lines() {
|
|
let lt = line.trim();
|
|
// Example line: "inet6 578:9fcf:.../7 scope global"
|
|
if lt.starts_with("inet6 ") && lt.contains("scope global") {
|
|
let parts: Vec<&str> = lt.split_whitespace().collect();
|
|
if let Some(addr_cidr) = parts.get(1) {
|
|
let addr_only = addr_cidr.split('/').next().unwrap_or("").trim();
|
|
println!("got addr from host: {addr_only}");
|
|
if !addr_only.is_empty() && addr_only.parse::<std::net::Ipv6Addr>().is_ok() {
|
|
return Ok((iface, addr_only.to_string()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Err(CloudHvError::DependencyMissing(format!(
|
|
"no global IPv6 found on interface '{}'",
|
|
iface
|
|
)))
|
|
}
|
|
|
|
/// Derive a /64 prefix P from the mycelium IPv6 and return (P/64, P::2/64).
|
|
fn derive_ipv6_prefix_from_mycelium(m: &str) -> Result<(String, String), CloudHvError> {
|
|
let ip = m
|
|
.parse::<std::net::Ipv6Addr>()
|
|
.map_err(|e| CloudHvError::InvalidSpec(format!("invalid mycelium IPv6 address '{}': {}", m, e)))?;
|
|
let seg = ip.segments(); // [u16; 8]
|
|
// Take the top /64 from the mycelium address; zero the host half
|
|
let pfx = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 0);
|
|
// Router address for the bridge = P::2
|
|
let router = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 2);
|
|
let pfx_str = format!("{}/64", pfx);
|
|
let router_cidr = format!("{}/64", router);
|
|
Ok((pfx_str, router_cidr))
|
|
}
|
|
|
|
fn ensure_host_net_prereq_dnsmasq_nftables(
|
|
bridge_name: &str,
|
|
bridge_addr_cidr: &str,
|
|
subnet_cidr: &str,
|
|
dhcp_start: &str,
|
|
dhcp_end: &str,
|
|
ipv6_bridge_cidr: Option<&str>,
|
|
mycelium_if: Option<&str>,
|
|
) -> Result<(), CloudHvError> {
|
|
// Dependencies
|
|
for bin in ["ip", "nft", "dnsmasq", "systemctl"] {
|
|
if sal_process::which(bin).is_none() {
|
|
return Err(CloudHvError::DependencyMissing(format!(
|
|
"{} not found on PATH; required for VM networking",
|
|
bin
|
|
)));
|
|
}
|
|
}
|
|
|
|
// Prepare optional IPv6 value (empty string when disabled)
|
|
let ipv6_cidr = ipv6_bridge_cidr.unwrap_or("");
|
|
|
|
// Build idempotent setup script
|
|
let body = format!(
|
|
"set -e
|
|
BR={br}
|
|
BR_ADDR={br_addr}
|
|
SUBNET={subnet}
|
|
DHCP_START={dstart}
|
|
DHCP_END={dend}
|
|
IPV6_CIDR={v6cidr}
|
|
LEASE_FILE=/var/lib/misc/dnsmasq-hero-$BR.leases
|
|
|
|
# Determine default WAN interface
|
|
WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1)
|
|
if [ -z \"$WAN_IF\" ]; then
|
|
echo \"No default WAN interface detected (required for IPv4 NAT)\" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Bridge creation (idempotent)
|
|
ip link show \"$BR\" >/dev/null 2>&1 || ip link add name \"$BR\" type bridge
|
|
ip addr replace \"$BR_ADDR\" dev \"$BR\"
|
|
ip link set \"$BR\" up
|
|
|
|
# IPv6 bridge address and forwarding (optional)
|
|
if [ -n \"$IPV6_CIDR\" ]; then
|
|
ip -6 addr replace \"$IPV6_CIDR\" dev \"$BR\"
|
|
sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null || true
|
|
fi
|
|
|
|
# IPv4 forwarding
|
|
sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
|
|
# nftables NAT (idempotent) for IPv4
|
|
nft list table ip hero >/dev/null 2>&1 || nft add table ip hero
|
|
nft list chain ip hero postrouting >/dev/null 2>&1 || nft add chain ip hero postrouting {{ type nat hook postrouting priority 100 \\; }}
|
|
nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN_IF\\\" masquerade\" \
|
|
|| nft add rule ip hero postrouting ip saddr $SUBNET oifname \"$WAN_IF\" masquerade
|
|
|
|
# dnsmasq DHCPv4 + RA/DHCPv6 config (idempotent)
|
|
mkdir -p /etc/dnsmasq.d
|
|
mkdir -p /var/lib/misc
|
|
CFG=/etc/dnsmasq.d/hero-$BR.conf
|
|
TMP=/etc/dnsmasq.d/.hero-$BR.conf.new
|
|
|
|
RELOAD=0
|
|
CONF=/etc/dnsmasq.conf
|
|
# Ensure conf-dir includes /etc/dnsmasq.d (simple fixed-string check to avoid regex escapes in Rust)
|
|
if ! grep -qF \"conf-dir=/etc/dnsmasq.d\" \"$CONF\"; then
|
|
printf '%s\n' 'conf-dir=/etc/dnsmasq.d,*.conf' >> \"$CONF\"
|
|
RELOAD=1
|
|
fi
|
|
|
|
|
|
# Ensure lease file exists and is writable by dnsmasq user
|
|
touch \"$LEASE_FILE\" || true
|
|
chown dnsmasq:dnsmasq \"$LEASE_FILE\" 2>/dev/null || true
|
|
|
|
# Always include IPv4 section
|
|
printf '%s\n' \
|
|
\"interface=$BR\" \
|
|
\"bind-interfaces\" \
|
|
\"dhcp-authoritative\" \
|
|
\"dhcp-range=$DHCP_START,$DHCP_END,12h\" \
|
|
\"dhcp-option=option:dns-server,1.1.1.1,8.8.8.8\" \
|
|
\"dhcp-leasefile=$LEASE_FILE\" >\"$TMP\"
|
|
|
|
# Optionally append IPv6 RA/DHCPv6
|
|
if [ -n \"$IPV6_CIDR\" ]; then
|
|
printf '%s\n' \
|
|
\"enable-ra\" \
|
|
\"dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h\" \
|
|
\"dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111]\" >>\"$TMP\"
|
|
sed -i \"s/BR_PLACEHOLDER/$BR/g\" \"$TMP\"
|
|
fi
|
|
|
|
if [ ! -f \"$CFG\" ] || ! cmp -s \"$CFG\" \"$TMP\"; then
|
|
mv \"$TMP\" \"$CFG\"
|
|
if systemctl is-active --quiet dnsmasq; then
|
|
systemctl reload dnsmasq || systemctl restart dnsmasq || true
|
|
else
|
|
systemctl enable --now dnsmasq || true
|
|
fi
|
|
else
|
|
rm -f \"$TMP\"
|
|
systemctl enable --now dnsmasq || true
|
|
fi
|
|
|
|
# Reload if main conf was updated to include conf-dir
|
|
if [ \"$RELOAD\" = \"1\" ]; then
|
|
systemctl reload dnsmasq || systemctl restart dnsmasq || true
|
|
fi
|
|
",
|
|
br = shell_escape(bridge_name),
|
|
br_addr = shell_escape(bridge_addr_cidr),
|
|
subnet = shell_escape(subnet_cidr),
|
|
dstart = shell_escape(dhcp_start),
|
|
dend = shell_escape(dhcp_end),
|
|
v6cidr = shell_escape(ipv6_cidr),
|
|
);
|
|
|
|
// Use a unique heredoc delimiter to avoid clashing with inner <<EOF blocks
|
|
let heredoc_net = format!("bash -e -s <<'HERONET'\n{}\nHERONET\n", body);
|
|
println!("executing command:\n{heredoc_net}");
|
|
|
|
match sal_process::run(&heredoc_net).silent(true).execute() {
|
|
Ok(res) if res.success => Ok(()),
|
|
Ok(res) => Err(CloudHvError::CommandFailed(format!(
|
|
"Host networking setup failed: {}",
|
|
res.stderr
|
|
))),
|
|
Err(e) => Err(CloudHvError::CommandFailed(format!(
|
|
"Host networking setup failed: {}",
|
|
e
|
|
))),
|
|
}
|
|
}
|
|
|
|
/// Render a shell-safe command string from vector of tokens
|
|
fn shell_join(parts: &Vec<String>) -> String {
|
|
let mut s = String::new();
|
|
for (i, p) in parts.iter().enumerate() {
|
|
if i > 0 {
|
|
s.push(' ');
|
|
}
|
|
s.push_str(&shell_escape(p));
|
|
}
|
|
s
|
|
}
|
|
|
|
fn shell_escape(s: &str) -> String {
|
|
if s.is_empty() {
|
|
return "''".into();
|
|
}
|
|
if s
|
|
.chars()
|
|
.all(|c| c.is_ascii_alphanumeric() || "-_./=:".contains(c))
|
|
{
|
|
return s.into();
|
|
}
|
|
// single-quote wrap, escape existing quotes
|
|
let mut out = String::from("'");
|
|
for ch in s.chars() {
|
|
if ch == '\'' {
|
|
out.push_str("'\"'\"'");
|
|
} else {
|
|
out.push(ch);
|
|
}
|
|
}
|
|
out.push('\'');
|
|
out
|
|
} |