From 784f87db974e96f0ff8ed81163ecd10a522397fd Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Wed, 27 Aug 2025 16:03:32 +0200 Subject: [PATCH] WIP2 --- packages/system/virt/src/cloudhv/mod.rs | 160 +++++++++++++++--- packages/system/virt/src/hostcheck/mod.rs | 47 ++++- packages/system/virt/src/image_prep/mod.rs | 15 +- .../virt/tests/rhai/10_vm_end_to_end.rhai | 9 +- 4 files changed, 183 insertions(+), 48 deletions(-) diff --git a/packages/system/virt/src/cloudhv/mod.rs b/packages/system/virt/src/cloudhv/mod.rs index 71d12d4..741fc24 100644 --- a/packages/system/virt/src/cloudhv/mod.rs +++ b/packages/system/virt/src/cloudhv/mod.rs @@ -266,12 +266,12 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { if !converted_ok { // Attempt 2: pipe via stdout into dd (avoids qemu-img destination locking semantics on some FS) - let cmd2 = format!( - "#!/bin/bash -euo pipefail\nqemu-img convert -O raw {} - | dd of={} bs=4M status=none", + let heredoc2 = format!( + "bash -e -s <<'EOF'\nset -euo pipefail\nqemu-img convert -O raw {} - | dd of={} bs=4M status=none\nEOF\n", shell_escape(&disk_to_use), shell_escape(&dest) ); - match sal_process::run(&cmd2).silent(true).die(false).execute() { + match sal_process::run(&heredoc2).silent(true).die(false).execute() { Ok(res) if res.success => { converted_ok = true; } @@ -407,7 +407,31 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { std::env::var("HERO_VIRT_DHCP_START").unwrap_or_else(|_| "172.30.0.50".into()); let dhcp_end = std::env::var("HERO_VIRT_DHCP_END").unwrap_or_else(|_| "172.30.0.250".into()); - + + // Optional IPv6 over Mycelium provisioning (bridge P::2/64 + RA) guarded by env and detection. + let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into()); + let mut ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true"); + let mycelium_if_cfg = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into()); + let mut ipv6_bridge_cidr: Option = None; + let mut mycelium_if_opt: Option = None; + let enable_auto = ipv6_env.is_empty(); // auto-enable if mycelium is detected and not explicitly disabled + + if ipv6_enabled || enable_auto { + if let Ok(cidr) = std::env::var("HERO_VIRT_IPV6_BRIDGE_CIDR") { + // Explicit override for bridge IPv6 (e.g., "400:...::2/64") + ipv6_bridge_cidr = Some(cidr); + mycelium_if_opt = Some(mycelium_if_cfg.clone()); + ipv6_enabled = true; + } else if let Ok((ifname, myc_addr)) = get_mycelium_ipv6_addr(&mycelium_if_cfg) { + // Derive P::2/64 from the mycelium node address + if let Ok((_pfx, router_cidr)) = derive_ipv6_prefix_from_mycelium(&myc_addr) { + ipv6_bridge_cidr = Some(router_cidr); + mycelium_if_opt = Some(ifname); + ipv6_enabled = true; + } + } + } + // Ensure host-side networking (requires root privileges / CAP_NET_ADMIN) ensure_host_net_prereq_dnsmasq_nftables( &bridge_name, @@ -415,6 +439,8 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { &subnet_cidr, &dhcp_start, &dhcp_end, + ipv6_bridge_cidr.as_deref(), + mycelium_if_opt.as_deref(), )?; // Ensure a TAP device for this VM and attach to the bridge @@ -432,18 +458,15 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { } let args_str = shell_join(&parts); - let script = format!( - "#!/bin/bash -e -nohup {} > '{}' 2>&1 & -echo $! > '{}' -", + // Execute via a bash heredoc to avoid any quoting pitfalls + let heredoc = format!( + "bash -e -s <<'EOF'\nnohup {} > '{}' 2>&1 &\necho $! > '{}'\nEOF\n", args_str, log_file, vm_pid_path(id).to_string_lossy() ); - - // Execute script; this will background cloud-hypervisor and return - let result = sal_process::run(&script).execute(); + // Execute command; this will background cloud-hypervisor and return + let result = sal_process::run(&heredoc).execute(); match result { Ok(res) => { if !res.success { @@ -644,9 +667,8 @@ fn tap_name_for_id(id: &str) -> String { fn ensure_tap_for_vm(bridge_name: &str, id: &str) -> Result { let tap = tap_name_for_id(id); - let script = format!( - "#!/bin/bash -e -BR={br} + let body = format!( + "BR={br} TAP={tap} UIDX=$(id -u) GIDX=$(id -g) @@ -661,8 +683,9 @@ ip link set \"$TAP\" up br = shell_escape(bridge_name), tap = shell_escape(&tap), ); - - match sal_process::run(&script).silent(true).execute() { + let heredoc_tap = format!("bash -e -s <<'EOF'\n{}\nEOF\n", body); + + match sal_process::run(&heredoc_tap).silent(true).execute() { Ok(res) if res.success => Ok(tap), Ok(res) => Err(CloudHvError::CommandFailed(format!( "Failed to ensure TAP '{}': {}", @@ -688,12 +711,75 @@ fn stable_mac_from_id(id: &str) -> String { format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5) } +/// Discover the mycelium IPv6 address and validate the interface. +/// +/// Returns (interface_name, address_string). +fn get_mycelium_ipv6_addr(iface_hint: &str) -> Result<(String, String), CloudHvError> { + // Parse `mycelium inspect` for "Address: ..." + let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute(); + let mut addr: Option = None; + if let Ok(res) = insp { + if res.success { + for l in res.stdout.lines() { + let lt = l.trim(); + if let Some(rest) = lt.strip_prefix("Address:") { + let val = rest.trim().trim_matches('"').to_string(); + if !val.is_empty() { + addr = Some(val); + break; + } + } + } + } + } + let iface = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| iface_hint.to_string()); + + // Validate interface exists and has IPv6 configured (best-effort) + let cmd = format!("ip -6 addr show dev {}", shell_escape(&iface)); + match sal_process::run(&cmd).silent(true).die(false).execute() { + Ok(r) if r.success => { + // proceed + } + _ => { + return Err(CloudHvError::DependencyMissing(format!( + "mycelium interface '{}' not found or no IPv6 configured", + iface + ))); + } + } + + if let Some(a) = addr { + Ok((iface, a)) + } else { + Err(CloudHvError::DependencyMissing( + "failed to read mycelium IPv6 address via `mycelium inspect`".into(), + )) + } +} + +/// Derive a /64 prefix P from the mycelium IPv6 and return (P/64, P::2/64). +fn derive_ipv6_prefix_from_mycelium(m: &str) -> Result<(String, String), CloudHvError> { + let ip = m + .parse::() + .map_err(|e| CloudHvError::InvalidSpec(format!("invalid mycelium IPv6 address '{}': {}", m, e)))?; + let seg = ip.segments(); // [u16; 8] + // Take the top /64 from the mycelium address; zero the host half + let pfx = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 0); + // Router address for the bridge = P::2 + let router = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 2); + let pfx_str = format!("{}/64", pfx); + let router_cidr = format!("{}/64", router); + Ok((pfx_str, router_cidr)) +} + fn ensure_host_net_prereq_dnsmasq_nftables( bridge_name: &str, bridge_addr_cidr: &str, subnet_cidr: &str, dhcp_start: &str, dhcp_end: &str, + ipv6_bridge_cidr: Option<&str>, + mycelium_if: Option<&str>, ) -> Result<(), CloudHvError> { // Dependencies for bin in ["ip", "nft", "dnsmasq", "systemctl"] { @@ -705,16 +791,18 @@ fn ensure_host_net_prereq_dnsmasq_nftables( } } - // Build idempotent setup script - let script = format!( - "#!/bin/bash -e -set -e + // Prepare optional IPv6 value (empty string when disabled) + let ipv6_cidr = ipv6_bridge_cidr.unwrap_or(""); + // Build idempotent setup script + let body = format!( + "set -e BR={br} BR_ADDR={br_addr} SUBNET={subnet} DHCP_START={dstart} DHCP_END={dend} +IPV6_CIDR={v6cidr} # Determine default WAN interface WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1) @@ -724,23 +812,27 @@ ip link show \"$BR\" >/dev/null 2>&1 || ip link add name \"$BR\" type bridge ip addr replace \"$BR_ADDR\" dev \"$BR\" ip link set \"$BR\" up -# IPv6 placeholder address + forward (temporary) -ip -6 addr add 400::1/64 dev \"$BR\" 2>/dev/null || true -sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null || true +# IPv6 bridge address and forwarding (optional) +if [ -n \"$IPV6_CIDR\" ]; then + ip -6 addr replace \"$IPV6_CIDR\" dev \"$BR\" + sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null || true +fi # IPv4 forwarding sysctl -w net.ipv4.ip_forward=1 >/dev/null -# nftables NAT (idempotent) +# nftables NAT (idempotent) for IPv4 nft list table ip hero >/dev/null 2>&1 || nft add table ip hero nft list chain ip hero postrouting >/dev/null 2>&1 || nft add chain ip hero postrouting {{ type nat hook postrouting priority 100 \\; }} nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN_IF\\\" masquerade\" \ || nft add rule ip hero postrouting ip saddr $SUBNET oifname \"$WAN_IF\" masquerade -# dnsmasq DHCP config (idempotent) +# dnsmasq DHCPv4 + RA/DHCPv6 config (idempotent) mkdir -p /etc/dnsmasq.d CFG=/etc/dnsmasq.d/hero-$BR.conf TMP=/etc/dnsmasq.d/.hero-$BR.conf.new + +# Always include IPv4 section cat >\"$TMP\" <>\"$TMP\" <<'EOFV6' +enable-ra +dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h +dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111] +EOFV6 + sed -i \"s/BR_PLACEHOLDER/$BR/g\" \"$TMP\" +fi + if [ ! -f \"$CFG\" ] || ! cmp -s \"$CFG\" \"$TMP\"; then mv \"$TMP\" \"$CFG\" if systemctl is-active --quiet dnsmasq; then @@ -765,9 +867,13 @@ fi subnet = shell_escape(subnet_cidr), dstart = shell_escape(dhcp_start), dend = shell_escape(dhcp_end), + v6cidr = shell_escape(ipv6_cidr), ); - match sal_process::run(&script).silent(true).execute() { + // Use a unique heredoc delimiter to avoid clashing with inner < Ok(()), Ok(res) => Err(CloudHvError::CommandFailed(format!( "Host networking setup failed: {}", diff --git a/packages/system/virt/src/hostcheck/mod.rs b/packages/system/virt/src/hostcheck/mod.rs index 330eaf0..60108b1 100644 --- a/packages/system/virt/src/hostcheck/mod.rs +++ b/packages/system/virt/src/hostcheck/mod.rs @@ -140,10 +140,53 @@ pub fn host_check_deps() -> Result { let _ = fs::remove_file(&probe_path); } } - + + // Optional Mycelium IPv6 checks when enabled via env + let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into()); + let ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true"); + if ipv6_enabled { + // Require mycelium CLI + if bin_missing("mycelium") { + critical.push("mycelium CLI not found on PATH (required when HERO_VIRT_IPV6_ENABLE=true)".into()); + } + // Validate interface presence and global IPv6 + let ifname = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into()); + let check_if = sal_process::run(&format!("ip -6 addr show dev {}", ifname)) + .silent(true) + .die(false) + .execute(); + match check_if { + Ok(r) if r.success => { + let out = r.stdout; + if !(out.contains("inet6") && out.contains("scope global")) { + notes.push(format!( + "iface '{}' present but no global IPv6 detected; Mycelium may not be up yet", + ifname + )); + } + } + _ => { + critical.push(format!( + "iface '{}' not found or no IPv6; ensure Mycelium is running", + ifname + )); + } + } + // Best-effort: parse `mycelium inspect` for Address + let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute(); + match insp { + Ok(res) if res.success && res.stdout.contains("Address:") => { + // good enough + } + _ => { + notes.push("`mycelium inspect` did not return an Address; IPv6 overlay may be unavailable".into()); + } + } + } + // Summarize ok flag let ok = critical.is_empty(); - + Ok(HostCheckReport { ok, critical, diff --git a/packages/system/virt/src/image_prep/mod.rs b/packages/system/virt/src/image_prep/mod.rs index 66b5c0d..8b403b3 100644 --- a/packages/system/virt/src/image_prep/mod.rs +++ b/packages/system/virt/src/image_prep/mod.rs @@ -52,9 +52,9 @@ impl Default for NetPlanOpts { fn default() -> Self { Self { dhcp4: true, - dhcp6: false, - ipv6_addr: Some("400::10/64".into()), - gw6: Some("400::1".into()), + dhcp6: true, + ipv6_addr: None, + gw6: None, } } } @@ -154,8 +154,6 @@ pub fn image_prepare(opts: &ImagePrepOptions) -> Result { // Build bash script that performs all steps and echos "RAW|ROOT_UUID|BOOT_UUID" at end - let net_ipv6 = opts.net.ipv6_addr.clone().unwrap_or_else(|| "400::10/64".into()); - let gw6 = opts.net.gw6.clone().unwrap_or_else(|| "400::1".into()); let disable_ci_net = opts.disable_cloud_init_net; // Keep script small and robust; avoid brace-heavy awk to simplify escaping. @@ -332,11 +330,6 @@ network: eth0: dhcp4: {dhcp4} dhcp6: {dhcp6} - addresses: - - {ipv6} - routes: - - to: \"::/0\" - via: {gw6} nameservers: addresses: [8.8.8.8, 1.1.1.1, 2001:4860:4860::8888] EOF @@ -370,8 +363,6 @@ exit 0 raw = shell_escape(&raw_path), dhcp4 = if opts.net.dhcp4 { "true" } else { "false" }, dhcp6 = if opts.net.dhcp6 { "true" } else { "false" }, - ipv6 = shell_escape(&net_ipv6), - gw6 = shell_escape(&gw6), disable_ci_net = if disable_ci_net { "true" } else { "false" }, ); diff --git a/packages/system/virt/tests/rhai/10_vm_end_to_end.rhai b/packages/system/virt/tests/rhai/10_vm_end_to_end.rhai index 5e350ae..cda4977 100644 --- a/packages/system/virt/tests/rhai/10_vm_end_to_end.rhai +++ b/packages/system/virt/tests/rhai/10_vm_end_to_end.rhai @@ -17,7 +17,7 @@ // /images/noble-server-cloudimg-amd64.img // /images/alpine-virt-cloudimg-amd64.qcow2 (Alpine prepare not implemented yet) // /images/hypervisor-fw (firmware binary used via --kernel) -// - Network defaults: IPv4 NAT + dnsmasq DHCP; placeholder IPv6 on bridge + guest netplan. + // - Network defaults: IPv4 NAT (dnsmasq DHCP) + IPv6 routed over Mycelium (RA/DHCPv6). No static IPv6 is written into the guest; it autoconfigures via RA. // // Conventions: // - Functional builder chaining: b = memory_mb(b, 4096), etc. @@ -81,12 +81,6 @@ let prep_opts = #{ flavor: "ubuntu", // source: optional override, default uses /images/noble-server-cloudimg-amd64.img // target_dir: optional override, default $HOME/hero/virt/vms/ - net: #{ - dhcp4: true, - dhcp6: false, - ipv6_addr: "400::10/64", - gw6: "400::1", - }, disable_cloud_init_net: true, }; @@ -150,6 +144,7 @@ try { fail("cloudhv_vm_list failed: " + e.to_string()); } +sleep(1000000); // ------------------------------------------------------------------------------------ // Phase 5: Stop & delete VM A // ------------------------------------------------------------------------------------