herocode/herolib_rust
12
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

working ipv6 ip assignment + ssh with login/passwd

Browse Source
This commit is contained in:
Maxime Van Hees
2025-08-28 15:19:37 +02:00
parent 784f87db97
commit da3da0ae30
2 changed files with 302 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
packages/system/virt/src/cloudhv/mod.rs
View File

@@ -408,27 +408,35 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
let dhcp_end =
std::env::var("HERO_VIRT_DHCP_END").unwrap_or_else(|_| "172.30.0.250".into());
// Optional IPv6 over Mycelium provisioning (bridge P::2/64 + RA) guarded by env and detection.
// IPv6 over Mycelium: enabled by default.
// If explicitly disabled via HERO_VIRT_IPV6_ENABLE=false|0, we skip.
// If enabled but Mycelium is not detected, return an error.
let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into());
let mut ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true");
let ipv6_requested = match ipv6_env.to_lowercase().as_str() {
"" | "1" | "true" | "yes" => true,
"0" | "false" | "no" => false,
_ => true,
};
let mycelium_if_cfg = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into());
let mut ipv6_bridge_cidr: Option<String> = None;
let mut mycelium_if_opt: Option<String> = None;
let enable_auto = ipv6_env.is_empty(); // auto-enable if mycelium is detected and not explicitly disabled
if ipv6_enabled || enable_auto {
if ipv6_requested {
if let Ok(cidr) = std::env::var("HERO_VIRT_IPV6_BRIDGE_CIDR") {
// Explicit override for bridge IPv6 (e.g., "400:...::2/64")
// Explicit override for bridge IPv6 (e.g., "400:...::2/64") but still require mycelium iface presence.
// Validate mycelium interface and that it has IPv6 configured.
let _ = get_mycelium_ipv6_addr(&mycelium_if_cfg)?; // returns DependencyMissing on failure
ipv6_bridge_cidr = Some(cidr);
mycelium_if_opt = Some(mycelium_if_cfg.clone());
ipv6_enabled = true;
} else if let Ok((ifname, myc_addr)) = get_mycelium_ipv6_addr(&mycelium_if_cfg) {
// Derive P::2/64 from the mycelium node address
if let Ok((_pfx, router_cidr)) = derive_ipv6_prefix_from_mycelium(&myc_addr) {
ipv6_bridge_cidr = Some(router_cidr);
mycelium_if_opt = Some(ifname);
ipv6_enabled = true;
}
} else {
// Auto-derive from mycelium node address; error out if not detected.
println!("auto-deriving mycelium address...");
let (ifname, myc_addr) = get_mycelium_ipv6_addr(&mycelium_if_cfg)?;
println!("on if {ifname}, got myc addr: {myc_addr}");
let (_pfx, router_cidr) = derive_ipv6_prefix_from_mycelium(&myc_addr)?;
println!("derived pfx: {_pfx} and router cidr: {router_cidr}");
ipv6_bridge_cidr = Some(router_cidr);
mycelium_if_opt = Some(ifname);
}
}
@@ -445,8 +453,10 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
// Ensure a TAP device for this VM and attach to the bridge
let tap_name = ensure_tap_for_vm(&bridge_name, id)?;
println!("TAP device for vm called: {tap_name}");
// Stable locally-administered MAC derived from VM id
let mac = stable_mac_from_id(id);
println!("MAC for vm: {mac}");
parts.push("--net".into());
parts.push(format!("tap={},mac={}", tap_name, mac));
@@ -465,6 +475,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
log_file,
vm_pid_path(id).to_string_lossy()
);
println!("executing command:\n{heredoc}");
// Execute command; this will background cloud-hypervisor and return
let result = sal_process::run(&heredoc).execute();
match result {
@@ -489,6 +500,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
Ok(s) => s.trim().parse::<i64>().ok(),
Err(_) => None,
};
println!("reading PID back: {} - (if 0 == not found)", pid.unwrap_or(0));
// Quick health check: ensure process did not exit immediately due to CLI errors (e.g., duplicate flags)
if let Some(pid_num) = pid {
@@ -496,6 +508,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
if !proc_exists(pid_num) {
// Tail log to surface the error cause
let tail_cmd = format!("tail -n 200 {}", shell_escape(&log_file));
println!("executing tail_cmd command:\n{tail_cmd}");
let tail = sal_process::run(&tail_cmd).die(false).silent(true).execute();
let mut log_snip = String::new();
if let Ok(res) = tail {
@@ -524,6 +537,76 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
let value = serde_json::to_value(&rec).map_err(|e| CloudHvError::JsonError(e.to_string()))?;
write_json(&vm_json_path(id), &value)?;
println!("wrote JSON for VM");
// Best-effort: discover and print guest IPv4/IPv6 addresses (default-net path)
// Give DHCP/ND a moment
println!("waiting 5 secs for DHCP/ND");
thread::sleep(Duration::from_millis(5000));
let bridge_name = std::env::var("HERO_VIRT_BRIDGE_NAME").unwrap_or_else(|_| "br-hero".into());
let mac_lower = stable_mac_from_id(id).to_lowercase();
// IPv4 from dnsmasq leases (pinned per-bridge leasefile)
// Path set in ensure_host_net_prereq_dnsmasq_nftables: /var/lib/misc/dnsmasq-hero-$BR.leases
let lease_path = std::env::var("HERO_VIRT_DHCP_LEASE_FILE")
.unwrap_or_else(|_| format!("/var/lib/misc/dnsmasq-hero-{}.leases", bridge_name));
// Parse dnsmasq leases directly to avoid shell quoting/pipelines
let ipv4 = (|| {
let deadline = std::time::Instant::now() + Duration::from_secs(12);
loop {
if let Ok(content) = fs::read_to_string(&lease_path) {
let mut last_ip: Option<String> = None;
for line in content.lines() {
let cols: Vec<&str> = line.split_whitespace().collect();
if cols.len() >= 3 && cols[1].eq_ignore_ascii_case(&mac_lower) {
last_ip = Some(cols[2].to_string());
}
}
if last_ip.is_some() {
return last_ip;
}
}
if std::time::Instant::now() >= deadline {
return None;
}
thread::sleep(Duration::from_millis(800));
}
})();
println!(
"Got IPv4 from dnsmasq lease ({}): {}",
lease_path,
ipv4.clone().unwrap_or("not found".to_string())
);
// IPv6 from neighbor table on the bridge (exclude link-local), parsed in Rust
let ipv6 = (|| {
let cmd = format!("ip -6 neigh show dev {}", bridge_name);
if let Ok(res) = sal_process::run(&cmd).silent(true).die(false).execute() {
if res.success {
let mac_pat = format!("lladdr {}", mac_lower);
for line in res.stdout.lines() {
let lt = line.trim();
if lt.to_lowercase().contains(&mac_pat) {
if let Some(addr) = lt.split_whitespace().next() {
if !addr.starts_with("fe80") && !addr.is_empty() {
return Some(addr.to_string());
}
}
}
}
}
}
None
})();
println!("Got IPv6 from neighbor table on bridge: {}", ipv6.clone().unwrap_or("not found".to_string()));
println!(
"[cloudhv] VM '{}' guest addresses: IPv4={}, IPv6={}",
id,
ipv4.as_deref().unwrap_or(""),
ipv6.as_deref().unwrap_or("")
);
Ok(())
}
@@ -711,50 +794,44 @@ fn stable_mac_from_id(id: &str) -> String {
format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5)
}
/// Discover the mycelium IPv6 address and validate the interface.
///
/// Returns (interface_name, address_string).
/// Discover the mycelium IPv6 address by inspecting the interface itself (no CLI dependency).
/// Returns (interface_name, first global IPv6 address found on the interface).
fn get_mycelium_ipv6_addr(iface_hint: &str) -> Result<(String, String), CloudHvError> {
// Parse `mycelium inspect` for "Address: ..."
let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute();
let mut addr: Option<String> = None;
if let Ok(res) = insp {
if res.success {
for l in res.stdout.lines() {
let lt = l.trim();
if let Some(rest) = lt.strip_prefix("Address:") {
let val = rest.trim().trim_matches('"').to_string();
if !val.is_empty() {
addr = Some(val);
break;
}
}
}
}
}
let iface = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| iface_hint.to_string());
// Validate interface exists and has IPv6 configured (best-effort)
// Query IPv6 addresses on the interface
let cmd = format!("ip -6 addr show dev {}", shell_escape(&iface));
match sal_process::run(&cmd).silent(true).die(false).execute() {
Ok(r) if r.success => {
// proceed
}
let res = sal_process::run(&cmd).silent(true).die(false).execute();
let out = match res {
Ok(r) if r.success => r.stdout,
_ => {
return Err(CloudHvError::DependencyMissing(format!(
"mycelium interface '{}' not found or no IPv6 configured",
iface
)));
)))
}
};
// Extract the first global IPv6 address present on the interface.
for line in out.lines() {
let lt = line.trim();
// Example line: "inet6 578:9fcf:.../7 scope global"
if lt.starts_with("inet6 ") && lt.contains("scope global") {
let parts: Vec<&str> = lt.split_whitespace().collect();
if let Some(addr_cidr) = parts.get(1) {
let addr_only = addr_cidr.split('/').next().unwrap_or("").trim();
println!("got addr from host: {addr_only}");
if !addr_only.is_empty() && addr_only.parse::<std::net::Ipv6Addr>().is_ok() {
return Ok((iface, addr_only.to_string()));
}
}
}
}
if let Some(a) = addr {
Ok((iface, a))
} else {
Err(CloudHvError::DependencyMissing(
"failed to read mycelium IPv6 address via `mycelium inspect`".into(),
))
}
Err(CloudHvError::DependencyMissing(format!(
"no global IPv6 found on interface '{}'",
iface
)))
}
/// Derive a /64 prefix P from the mycelium IPv6 and return (P/64, P::2/64).
@@ -803,10 +880,15 @@ SUBNET={subnet}
DHCP_START={dstart}
DHCP_END={dend}
IPV6_CIDR={v6cidr}
LEASE_FILE=/var/lib/misc/dnsmasq-hero-$BR.leases
# Determine default WAN interface
WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1)
if [ -z \"$WAN_IF\" ]; then
echo \"No default WAN interface detected (required for IPv4 NAT)\" >&2
exit 2
fi
# Bridge creation (idempotent)
ip link show \"$BR\" >/dev/null 2>&1 || ip link add name \"$BR\" type bridge
ip addr replace \"$BR_ADDR\" dev \"$BR\"
@@ -829,24 +911,38 @@ nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN
# dnsmasq DHCPv4 + RA/DHCPv6 config (idempotent)
mkdir -p /etc/dnsmasq.d
mkdir -p /var/lib/misc
CFG=/etc/dnsmasq.d/hero-$BR.conf
TMP=/etc/dnsmasq.d/.hero-$BR.conf.new
# Always include IPv4 section
cat >\"$TMP\" <<EOF
interface=$BR
bind-interfaces
dhcp-range=$DHCP_START,$DHCP_END,12h
dhcp-option=option:dns-server,1.1.1.1,8.8.8.8
EOF
RELOAD=0
CONF=/etc/dnsmasq.conf
# Ensure conf-dir includes /etc/dnsmasq.d (simple fixed-string check to avoid regex escapes in Rust)
if ! grep -qF \"conf-dir=/etc/dnsmasq.d\" \"$CONF\"; then
printf '%s\n' 'conf-dir=/etc/dnsmasq.d,*.conf' >> \"$CONF\"
RELOAD=1
fi
# Ensure lease file exists and is writable by dnsmasq user
touch \"$LEASE_FILE\" || true
chown dnsmasq:dnsmasq \"$LEASE_FILE\" 2>/dev/null || true
# Always include IPv4 section
printf '%s\n' \
\"interface=$BR\" \
\"bind-interfaces\" \
\"dhcp-authoritative\" \
\"dhcp-range=$DHCP_START,$DHCP_END,12h\" \
\"dhcp-option=option:dns-server,1.1.1.1,8.8.8.8\" \
\"dhcp-leasefile=$LEASE_FILE\" >\"$TMP\"
# Optionally append IPv6 RA/DHCPv6
if [ -n \"$IPV6_CIDR\" ]; then
cat >>\"$TMP\" <<'EOFV6'
enable-ra
dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h
dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111]
EOFV6
printf '%s\n' \
\"enable-ra\" \
\"dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h\" \
\"dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111]\" >>\"$TMP\"
sed -i \"s/BR_PLACEHOLDER/$BR/g\" \"$TMP\"
fi
@@ -861,6 +957,11 @@ else
rm -f \"$TMP\"
systemctl enable --now dnsmasq || true
fi
# Reload if main conf was updated to include conf-dir
if [ \"$RELOAD\" = \"1\" ]; then
systemctl reload dnsmasq || systemctl restart dnsmasq || true
fi
",
br = shell_escape(bridge_name),
br_addr = shell_escape(bridge_addr_cidr),
@@ -872,6 +973,7 @@ fi
// Use a unique heredoc delimiter to avoid clashing with inner <<EOF blocks
let heredoc_net = format!("bash -e -s <<'HERONET'\n{}\nHERONET\n", body);
println!("executing command:\n{heredoc_net}");
match sal_process::run(&heredoc_net).silent(true).execute() {
Ok(res) if res.success => Ok(()),

143
packages/system/virt/src/image_prep/mod.rs
View File

@@ -4,6 +4,9 @@ use std::path::Path;
use sal_os;
use sal_process;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::net::Ipv6Addr;
#[derive(Debug)]
pub enum ImagePrepError {
@@ -80,6 +83,19 @@ fn default_disable_cloud_init_net() -> bool {
true
}
fn stable_mac_from_id(id: &str) -> String {
let mut h = DefaultHasher::new();
id.hash(&mut h);
let v = h.finish();
let b0 = (((v >> 40) & 0xff) as u8 & 0xfe) | 0x02; // locally administered, unicast
let b1 = ((v >> 32) & 0xff) as u8;
let b2 = ((v >> 24) & 0xff) as u8;
let b3 = ((v >> 16) & 0xff) as u8;
let b4 = ((v >> 8) & 0xff) as u8;
let b5 = (v & 0xff) as u8;
format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImagePrepResult {
pub raw_disk: String,
@@ -156,7 +172,67 @@ pub fn image_prepare(opts: &ImagePrepOptions) -> Result<ImagePrepResult, ImagePr
// Build bash script that performs all steps and echos "RAW|ROOT_UUID|BOOT_UUID" at end
let disable_ci_net = opts.disable_cloud_init_net;
// IPv6 static guest assignment (derive from mycelium interface) - enabled by default
// If HERO_VIRT_IPV6_STATIC_GUEST=false, keep dynamic behavior (SLAAC/DHCPv6).
let static_v6 = std::env::var("HERO_VIRT_IPV6_STATIC_GUEST")
.map(|v| matches!(v.to_lowercase().as_str(), "" | "1" | "true" | "yes"))
.unwrap_or(true);
let myc_if = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into());
// Discover host mycelium global IPv6 in 400::/7 from the interface
let mut host_v6: Option<Ipv6Addr> = None;
if static_v6 {
let cmd = format!("ip -6 addr show dev {}", shell_escape(&myc_if));
if let Ok(r) = sal_process::run(&cmd).silent(true).die(false).execute() {
if r.success {
for l in r.stdout.lines() {
let lt = l.trim();
if lt.starts_with("inet6 ") && lt.contains("scope global") {
if let Some(addr_cidr) = lt.split_whitespace().nth(1) {
let addr_only = addr_cidr.split('/').next().unwrap_or("").trim();
if let Ok(ip) = addr_only.parse::<Ipv6Addr>() {
let seg0 = ip.segments()[0];
if (seg0 & 0xFE00) == 0x0400 {
host_v6 = Some(ip);
break;
}
}
}
}
}
}
}
}
// Derive per-host /64 from mycelium and deterministic per-VM guest address
let mut np_v6_block = String::new();
let mut dhcp6_effective = opts.net.dhcp6;
if static_v6 {
if let Some(h) = host_v6 {
let seg = h.segments();
// Router = P::2; Guest address = P::<stable suffix>
let mut hasher = DefaultHasher::new();
opts.id.hash(&mut hasher);
let mut suffix = (hasher.finish() as u16) & 0xfffe;
if suffix == 0 || suffix == 2 {
suffix = 0x100;
}
let guest_ip = Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, suffix).to_string();
let gw_ip = Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 2).to_string();
// Inject a YAML block for static v6
np_v6_block = format!(
" addresses:\n - {}/64\n routes:\n - to: \"::/0\"\n via: {}\n",
guest_ip, gw_ip
);
// Disable dhcp6 when we provide a static address
dhcp6_effective = false;
}
}
// Keep script small and robust; avoid brace-heavy awk to simplify escaping.
// Compute stable MAC (must match what vm_start() uses) and use it to match NIC in netplan.
let vm_mac = stable_mac_from_id(&opts.id);
let script = format!(
"#!/bin/bash -e
set -euo pipefail
@@ -328,11 +404,69 @@ network:
version: 2
ethernets:
eth0:
match:
macaddress: {vm_mac}
set-name: eth0
dhcp4: {dhcp4}
dhcp6: {dhcp6}
nameservers:
{np_v6_block} nameservers:
addresses: [8.8.8.8, 1.1.1.1, 2001:4860:4860::8888]
EOF
# Enable SSH password authentication and set a default password for 'ubuntu'
mkdir -p \"$MNT_ROOT/etc/cloud/cloud.cfg.d\"
printf '%s\n' 'ssh_pwauth: true' > \"$MNT_ROOT/etc/cloud/cloud.cfg.d/99-ssh-password-auth.cfg\"
mkdir -p \"$MNT_ROOT/etc/ssh/sshd_config.d\"
cat > \"$MNT_ROOT/etc/ssh/sshd_config.d/99-hero-password-auth.conf\" << EOF
PasswordAuthentication yes
KbdInteractiveAuthentication yes
UsePAM yes
EOF
# Set password for default user 'ubuntu'
if chroot \"$MNT_ROOT\" getent passwd ubuntu >/dev/null 2>&1; then
chroot \"$MNT_ROOT\" bash -c \"echo 'ubuntu:ubuntu' | chpasswd\" || true
fi
# Ensure openssh-server is present (some cloud images may omit it)
# Ensure SSH service enabled and keys generated on boot
chroot \"$MNT_ROOT\" systemctl unmask ssh 2>/dev/null || true
chroot \"$MNT_ROOT\" systemctl enable ssh 2>/dev/null || true
chroot \"$MNT_ROOT\" systemctl enable ssh-keygen.service 2>/dev/null || true
# Ensure sshd listens on both IPv4 and IPv6 explicitly
cat > \"$MNT_ROOT/etc/ssh/sshd_config.d/99-hero-address-family.conf\" << EOF
AddressFamily any
ListenAddress ::
ListenAddress 0.0.0.0
EOF
# If UFW is present, allow SSH and disable firewall (for tests)
if chroot \"$MNT_ROOT\" command -v ufw >/dev/null 2>&1; then
chroot \"$MNT_ROOT\" ufw allow OpenSSH || true
chroot \"$MNT_ROOT\" ufw disable || true
fi
if ! chroot \"$MNT_ROOT\" test -x /usr/sbin/sshd; then
cp -f /etc/resolv.conf \"$MNT_ROOT/etc/resolv.conf\" 2>/dev/null || true
chroot \"$MNT_ROOT\" bash -c \"apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server\" || true
fi
# Ensure default user 'ubuntu' exists (fallback for minimal images)
if ! chroot \"$MNT_ROOT\" id -u ubuntu >/dev/null 2>&1; then
chroot \"$MNT_ROOT\" useradd -m -s /bin/bash ubuntu || true
echo \"ubuntu ALL=(ALL) NOPASSWD:ALL\" > \"$MNT_ROOT/etc/sudoers.d/90-ubuntu\" || true
chmod 0440 \"$MNT_ROOT/etc/sudoers.d/90-ubuntu\" || true
fi
# Re-assert password (covers both existing and newly created users)
if chroot \"$MNT_ROOT\" getent passwd ubuntu >/dev/null 2>&1; then
chroot \"$MNT_ROOT\" bash -c \"echo 'ubuntu:ubuntu' | chpasswd\" || true
fi
# Pre-generate host SSH keys so sshd can start immediately
chroot \"$MNT_ROOT\" ssh-keygen -A 2>/dev/null || true
mkdir -p \"$MNT_ROOT/var/run/sshd\"
# Also enable socket activation as a fallback
chroot \"$MNT_ROOT\" systemctl enable ssh.socket 2>/dev/null || true
# Disable cloud-init networking (optional but default)
if [ \"{disable_ci_net}\" = \"true\" ]; then
@@ -361,11 +495,16 @@ exit 0
mnt_root = shell_escape(&mnt_root),
mnt_boot = shell_escape(&mnt_boot),
raw = shell_escape(&raw_path),
vm_mac = vm_mac,
dhcp4 = if opts.net.dhcp4 { "true" } else { "false" },
dhcp6 = if opts.net.dhcp6 { "true" } else { "false" },
dhcp6 = if dhcp6_effective { "true" } else { "false" },
np_v6_block = np_v6_block,
disable_ci_net = if disable_ci_net { "true" } else { "false" },
);
// image prep script printout for debugging:
println!("{script}");
let res = run_script(&script)?;
// Prefer a RESULT:-prefixed line (robust against extra stdout noise)
let mut marker: Option<String> = None;