Files
zosbuilder/scripts/lib/components.sh
Jan De Landtsheer 334821dacf
Some checks failed
Build Zero OS Initramfs / build (push) Has been cancelled
Build Zero OS Initramfs / test-matrix (qemu, basic) (push) Has been cancelled
Build Zero OS Initramfs / test-matrix (qemu, serial) (push) Has been cancelled
Integrate zosstorage build path and runtime orchestration
Summary:

* add openssh-client to the builder image and mount host SSH keys into the dev container when available

* switch RFS to git builds, register the zosstorage source, and document the extra Rust component

* wire zosstorage into the build: add build_zosstorage(), ship the binary in the initramfs, and extend component validation

* refresh kernel configuration to 6.12.49 while dropping Xen guest selections and enabling counted-by support

* tighten runtime configs: use cached mycelium key path, add zosstorage zinit unit, bootstrap ovsdb-server, and enable openvswitch module

* adjust the network health check ping invocation and fix the RFS pack-tree --debug flag order

* update NOTES changelog, README component list, and introduce a runit helper for qemu/cloud-hypervisor testing

* add ovsdb init script wiring under config/zinit/init and ensure zosstorage is available before mycelium
2025-10-14 17:47:13 +02:00

660 lines
22 KiB
Bash

#!/bin/bash
# Component download and build system for ThreeFold Zero OS
# Source common functions
LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${LIB_SCRIPT_DIR}/common.sh"
# Component configuration
RUST_TARGET="${RUST_TARGET:-x86_64-unknown-linux-musl}"
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}"
# Parse and process all components from sources.conf
function components_parse_sources_conf() {
local sources_file="$1"
local components_dir="$2"
local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}"
section_header "Parsing Sources Configuration"
if [[ ! -f "$sources_file" ]]; then
log_error "Sources file not found: ${sources_file}"
return 1
fi
# Ensure components directory exists
safe_mkdir "$components_dir"
# Export install directory for build functions
export INSTALL_DIR="$install_dir"
log_info "Processing components from: ${sources_file}"
log_info "Components directory: ${components_dir}"
log_info "Install directory: ${install_dir}"
local component_count=0
# Read entries from sources.conf (TYPE NAME URL VERSION BUILD_FUNCTION [EXTRA])
while IFS= read -r _raw || [[ -n "$_raw" ]]; do
# Strip comments and trim whitespace
local line="${_raw%%#*}"
line="${line#"${line%%[![:space:]]*}"}"
line="${line%"${line##*[![:space:]]}"}"
[[ -z "$line" ]] && continue
local type name url version build_func extra
# shellcheck disable=SC2086
read -r type name url version build_func extra <<< "$line"
if [[ -z "${type:-}" || -z "${name:-}" || -z "${url:-}" || -z "${version:-}" || -z "${build_func:-}" ]]; then
log_warn "Skipping malformed entry: ${_raw}"
continue
fi
component_count=$((component_count + 1))
log_info "Processing component ${component_count}: ${name} (${type})"
case "$type" in
git)
components_download_git "$name" "$url" "$version" "$components_dir"
;;
release)
components_download_release "$name" "$url" "$version" "$components_dir" "$extra"
;;
*)
log_error "Unknown component type in sources.conf: ${type}"
return 1
;;
esac
components_build_component "$name" "$build_func" "$components_dir"
done < "$sources_file"
if [[ $component_count -eq 0 ]]; then
log_warn "No components found in sources configuration"
else
log_info "Processed ${component_count} components successfully"
fi
}
# Download Git repository (reuse tree; only reclone if invalid or version not reachable)
function components_download_git() {
local name="$1"
local url="$2"
local version="$3"
local components_dir="$4"
section_header "Downloading Git Component: ${name}"
local target_dir="${components_dir}/${name}"
log_info "Repository: ${url}"
log_info "Version/Branch/Tag: ${version}"
log_info "Target directory: ${target_dir}"
# Ensure parent exists
safe_mkdir "$components_dir"
# Decide whether we can reuse the existing working tree
local need_fresh_clone="0"
if [[ -d "$target_dir/.git" ]]; then
if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
log_warn "Existing ${name} directory is not a valid git repo; will reclone"
need_fresh_clone="1"
fi
elif [[ -d "$target_dir" ]]; then
log_warn "Existing ${name} directory without .git; will reclone"
need_fresh_clone="1"
fi
if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then
log_info "Cloning ${name} (fresh) from ${url}"
safe_execute git clone "$url" "$target_dir"
fi
# Ensure origin URL is correct (do not delete the tree if URL changed)
local current_url
current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "")
if [[ -n "$current_url" && "$current_url" != "$url" ]]; then
log_info "Updating origin URL: ${current_url} -> ${url}"
safe_execute git -C "$target_dir" remote set-url origin "$url"
elif [[ -z "$current_url" ]]; then
log_info "Setting origin URL to ${url}"
safe_execute git -C "$target_dir" remote add origin "$url" || true
fi
# Fetch updates and tags
safe_execute git -C "$target_dir" fetch --tags --prune origin
# Resolve desired commit for the requested version/branch/tag
local desired_rev=""
if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then
desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}")
elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then
desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}")
else
log_warn "Version '${version}' not directly resolvable; fetching explicitly"
if git -C "$target_dir" fetch origin "${version}" --depth 1; then
desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD)
fi
fi
# Fallback: shallow clone at the requested ref if we still can't resolve
if [[ -z "$desired_rev" ]]; then
log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref."
safe_execute rm -rf "${target_dir}.tmp"
if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then
safe_execute rm -rf "$target_dir"
safe_execute mv "${target_dir}.tmp" "$target_dir"
desired_rev=$(git -C "$target_dir" rev-parse HEAD)
else
log_error "Failed to clone ${url} at '${version}'"
return 1
fi
fi
local current_rev
current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "")
log_info "Current commit: ${current_rev:-<none>}"
log_info "Desired commit: ${desired_rev}"
if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then
log_info "Repository already at requested version; reusing working tree"
else
log_info "Checking out requested version"
# Prefer named refs when available; otherwise detach to exact commit
if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then
safe_execute git -C "$target_dir" checkout -f "${version}"
elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then
safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}"
elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then
safe_execute git -C "$target_dir" checkout -f "tags/${version}"
else
safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}"
fi
# Initialize submodules if present (non-fatal)
safe_execute git -C "$target_dir" submodule update --init --recursive || true
fi
log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)"
}
# Download release binary/archive
function components_download_release() {
local name="$1"
local url="$2"
local version="$3"
local components_dir="$4"
local extra="$5"
section_header "Downloading Release Component: ${name}"
local target_dir="${components_dir}/${name}"
local filename=$(basename "$url")
log_info "Release URL: ${url}"
log_info "Version: ${version}"
log_info "Target directory: ${target_dir}"
safe_mkdir "$target_dir"
# Download release
log_info "Downloading release: ${filename}"
safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url"
# Verify download
if [[ ! -f "${target_dir}/${filename}" ]]; then
log_error "Failed to download release: ${filename}"
return 1
fi
local file_size=$(get_file_size "${target_dir}/${filename}")
log_info "Downloaded file size: ${file_size}"
# Handle extra options (like rename)
if [[ -n "$extra" ]]; then
components_process_extra_options "$target_dir" "$filename" "$extra"
fi
log_info "Release component download complete: ${name}"
}
# Process extra options for components
function components_process_extra_options() {
local target_dir="$1"
local filename="$2"
local extra="$3"
log_info "Processing extra options: ${extra}"
# Handle rename option
if [[ "$extra" =~ rename=(.+) ]]; then
local new_name="${BASH_REMATCH[1]}"
log_info "Renaming ${filename} to ${new_name}"
safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}"
fi
# Handle extract option for archives
if [[ "$extra" =~ extract ]]; then
log_info "Extracting archive: ${filename}"
safe_execute cd "$target_dir"
case "$filename" in
*.tar.gz|*.tgz)
safe_execute tar -xzf "$filename"
;;
*.tar.bz2|*.tbz2)
safe_execute tar -xjf "$filename"
;;
*.tar.xz|*.txz)
safe_execute tar -xJf "$filename"
;;
*.zip)
safe_execute unzip "$filename"
;;
*)
log_warn "Unknown archive format: ${filename}"
;;
esac
fi
}
# Build component using specified build function
function components_build_component() {
local name="$1"
local build_func="$2"
local components_dir="$3"
section_header "Building Component: ${name}"
local component_dir="${components_dir}/${name}"
if [[ ! -d "$component_dir" ]]; then
log_error "Component directory not found: ${component_dir}"
return 1
fi
# Change to component directory
safe_execute cd "$component_dir"
log_info "Build function: ${build_func}"
log_info "Working directory: $(pwd)"
# Check if build function exists
if ! declare -f "$build_func" >/dev/null; then
log_error "Build function not found: ${build_func}"
return 1
fi
# Call the specific build function
log_info "Executing build function: ${build_func}"
"$build_func" "$name" "$component_dir"
log_info "Component build complete: ${name}"
}
# Setup Rust environment for musl builds
function components_setup_rust_env() {
section_header "Setting Up Rust Environment"
# Source cargo environment if available
if [[ -f /root/.cargo/env ]]; then
log_info "Sourcing cargo environment from /root/.cargo/env"
source /root/.cargo/env
fi
# Check if we have rustup (should be available now)
if command_exists "rustup"; then
log_info "Using rustup for Rust toolchain management"
# Ensure musl target is installed
if ! rustup target list --installed | grep -q "$RUST_TARGET"; then
log_info "Installing Rust target: ${RUST_TARGET}"
safe_execute rustup target add "$RUST_TARGET"
else
log_info "Rust target already installed: ${RUST_TARGET}"
fi
# Set environment variables for rustup (clean and simple)
export RUSTFLAGS="-C target-feature=+crt-static"
else
log_error "rustup not found after setup"
return 1
fi
log_info "Rust environment configured for musl builds"
log_info "RUST_TARGET: ${RUST_TARGET}"
log_info "RUSTFLAGS: ${RUSTFLAGS}"
log_info "CC: ${CC:-system-default}"
}
# Build function for zinit (standard Rust build)
function build_zinit() {
local name="$1"
local component_dir="$2"
section_header "Building zinit with musl target"
components_setup_rust_env
log_info "Building zinit from: ${component_dir}"
# Ensure we're in the correct directory
if [[ ! -d "$component_dir" ]]; then
log_error "Component directory not found: ${component_dir}"
return 1
fi
# Don't use safe_execute for cd - it runs in subshell
log_info "Executing: cd $component_dir"
cd "$component_dir" || {
log_error "Failed to change to directory: $component_dir"
return 1
}
local current_dir=$(pwd)
log_info "Current directory: ${current_dir}"
# Verify Cargo.toml exists
if [[ ! -f "Cargo.toml" ]]; then
log_error "Cargo.toml not found in: ${current_dir}"
return 1
fi
# Build with musl target (rustup properly configured)
safe_execute cargo build --release --target "$RUST_TARGET"
# Find and install binary
local binary_path="target/${RUST_TARGET}/release/zinit"
if [[ ! -f "$binary_path" ]]; then
log_error "zinit binary not found at: ${binary_path}"
return 1
fi
local binary_size=$(get_file_size "$binary_path")
log_info "Built zinit binary (${binary_size}) at: ${binary_path}"
}
# Build function for rfs (standard Rust build)
function build_rfs() {
local name="$1"
local component_dir="$2"
section_header "Building rfs with musl target"
components_setup_rust_env
log_info "Building rfs from: ${component_dir}"
# Ensure we're in the correct directory
if [[ ! -d "$component_dir" ]]; then
log_error "Component directory not found: ${component_dir}"
return 1
fi
# Don't use safe_execute for cd - it runs in subshell
log_info "Executing: cd $component_dir"
cd "$component_dir" || {
log_error "Failed to change to directory: $component_dir"
return 1
}
local current_dir=$(pwd)
log_info "Current directory: ${current_dir}"
# Verify Cargo.toml exists
if [[ ! -f "Cargo.toml" ]]; then
log_error "Cargo.toml not found in: ${current_dir}"
return 1
fi
# remove rust-toolchain.toml, as not needed with latest release
# Build with musl target
safe_execute cargo build --release --target "$RUST_TARGET" --features build-binary
# Find and install binary
local binary_path="target/${RUST_TARGET}/release/rfs"
if [[ ! -f "$binary_path" ]]; then
log_error "rfs binary not found at: ${binary_path}"
return 1
fi
local binary_size=$(get_file_size "$binary_path")
log_info "Built rfs binary (${binary_size}) at: ${binary_path}"
}
# Build function for zosstorage (standard Rust build)
function build_zosstorage() {
local name="$1"
local component_dir="$2"
section_header "Building zosstorage with musl target"
components_setup_rust_env
log_info "Building zosstorage from: ${component_dir}"
if [[ ! -d "$component_dir" ]]; then
log_error "Component directory not found: ${component_dir}"
return 1
fi
log_info "Executing: cd $component_dir"
cd "$component_dir" || {
log_error "Failed to change to directory: $component_dir"
return 1
}
local current_dir
current_dir=$(pwd)
log_info "Current directory: ${current_dir}"
if [[ ! -f "Cargo.toml" ]]; then
log_error "Cargo.toml not found in: ${current_dir}"
return 1
fi
safe_execute cargo build --release --target "$RUST_TARGET"
local binary_path="target/${RUST_TARGET}/release/zosstorage"
if [[ ! -f "$binary_path" ]]; then
log_error "zosstorage binary not found at: ${binary_path}"
return 1
fi
local binary_size
binary_size=$(get_file_size "$binary_path")
log_info "Built zosstorage binary (${binary_size}) at: ${binary_path}"
}
# Build function for mycelium (special subdirectory build)
function build_mycelium() {
local name="$1"
local component_dir="$2"
section_header "Building mycelium with musl target (special directory)"
components_setup_rust_env
log_info "Building mycelium from: ${component_dir}"
# Change to myceliumd subdirectory (special requirement)
local myceliumd_dir="${component_dir}/myceliumd"
if [[ ! -d "$myceliumd_dir" ]]; then
log_error "myceliumd directory not found at: ${myceliumd_dir}"
return 1
fi
# Don't use safe_execute for cd - it runs in subshell
log_info "Executing: cd $myceliumd_dir"
cd "$myceliumd_dir" || {
log_error "Failed to change to myceliumd directory: $myceliumd_dir"
return 1
}
log_info "Building in myceliumd subdirectory: $(pwd)"
# Build with musl target
safe_execute cargo build --release --target "$RUST_TARGET"
# Find and install binary (from target/x86.../release)
local binary_path="target/${RUST_TARGET}/release/mycelium"
if [[ ! -f "$binary_path" ]]; then
log_error "mycelium binary not found at: ${binary_path}"
return 1
fi
local binary_size=$(get_file_size "$binary_path")
log_info "Built mycelium binary (${binary_size}) at: ${binary_path}"
}
# Install function for rfs (pre-built binary)
function install_rfs() {
local name="$1"
local component_dir="$2"
section_header "Installing rfs binary"
log_info "Installing rfs from: ${component_dir}"
# Find the rfs binary
local binary_path="${component_dir}/rfs"
if [[ ! -f "$binary_path" ]]; then
log_error "rfs binary not found at: ${binary_path}"
return 1
fi
# Make executable
safe_execute chmod +x "$binary_path"
local binary_size=$(get_file_size "$binary_path")
log_info "Prepared rfs binary (${binary_size}) at: ${binary_path}"
}
# Install function for corex (pre-built binary)
function install_corex() {
local name="$1"
local component_dir="$2"
section_header "Installing corex binary"
log_info "Installing corex from: ${component_dir}"
# Find the corex binary (may have been renamed)
local binary_path
if [[ -f "${component_dir}/corex" ]]; then
binary_path="${component_dir}/corex"
elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then
binary_path="${component_dir}/corex-2.1.4-amd64-linux-static"
else
log_error "corex binary not found in: ${component_dir}"
return 1
fi
# Make executable
safe_execute chmod +x "$binary_path"
local binary_size=$(get_file_size "$binary_path")
log_info "Prepared corex binary (${binary_size}) at: ${binary_path}"
}
# Verify all components are built (not copied to initramfs yet)
function components_verify_installation() {
local components_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}"
section_header "Verifying Component Build"
local ok_count=0
local missing_count=0
# zinit
local zinit_bin="${components_dir}/zinit/target/x86_64-unknown-linux-musl/release/zinit"
if [[ -x "$zinit_bin" ]]; then
log_info "✓ zinit ($(get_file_size "$zinit_bin")) at: ${zinit_bin#${components_dir}/}"
((ok_count++))
else
log_error "✗ zinit missing: ${zinit_bin#${components_dir}/}"
((missing_count++))
fi
# rfs: accept both built and prebuilt locations
local rfs_built="${components_dir}/rfs/target/x86_64-unknown-linux-musl/release/rfs"
local rfs_release="${components_dir}/rfs/rfs"
if [[ -x "$rfs_built" ]]; then
log_info "✓ rfs (built) ($(get_file_size "$rfs_built")) at: ${rfs_built#${components_dir}/}"
((ok_count++))
elif [[ -x "$rfs_release" ]]; then
log_info "✓ rfs (release) ($(get_file_size "$rfs_release")) at: ${rfs_release#${components_dir}/}"
((ok_count++))
else
log_error "✗ rfs missing: checked rfs/target/.../rfs and rfs/rfs"
((missing_count++))
fi
# mycelium
local mycelium_bin="${components_dir}/mycelium/myceliumd/target/x86_64-unknown-linux-musl/release/mycelium"
if [[ -x "$mycelium_bin" ]]; then
log_info "✓ mycelium ($(get_file_size "$mycelium_bin")) at: ${mycelium_bin#${components_dir}/}"
((ok_count++))
else
log_error "✗ mycelium missing: ${mycelium_bin#${components_dir}/}"
((missing_count++))
fi
# zosstorage
local zosstorage_bin="${components_dir}/zosstorage/target/x86_64-unknown-linux-musl/release/zosstorage"
if [[ -x "$zosstorage_bin" ]]; then
log_info "✓ zosstorage ($(get_file_size "$zosstorage_bin")) at: ${zosstorage_bin#${components_dir}/}"
((ok_count++))
else
log_error "✗ zosstorage missing: ${zosstorage_bin#${components_dir}/}"
((missing_count++))
fi
# corex
local corex_bin="${components_dir}/corex/corex"
if [[ -x "$corex_bin" ]]; then
log_info "✓ corex ($(get_file_size "$corex_bin")) at: ${corex_bin#${components_dir}/}"
((ok_count++))
else
log_error "✗ corex missing: ${corex_bin#${components_dir}/}"
((missing_count++))
fi
if [[ $missing_count -eq 0 ]]; then
log_info "All components built/installed successfully"
return 0
else
log_error "${missing_count} components missing or failed to build"
return 1
fi
}
# Clean component build artifacts
function components_cleanup() {
local components_dir="$1"
local keep_sources="${2:-false}"
section_header "Cleaning Component Build Artifacts"
if [[ "$keep_sources" == "true" ]]; then
log_info "Keeping source directories, cleaning build artifacts only"
# Clean Rust build artifacts
find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true
find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true
else
log_info "Removing all component directories"
safe_rmdir "$components_dir"
fi
log_info "Component cleanup complete"
}
# Export functions
export -f components_parse_sources_conf
export -f components_download_git components_download_release components_process_extra_options
export -f components_build_component components_setup_rust_env
export -f build_zinit build_rfs build_zosstorage build_mycelium install_corex
export -f components_verify_installation components_cleanup
# Export functions for install_rfs
export -f install_rfs