forked from tfgrid/zosbuilder
Jan De Landtsheer
fb51f4eb4e
- Remove rustup dependency from Dockerfile (not available in Alpine) - Update Rust environment setup to handle both rustup and system Rust - Fix musl-gcc linker configuration for Alpine containers - Support both GitHub Actions (rustup) and Alpine container (system) environments
468 lines
15 KiB
Bash
468 lines
15 KiB
Bash
#!/bin/bash
|
|
# Component download and build system for ThreeFold Zero OS
|
|
|
|
# Source common functions
|
|
LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "${LIB_SCRIPT_DIR}/common.sh"
|
|
|
|
# Component configuration
|
|
RUST_TARGET="${RUST_TARGET:-x86_64-unknown-linux-musl}"
|
|
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}"
|
|
|
|
# Parse and process all components from sources.conf
|
|
function components_parse_sources_conf() {
|
|
local sources_file="$1"
|
|
local components_dir="$2"
|
|
local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}"
|
|
|
|
section_header "Parsing Sources Configuration"
|
|
|
|
if [[ ! -f "$sources_file" ]]; then
|
|
log_error "Sources file not found: ${sources_file}"
|
|
return 1
|
|
fi
|
|
|
|
# Ensure components directory exists
|
|
safe_mkdir "$components_dir"
|
|
|
|
# Export install directory for build functions
|
|
export INSTALL_DIR="$install_dir"
|
|
|
|
log_info "Processing components from: ${sources_file}"
|
|
log_info "Components directory: ${components_dir}"
|
|
log_info "Install directory: ${install_dir}"
|
|
|
|
local component_count=0
|
|
|
|
# Process each line in sources.conf
|
|
while IFS=: read -r type name url version build_func extra; do
|
|
# Skip comments and empty lines
|
|
if [[ "$type" =~ ^[[:space:]]*# ]] || [[ -z "${type// }" ]]; then
|
|
continue
|
|
fi
|
|
|
|
# Trim whitespace
|
|
type=$(echo "$type" | xargs)
|
|
name=$(echo "$name" | xargs)
|
|
url=$(echo "$url" | xargs)
|
|
version=$(echo "$version" | xargs)
|
|
build_func=$(echo "$build_func" | xargs)
|
|
extra=$(echo "$extra" | xargs)
|
|
|
|
if [[ -z "$type" || -z "$name" || -z "$url" || -z "$version" || -z "$build_func" ]]; then
|
|
log_warn "Skipping invalid line: ${type}:${name}:${url}:${version}:${build_func}:${extra}"
|
|
continue
|
|
fi
|
|
|
|
((component_count++))
|
|
log_info "Processing component ${component_count}: ${name} (${type})"
|
|
|
|
# Download component
|
|
case "$type" in
|
|
"git")
|
|
components_download_git "$name" "$url" "$version" "$components_dir"
|
|
;;
|
|
"release")
|
|
components_download_release "$name" "$url" "$version" "$components_dir" "$extra"
|
|
;;
|
|
*)
|
|
log_error "Unknown component type: $type"
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
# Build and install component
|
|
components_build_component "$name" "$build_func" "$components_dir"
|
|
|
|
done < "$sources_file"
|
|
|
|
if [[ $component_count -eq 0 ]]; then
|
|
log_warn "No components found in sources configuration"
|
|
else
|
|
log_info "Processed ${component_count} components successfully"
|
|
fi
|
|
}
|
|
|
|
# Download Git repository
|
|
function components_download_git() {
|
|
local name="$1"
|
|
local url="$2"
|
|
local version="$3"
|
|
local components_dir="$4"
|
|
|
|
section_header "Downloading Git Component: ${name}"
|
|
|
|
local target_dir="${components_dir}/${name}"
|
|
|
|
log_info "Repository: ${url}"
|
|
log_info "Version/Branch: ${version}"
|
|
log_info "Target directory: ${target_dir}"
|
|
|
|
if [[ -d "$target_dir" ]]; then
|
|
log_info "Component ${name} already exists, updating..."
|
|
safe_execute cd "$target_dir"
|
|
safe_execute git fetch --all
|
|
safe_execute git checkout "$version"
|
|
safe_execute git pull origin "$version" 2>/dev/null || log_info "Already up to date"
|
|
else
|
|
log_info "Cloning ${name} from ${url}"
|
|
safe_execute git clone --depth 1 --branch "$version" "$url" "$target_dir"
|
|
fi
|
|
|
|
# Verify checkout
|
|
safe_execute cd "$target_dir"
|
|
local current_ref=$(git rev-parse HEAD)
|
|
log_info "Current commit: ${current_ref}"
|
|
|
|
log_info "Git component download complete: ${name}"
|
|
}
|
|
|
|
# Download release binary/archive
|
|
function components_download_release() {
|
|
local name="$1"
|
|
local url="$2"
|
|
local version="$3"
|
|
local components_dir="$4"
|
|
local extra="$5"
|
|
|
|
section_header "Downloading Release Component: ${name}"
|
|
|
|
local target_dir="${components_dir}/${name}"
|
|
local filename=$(basename "$url")
|
|
|
|
log_info "Release URL: ${url}"
|
|
log_info "Version: ${version}"
|
|
log_info "Target directory: ${target_dir}"
|
|
|
|
safe_mkdir "$target_dir"
|
|
|
|
# Download release
|
|
log_info "Downloading release: ${filename}"
|
|
safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url"
|
|
|
|
# Verify download
|
|
if [[ ! -f "${target_dir}/${filename}" ]]; then
|
|
log_error "Failed to download release: ${filename}"
|
|
return 1
|
|
fi
|
|
|
|
local file_size=$(get_file_size "${target_dir}/${filename}")
|
|
log_info "Downloaded file size: ${file_size}"
|
|
|
|
# Handle extra options (like rename)
|
|
if [[ -n "$extra" ]]; then
|
|
components_process_extra_options "$target_dir" "$filename" "$extra"
|
|
fi
|
|
|
|
log_info "Release component download complete: ${name}"
|
|
}
|
|
|
|
# Process extra options for components
|
|
function components_process_extra_options() {
|
|
local target_dir="$1"
|
|
local filename="$2"
|
|
local extra="$3"
|
|
|
|
log_info "Processing extra options: ${extra}"
|
|
|
|
# Handle rename option
|
|
if [[ "$extra" =~ rename=(.+) ]]; then
|
|
local new_name="${BASH_REMATCH[1]}"
|
|
log_info "Renaming ${filename} to ${new_name}"
|
|
safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}"
|
|
fi
|
|
|
|
# Handle extract option for archives
|
|
if [[ "$extra" =~ extract ]]; then
|
|
log_info "Extracting archive: ${filename}"
|
|
safe_execute cd "$target_dir"
|
|
case "$filename" in
|
|
*.tar.gz|*.tgz)
|
|
safe_execute tar -xzf "$filename"
|
|
;;
|
|
*.tar.bz2|*.tbz2)
|
|
safe_execute tar -xjf "$filename"
|
|
;;
|
|
*.tar.xz|*.txz)
|
|
safe_execute tar -xJf "$filename"
|
|
;;
|
|
*.zip)
|
|
safe_execute unzip "$filename"
|
|
;;
|
|
*)
|
|
log_warn "Unknown archive format: ${filename}"
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# Build component using specified build function
|
|
function components_build_component() {
|
|
local name="$1"
|
|
local build_func="$2"
|
|
local components_dir="$3"
|
|
|
|
section_header "Building Component: ${name}"
|
|
|
|
local component_dir="${components_dir}/${name}"
|
|
|
|
if [[ ! -d "$component_dir" ]]; then
|
|
log_error "Component directory not found: ${component_dir}"
|
|
return 1
|
|
fi
|
|
|
|
# Change to component directory
|
|
safe_execute cd "$component_dir"
|
|
|
|
log_info "Build function: ${build_func}"
|
|
log_info "Working directory: $(pwd)"
|
|
|
|
# Check if build function exists
|
|
if ! declare -f "$build_func" >/dev/null; then
|
|
log_error "Build function not found: ${build_func}"
|
|
return 1
|
|
fi
|
|
|
|
# Call the specific build function
|
|
log_info "Executing build function: ${build_func}"
|
|
"$build_func" "$name" "$component_dir"
|
|
|
|
log_info "Component build complete: ${name}"
|
|
}
|
|
|
|
# Setup Rust environment for musl builds
|
|
function components_setup_rust_env() {
|
|
section_header "Setting Up Rust Environment"
|
|
|
|
# Check if we have rustup (Ubuntu/GitHub Actions) or system Rust (Alpine container)
|
|
if command_exists "rustup"; then
|
|
log_info "Using rustup for Rust toolchain management"
|
|
|
|
# Ensure musl target is installed
|
|
if ! rustup target list --installed | grep -q "$RUST_TARGET"; then
|
|
log_info "Installing Rust target: ${RUST_TARGET}"
|
|
safe_execute rustup target add "$RUST_TARGET"
|
|
else
|
|
log_info "Rust target already installed: ${RUST_TARGET}"
|
|
fi
|
|
|
|
# Set environment variables for rustup
|
|
export RUSTFLAGS="-C target-feature=+crt-static"
|
|
export CC_x86_64_unknown_linux_musl="musl-gcc"
|
|
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER="musl-gcc"
|
|
else
|
|
log_info "Using system Rust (Alpine) with musl"
|
|
|
|
# Verify musl-gcc is available
|
|
if ! command_exists "musl-gcc"; then
|
|
log_error "musl-gcc not found. Install with: apk add musl-dev"
|
|
return 1
|
|
fi
|
|
|
|
# Set environment variables for system Rust
|
|
export RUSTFLAGS="-C target-feature=+crt-static -C linker=musl-gcc"
|
|
export CC="musl-gcc"
|
|
export TARGET_CC="musl-gcc"
|
|
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER="musl-gcc"
|
|
fi
|
|
|
|
log_info "Rust environment configured for musl builds"
|
|
log_info "RUST_TARGET: ${RUST_TARGET}"
|
|
log_info "RUSTFLAGS: ${RUSTFLAGS}"
|
|
log_info "CC: ${CC:-system-default}"
|
|
}
|
|
|
|
# Build function for zinit (standard Rust build)
|
|
function build_zinit() {
|
|
local name="$1"
|
|
local component_dir="$2"
|
|
|
|
section_header "Building zinit with musl target"
|
|
|
|
components_setup_rust_env
|
|
|
|
log_info "Building zinit from: ${component_dir}"
|
|
|
|
# Build with musl target
|
|
safe_execute cargo build --release --target "$RUST_TARGET"
|
|
|
|
# Find and install binary
|
|
local binary_path="target/${RUST_TARGET}/release/zinit"
|
|
if [[ ! -f "$binary_path" ]]; then
|
|
log_error "zinit binary not found at: ${binary_path}"
|
|
return 1
|
|
fi
|
|
|
|
# Install to initramfs
|
|
safe_mkdir "${INSTALL_DIR}/sbin"
|
|
safe_execute cp "$binary_path" "${INSTALL_DIR}/sbin/zinit"
|
|
safe_execute chmod +x "${INSTALL_DIR}/sbin/zinit"
|
|
|
|
local binary_size=$(get_file_size "${INSTALL_DIR}/sbin/zinit")
|
|
log_info "Installed zinit binary (${binary_size}) to: ${INSTALL_DIR}/sbin/zinit"
|
|
}
|
|
|
|
# Build function for rfs (standard Rust build)
|
|
function build_rfs() {
|
|
local name="$1"
|
|
local component_dir="$2"
|
|
|
|
section_header "Building rfs with musl target"
|
|
|
|
components_setup_rust_env
|
|
|
|
log_info "Building rfs from: ${component_dir}"
|
|
|
|
# Build with musl target
|
|
safe_execute cargo build --release --target "$RUST_TARGET"
|
|
|
|
# Find and install binary
|
|
local binary_path="target/${RUST_TARGET}/release/rfs"
|
|
if [[ ! -f "$binary_path" ]]; then
|
|
log_error "rfs binary not found at: ${binary_path}"
|
|
return 1
|
|
fi
|
|
|
|
# Install to initramfs
|
|
safe_mkdir "${INSTALL_DIR}/usr/bin"
|
|
safe_execute cp "$binary_path" "${INSTALL_DIR}/usr/bin/rfs"
|
|
safe_execute chmod +x "${INSTALL_DIR}/usr/bin/rfs"
|
|
|
|
local binary_size=$(get_file_size "${INSTALL_DIR}/usr/bin/rfs")
|
|
log_info "Installed rfs binary (${binary_size}) to: ${INSTALL_DIR}/usr/bin/rfs"
|
|
}
|
|
|
|
# Build function for mycelium (special subdirectory build)
|
|
function build_mycelium() {
|
|
local name="$1"
|
|
local component_dir="$2"
|
|
|
|
section_header "Building mycelium with musl target (special directory)"
|
|
|
|
components_setup_rust_env
|
|
|
|
log_info "Building mycelium from: ${component_dir}"
|
|
|
|
# Change to myceliumd subdirectory (special requirement)
|
|
local myceliumd_dir="${component_dir}/myceliumd"
|
|
if [[ ! -d "$myceliumd_dir" ]]; then
|
|
log_error "myceliumd directory not found at: ${myceliumd_dir}"
|
|
return 1
|
|
fi
|
|
|
|
safe_execute cd "$myceliumd_dir"
|
|
log_info "Building in myceliumd subdirectory: $(pwd)"
|
|
|
|
# Build with musl target
|
|
safe_execute cargo build --release --target "$RUST_TARGET"
|
|
|
|
# Find and install binary (from target/x86.../release)
|
|
local binary_path="target/${RUST_TARGET}/release/mycelium"
|
|
if [[ ! -f "$binary_path" ]]; then
|
|
log_error "mycelium binary not found at: ${binary_path}"
|
|
return 1
|
|
fi
|
|
|
|
# Install to initramfs
|
|
safe_mkdir "${INSTALL_DIR}/usr/bin"
|
|
safe_execute cp "$binary_path" "${INSTALL_DIR}/usr/bin/mycelium"
|
|
safe_execute chmod +x "${INSTALL_DIR}/usr/bin/mycelium"
|
|
|
|
local binary_size=$(get_file_size "${INSTALL_DIR}/usr/bin/mycelium")
|
|
log_info "Installed mycelium binary (${binary_size}) to: ${INSTALL_DIR}/usr/bin/mycelium"
|
|
}
|
|
|
|
# Install function for corex (pre-built binary)
|
|
function install_corex() {
|
|
local name="$1"
|
|
local component_dir="$2"
|
|
|
|
section_header "Installing corex binary"
|
|
|
|
log_info "Installing corex from: ${component_dir}"
|
|
|
|
# Find the corex binary (may have been renamed)
|
|
local binary_path
|
|
if [[ -f "${component_dir}/corex" ]]; then
|
|
binary_path="${component_dir}/corex"
|
|
elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then
|
|
binary_path="${component_dir}/corex-2.1.4-amd64-linux-static"
|
|
else
|
|
log_error "corex binary not found in: ${component_dir}"
|
|
return 1
|
|
fi
|
|
|
|
# Make executable and install
|
|
safe_execute chmod +x "$binary_path"
|
|
safe_mkdir "${INSTALL_DIR}/usr/bin"
|
|
safe_execute cp "$binary_path" "${INSTALL_DIR}/usr/bin/corex"
|
|
safe_execute chmod +x "${INSTALL_DIR}/usr/bin/corex"
|
|
|
|
local binary_size=$(get_file_size "${INSTALL_DIR}/usr/bin/corex")
|
|
log_info "Installed corex binary (${binary_size}) to: ${INSTALL_DIR}/usr/bin/corex"
|
|
}
|
|
|
|
# Verify all built components
|
|
function components_verify_installation() {
|
|
local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}"
|
|
|
|
section_header "Verifying Component Installation"
|
|
|
|
# List of expected binaries and their locations
|
|
local expected_binaries=(
|
|
"sbin/zinit"
|
|
"usr/bin/rfs"
|
|
"usr/bin/mycelium"
|
|
"usr/bin/corex"
|
|
)
|
|
|
|
local missing_count=0
|
|
|
|
for binary in "${expected_binaries[@]}"; do
|
|
local full_path="${install_dir}/${binary}"
|
|
if [[ -f "$full_path" && -x "$full_path" ]]; then
|
|
local size=$(get_file_size "$full_path")
|
|
log_info "✓ ${binary} (${size})"
|
|
else
|
|
log_error "✗ Missing or not executable: ${binary}"
|
|
((missing_count++))
|
|
fi
|
|
done
|
|
|
|
if [[ $missing_count -eq 0 ]]; then
|
|
log_info "All components installed successfully"
|
|
return 0
|
|
else
|
|
log_error "${missing_count} components missing or invalid"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Clean component build artifacts
|
|
function components_cleanup() {
|
|
local components_dir="$1"
|
|
local keep_sources="${2:-false}"
|
|
|
|
section_header "Cleaning Component Build Artifacts"
|
|
|
|
if [[ "$keep_sources" == "true" ]]; then
|
|
log_info "Keeping source directories, cleaning build artifacts only"
|
|
|
|
# Clean Rust build artifacts
|
|
find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true
|
|
find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true
|
|
|
|
else
|
|
log_info "Removing all component directories"
|
|
safe_rmdir "$components_dir"
|
|
fi
|
|
|
|
log_info "Component cleanup complete"
|
|
}
|
|
|
|
# Export functions
|
|
export -f components_parse_sources_conf
|
|
export -f components_download_git components_download_release components_process_extra_options
|
|
export -f components_build_component components_setup_rust_env
|
|
export -f build_zinit build_rfs build_mycelium install_corex
|
|
export -f components_verify_installation components_cleanup |