Files
zosbuilder/scripts/lib/initramfs.sh
Jan De Landtsheer b9f94105cf fix: major build system improvements and container output issues
- Fix container output visibility with proper TTY handling and debug mode
- Fix build order: kernel modules built before initramfs creation
- Implement two-stage kernel build to resolve chicken-and-egg dependency
- Fix sed command issues in kernel configuration with direct execution
- Add diffutils package to container for proper kernel build support
- Enhance NIC module/firmware correlation with intelligent selection
- Fix module staging logic: all NICs loaded in stage1 before network up
- Add smart firmware installation based on module requirements
- Create comprehensive function documentation (scripts/functionlist.md)
- Add debug container script for troubleshooting

Major fixes:
* Container builds now show real-time output
* Kernel builds work with proper GNU diff support
* Module/firmware selection optimized for common hardware
* Build process handles dependencies correctly
* Documentation provides complete function reference
2025-09-03 14:06:44 +02:00

628 lines
21 KiB
Bash

#!/bin/bash
# Initramfs assembly and optimization
# Source common functions
LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${LIB_SCRIPT_DIR}/common.sh"
# Initramfs configuration
INITRAMFS_COMPRESSION="${INITRAMFS_COMPRESSION:-xz}"
XZ_COMPRESSION_LEVEL="${XZ_COMPRESSION_LEVEL:-9}"
# Setup zinit as init system (replaces OpenRC completely)
function initramfs_setup_zinit() {
local initramfs_dir="$1"
local zinit_config_dir="$2"
section_header "Setting up zinit as init system"
if [[ ! -d "$zinit_config_dir" ]]; then
log_error "zinit configuration directory not found: ${zinit_config_dir}"
return 1
fi
# Verify zinit binary exists
if [[ ! -x "${initramfs_dir}/sbin/zinit" ]]; then
log_error "zinit binary not found or not executable: ${initramfs_dir}/sbin/zinit"
return 1
fi
# Remove existing init (if any) and replace with zinit
log_info "Replacing system init with zinit"
safe_execute rm -f "${initramfs_dir}/sbin/init"
safe_execute ln -sf zinit "${initramfs_dir}/sbin/init"
# Copy zinit configuration (all YAML and scripts)
log_info "Installing zinit configuration"
safe_mkdir "${initramfs_dir}/etc/zinit"
safe_execute cp -r "${zinit_config_dir}"/* "${initramfs_dir}/etc/zinit/"
# Ensure proper permissions
safe_execute chmod 755 "${initramfs_dir}/sbin/zinit"
safe_execute chmod -R 644 "${initramfs_dir}/etc/zinit"
safe_execute find "${initramfs_dir}/etc/zinit" -name "*.sh" -exec chmod 755 {} \;
# Create zinit working directories
safe_mkdir "${initramfs_dir}/var/log/zinit"
safe_mkdir "${initramfs_dir}/run/zinit"
# Remove any OpenRC remnants (ensure complete replacement)
local openrc_paths=(
"/etc/init.d"
"/etc/runlevels"
"/etc/conf.d"
"/sbin/openrc"
"/sbin/rc-service"
"/sbin/rc-status"
"/sbin/rc-update"
)
for path in "${openrc_paths[@]}"; do
if [[ -e "${initramfs_dir}${path}" ]]; then
log_info "Removing OpenRC remnant: ${path}"
safe_execute rm -rf "${initramfs_dir}${path}"
fi
done
log_info "zinit setup complete - OpenRC completely replaced"
}
# Install the critical /init script for initramfs boot
function initramfs_install_init_script() {
local initramfs_dir="$1"
local init_script_path="$2"
section_header "Installing initramfs /init script"
if [[ ! -f "$init_script_path" ]]; then
log_error "Init script not found: ${init_script_path}"
return 1
fi
# Install the init script as /init in initramfs root
log_info "Installing init script from: ${init_script_path}"
safe_execute cp "$init_script_path" "${initramfs_dir}/init"
safe_execute chmod 755 "${initramfs_dir}/init"
# Verify installation
if [[ ! -x "${initramfs_dir}/init" ]]; then
log_error "Failed to install init script"
return 1
fi
log_info "✓ Initramfs /init script installed successfully"
log_info " Boot flow: /init -> setup environment -> switch_root -> /sbin/zinit init"
}
# Setup 2-stage module loading system with dependency resolution and firmware correlation
function initramfs_setup_modules() {
local initramfs_dir="$1"
local modules_conf="$2"
local kernel_version="${3:-$(uname -r)}"
section_header "Setting up 2-stage module loading with firmware correlation"
if [[ ! -f "$modules_conf" ]]; then
log_error "Modules configuration file not found: ${modules_conf}"
return 1
fi
local modules_dir="${initramfs_dir}/lib/modules/${kernel_version}"
safe_mkdir "$modules_dir"
# Track required firmware packages
local required_firmware=()
# Create stage1 module list with dependencies and firmware
log_info "Resolving stage1 module dependencies (critical boot modules)"
local stage1_modules=()
local stage1_firmware=()
while IFS=: read -r stage module firmware; do
if [[ "$stage" == "stage1" && -n "$module" ]]; then
stage1_modules+=("$module")
if [[ -n "$firmware" && "$firmware" != "none" ]]; then
stage1_firmware+=("$firmware")
required_firmware+=("$firmware")
log_debug "Module $module requires firmware: $firmware"
fi
fi
done < <(grep "^stage1:" "$modules_conf")
local stage1_with_deps=()
if [[ ${#stage1_modules[@]} -gt 0 ]]; then
stage1_with_deps=($(initramfs_resolve_module_dependencies "${stage1_modules[@]}"))
fi
# Write stage1 list with dependencies
printf '%s\n' "${stage1_with_deps[@]}" > "${modules_dir}/stage1.list"
# Create stage2 module list with dependencies and firmware
log_info "Resolving stage2 module dependencies (extended hardware)"
local stage2_modules=()
local stage2_firmware=()
while IFS=: read -r stage module firmware; do
if [[ "$stage" == "stage2" && -n "$module" ]]; then
stage2_modules+=("$module")
if [[ -n "$firmware" && "$firmware" != "none" ]]; then
stage2_firmware+=("$firmware")
required_firmware+=("$firmware")
log_debug "Module $module requires firmware: $firmware"
fi
fi
done < <(grep "^stage2:" "$modules_conf")
local stage2_with_deps=()
if [[ ${#stage2_modules[@]} -gt 0 ]]; then
stage2_with_deps=($(initramfs_resolve_module_dependencies "${stage2_modules[@]}"))
# Remove stage1 modules from stage2 to avoid duplicates
local stage2_unique=()
for mod in "${stage2_with_deps[@]}"; do
if ! printf '%s\n' "${stage1_with_deps[@]}" | grep -q "^${mod}$"; then
stage2_unique+=("$mod")
fi
done
stage2_with_deps=("${stage2_unique[@]}")
fi
# Write stage2 list with unique dependencies
printf '%s\n' "${stage2_with_deps[@]}" > "${modules_dir}/stage2.list"
# Create firmware requirements list (remove duplicates)
local unique_firmware=($(printf '%s\n' "${required_firmware[@]}" | sort -u))
if [[ ${#unique_firmware[@]} -gt 0 ]]; then
printf '%s\n' "${unique_firmware[@]}" > "${modules_dir}/required-firmware.list"
log_info "Created firmware requirements list: ${#unique_firmware[@]} packages"
for fw in "${unique_firmware[@]}"; do
log_info " Required firmware: $fw"
done
# Export for use by firmware installation
export REQUIRED_FIRMWARE_PACKAGES="${unique_firmware[*]}"
else
log_info "No firmware packages required"
export REQUIRED_FIRMWARE_PACKAGES=""
fi
# Create module loading scripts
initramfs_create_module_scripts "$initramfs_dir" "$kernel_version"
# Report final counts
local stage1_count=${#stage1_with_deps[@]}
local stage2_count=${#stage2_with_deps[@]}
local firmware_count=${#unique_firmware[@]}
log_info "Module configuration complete:"
log_info " Stage1 (critical + deps): ${stage1_count} modules"
log_info " Stage2 (extended + deps): ${stage2_count} modules"
log_info " Required firmware packages: ${firmware_count}"
log_info " Total unique modules: $((stage1_count + stage2_count))"
}
# Resolve module dependencies recursively using modinfo
function initramfs_resolve_module_dependencies() {
local modules=("$@")
local resolved_modules=()
local processed_modules=()
log_debug "Resolving dependencies for modules: ${modules[*]}"
# Function to recursively resolve a single module's dependencies
function resolve_single_module() {
local module="$1"
# Skip if already processed
if printf '%s\n' "${processed_modules[@]}" | grep -q "^${module}$"; then
return 0
fi
processed_modules+=("$module")
# Get module dependencies using modinfo
local deps=()
if command_exists "modinfo"; then
# Try to get dependencies - modinfo may fail for built-in modules
local modinfo_output
if modinfo_output=$(modinfo "$module" 2>/dev/null); then
# Extract depends line and parse comma-separated dependencies
local depends_line=$(echo "$modinfo_output" | grep '^depends:' | head -1 | cut -d: -f2- | tr -d ' ')
if [[ -n "$depends_line" && "$depends_line" != "-" ]]; then
IFS=',' read -ra deps <<< "$depends_line"
fi
else
log_debug "modinfo failed for module: $module (may be built-in)"
fi
else
log_warn "modinfo not available, skipping dependency resolution"
fi
# Recursively resolve dependencies first
for dep in "${deps[@]}"; do
if [[ -n "$dep" ]]; then
log_debug "Module $module depends on: $dep"
resolve_single_module "$dep"
fi
done
# Add current module to resolved list (after dependencies)
resolved_modules+=("$module")
}
# Process all requested modules
for module in "${modules[@]}"; do
resolve_single_module "$module"
done
# Remove duplicates while preserving order
local unique_modules=()
local seen_modules=()
for module in "${resolved_modules[@]}"; do
if ! printf '%s\n' "${seen_modules[@]}" | grep -q "^${module}$"; then
unique_modules+=("$module")
seen_modules+=("$module")
fi
done
log_debug "Resolved ${#unique_modules[@]} unique modules with dependencies"
printf '%s\n' "${unique_modules[@]}"
}
# Create module loading scripts for zinit
function initramfs_create_module_scripts() {
local initramfs_dir="$1"
local kernel_version="$2"
log_info "Creating module loading scripts"
safe_mkdir "${initramfs_dir}/etc/zinit/init"
# Stage1 module loading script (critical modules)
cat > "${initramfs_dir}/etc/zinit/init/stage1-modules.sh" << 'EOF'
#!/bin/sh
# Stage1 module loading - critical boot modules
KERNEL_VERSION=$(uname -r)
STAGE1_LIST="/lib/modules/${KERNEL_VERSION}/stage1.list"
echo "Loading stage1 modules (critical boot)"
if [ -f "$STAGE1_LIST" ]; then
while read -r module; do
if [ -n "$module" ] && [ "$module" != "#"* ]; then
echo "Loading critical module: $module"
modprobe "$module" 2>/dev/null || echo "Warning: Failed to load $module"
fi
done < "$STAGE1_LIST"
else
echo "Warning: Stage1 module list not found: $STAGE1_LIST"
fi
echo "Stage1 module loading complete"
EOF
# Stage2 module loading script (extended hardware)
cat > "${initramfs_dir}/etc/zinit/init/stage2-modules.sh" << 'EOF'
#!/bin/sh
# Stage2 module loading - extended hardware support
KERNEL_VERSION=$(uname -r)
STAGE2_LIST="/lib/modules/${KERNEL_VERSION}/stage2.list"
echo "Loading stage2 modules (extended hardware)"
if [ -f "$STAGE2_LIST" ]; then
while read -r module; do
if [ -n "$module" ] && [ "$module" != "#"* ]; then
echo "Loading hardware module: $module"
modprobe "$module" 2>/dev/null || echo "Info: Module $module not available"
fi
done < "$STAGE2_LIST"
else
echo "Warning: Stage2 module list not found: $STAGE2_LIST"
fi
echo "Stage2 module loading complete"
EOF
# Make scripts executable
safe_execute chmod 755 "${initramfs_dir}/etc/zinit/init/stage1-modules.sh"
safe_execute chmod 755 "${initramfs_dir}/etc/zinit/init/stage2-modules.sh"
log_info "Module loading scripts created"
}
# Strip and UPX compress all binaries for maximum size optimization
function initramfs_strip_and_upx() {
local initramfs_dir="$1"
section_header "Stripping and UPX compressing binaries"
local stripped_count=0
local upx_count=0
local failed_strip=0
local failed_upx=0
# Find and process all executable files
log_info "Processing executable files..."
while IFS= read -r -d '' file; do
# Check if it's a valid ELF executable
if file "$file" | grep -q "ELF.*executable"; then
log_debug "Processing executable: $file"
# Strip debug symbols (ignore already stripped files)
if strip "$file" 2>/dev/null || true; then
if [[ $? -eq 0 ]]; then
((stripped_count++))
log_debug "Stripped: $file"
else
((failed_strip++))
log_debug "Already stripped or failed: $file"
fi
fi
# UPX compress (best compression) - skip if already compressed or fails
if command_exists "upx"; then
if upx --best --force "$file" >/dev/null 2>&1 || true; then
if [[ $? -eq 0 ]]; then
((upx_count++))
log_debug "UPX compressed: $file"
else
((failed_upx++))
log_debug "UPX failed or already compressed: $file"
fi
fi
else
log_debug "UPX not available, skipping compression for: $file"
((failed_upx++))
fi
fi
done < <(find "$initramfs_dir" -type f -executable -print0)
# Process shared libraries
log_info "Processing shared libraries..."
local lib_stripped=0
local lib_failed=0
while IFS= read -r -d '' file; do
if file "$file" | grep -q "ELF.*shared object"; then
log_debug "Processing library: $file"
# Strip libraries (more conservative - keep function symbols)
if strip --strip-unneeded "$file" 2>/dev/null || true; then
if [[ $? -eq 0 ]]; then
((lib_stripped++))
log_debug "Stripped library: $file"
else
((lib_failed++))
log_debug "Library already stripped or failed: $file"
fi
fi
fi
done < <(find "$initramfs_dir" -name "*.so*" -type f -print0)
# Summary
log_info "Binary optimization complete:"
log_info " Executables stripped: ${stripped_count} (${failed_strip} failed)"
log_info " Executables UPX compressed: ${upx_count} (${failed_upx} failed)"
log_info " Libraries stripped: ${lib_stripped} (${lib_failed} failed)"
# Calculate space savings
local total_size=$(du -sb "$initramfs_dir" 2>/dev/null | cut -f1 || echo "0")
local total_mb=$((total_size / 1024 / 1024))
log_info "Total initramfs size after optimization: ${total_mb}MB"
}
# Create final initramfs.cpio.xz archive
function initramfs_create_cpio() {
local initramfs_dir="$1"
local output_file="$2"
local compression="${3:-$INITRAMFS_COMPRESSION}"
section_header "Creating initramfs.cpio.${compression}"
if [[ ! -d "$initramfs_dir" ]]; then
log_error "Initramfs directory not found: ${initramfs_dir}"
return 1
fi
# Ensure output directory exists
local output_dir=$(dirname "$output_file")
safe_mkdir "$output_dir"
# Remove any existing output file
safe_execute rm -f "$output_file"
log_info "Source directory: ${initramfs_dir}"
log_info "Output file: ${output_file}"
log_info "Compression: ${compression}"
# Change to initramfs directory for relative paths
safe_execute cd "$initramfs_dir"
case "$compression" in
"xz")
log_info "Creating XZ compressed CPIO archive"
safe_execute find . -print0 | cpio -o -H newc -0 | xz -${XZ_COMPRESSION_LEVEL} --check=crc32 > "$output_file"
;;
"gzip"|"gz")
log_info "Creating gzip compressed CPIO archive"
safe_execute find . -print0 | cpio -o -H newc -0 | gzip -9 > "$output_file"
;;
"zstd")
log_info "Creating zstd compressed CPIO archive"
safe_execute find . -print0 | cpio -o -H newc -0 | zstd -19 > "$output_file"
;;
"none"|"uncompressed")
log_info "Creating uncompressed CPIO archive"
safe_execute find . -print0 | cpio -o -H newc -0 > "$output_file"
;;
*)
log_error "Unsupported compression format: ${compression}"
return 1
;;
esac
# Verify output file was created
if [[ ! -f "$output_file" ]]; then
log_error "Failed to create initramfs archive: ${output_file}"
return 1
fi
# Report final size
local final_size=$(get_file_size "$output_file")
local uncompressed_size=$(du -sh "$initramfs_dir" | cut -f1)
log_info "Initramfs creation complete:"
log_info " Uncompressed size: ${uncompressed_size}"
log_info " Final archive size: ${final_size}"
log_info " Archive: ${output_file}"
}
# Validate initramfs contents
function initramfs_validate() {
local initramfs_dir="$1"
section_header "Validating initramfs contents"
local errors=0
# Check essential files and directories
local essential_items=(
"init"
"sbin/init"
"sbin/zinit"
"bin/busybox"
"etc/zinit"
"lib"
"usr/bin"
"var"
"tmp"
"proc"
"sys"
"dev"
)
for item in "${essential_items[@]}"; do
if [[ ! -e "${initramfs_dir}/${item}" ]]; then
log_error "Missing essential item: ${item}"
((errors++))
else
log_debug "Found: ${item}"
fi
done
# Check that initramfs /init script exists
if [[ -x "${initramfs_dir}/init" ]]; then
log_info "✓ Initramfs /init script found"
else
log_error "✗ Initramfs /init script missing"
((errors++))
fi
# Check that /sbin/init is properly linked to zinit
if [[ -L "${initramfs_dir}/sbin/init" ]]; then
local link_target=$(readlink "${initramfs_dir}/sbin/init")
if [[ "$link_target" == "zinit" ]]; then
log_info "✓ /sbin/init correctly linked to zinit"
else
log_error "✗ /sbin/init linked to wrong target: ${link_target}"
((errors++))
fi
else
log_error "✗ /sbin/init is not a symbolic link"
((errors++))
fi
# Check zinit configuration
if [[ -f "${initramfs_dir}/etc/zinit/zinit.conf" ]]; then
log_info "✓ zinit configuration found"
else
log_error "✗ zinit configuration missing"
((errors++))
fi
# Verify no OpenRC remnants
local openrc_check=(
"etc/init.d"
"etc/runlevels"
"sbin/openrc"
)
for path in "${openrc_check[@]}"; do
if [[ -e "${initramfs_dir}/${path}" ]]; then
log_warn "OpenRC remnant found: ${path}"
fi
done
# Check component binaries
local component_binaries=(
"usr/bin/rfs"
"usr/bin/mycelium"
"usr/bin/corex"
)
for binary in "${component_binaries[@]}"; do
if [[ -x "${initramfs_dir}/${binary}" ]]; then
log_info "✓ Component binary: ${binary}"
else
log_warn "Component binary missing or not executable: ${binary}"
fi
done
if [[ $errors -eq 0 ]]; then
log_info "Initramfs validation passed"
return 0
else
log_error "Initramfs validation failed with ${errors} errors"
return 1
fi
}
# Test initramfs archive integrity
function initramfs_test_archive() {
local archive_file="$1"
section_header "Testing initramfs archive integrity"
if [[ ! -f "$archive_file" ]]; then
log_error "Archive file not found: ${archive_file}"
return 1
fi
# Test based on file extension
case "$archive_file" in
*.xz)
log_info "Testing XZ archive integrity"
safe_execute xz -t "$archive_file"
;;
*.gz)
log_info "Testing gzip archive integrity"
safe_execute gzip -t "$archive_file"
;;
*.zst)
log_info "Testing zstd archive integrity"
safe_execute zstd -t "$archive_file"
;;
*.cpio)
log_info "Testing CPIO archive integrity"
safe_execute cpio -t < "$archive_file" >/dev/null
;;
*)
log_warn "Unknown archive format, skipping integrity test"
return 0
;;
esac
log_info "Archive integrity test passed"
}
# Export all functions at the end after they're all defined
export -f initramfs_setup_zinit initramfs_setup_modules initramfs_resolve_module_dependencies
export -f initramfs_install_init_script initramfs_create_module_scripts initramfs_strip_and_upx
export -f initramfs_create_cpio initramfs_validate initramfs_test_archive