forked from tfgrid/zosbuilder
521 lines
18 KiB
Bash
521 lines
18 KiB
Bash
#!/bin/bash
|
|
# Kernel building with embedded initramfs
|
|
|
|
# Source common functions
|
|
LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "${LIB_SCRIPT_DIR}/common.sh"
|
|
|
|
# Kernel configuration
|
|
KERNEL_VERSION="${KERNEL_VERSION:-6.12.44}"
|
|
KERNEL_SOURCE_URL="${KERNEL_SOURCE_URL:-https://cdn.kernel.org/pub/linux/kernel}"
|
|
KERNEL_CONFIG_SOURCE="${KERNEL_CONFIG_SOURCE:-${PROJECT_ROOT}/configs/kernel-config-generic}"
|
|
|
|
# Get actual kernel version including LOCALVERSION from kernel config
|
|
function kernel_get_full_version() {
|
|
local base_version="${1:-$KERNEL_VERSION}"
|
|
local config_file="${2:-${PROJECT_ROOT}/config/kernel.config}"
|
|
|
|
# Extract LOCALVERSION from kernel config
|
|
local localversion=""
|
|
if [[ -f "$config_file" ]] && grep -q "^CONFIG_LOCALVERSION=" "$config_file"; then
|
|
localversion=$(grep "^CONFIG_LOCALVERSION=" "$config_file" | cut -d'"' -f2)
|
|
fi
|
|
|
|
echo "${base_version}${localversion}"
|
|
}
|
|
|
|
# Download kernel source
|
|
function kernel_download_source() {
|
|
local kernel_dir="$1"
|
|
local version="${2:-$KERNEL_VERSION}"
|
|
|
|
section_header "Downloading Kernel Source"
|
|
|
|
local major_version=$(echo "$version" | cut -d. -f1)
|
|
local url="${KERNEL_SOURCE_URL}/v${major_version}.x/linux-${version}.tar.xz"
|
|
local temp_file="/tmp/linux-${version}.tar.xz"
|
|
local source_dir="${kernel_dir}/linux-${version}"
|
|
|
|
log_info "Kernel version: ${version}"
|
|
log_info "Download URL: ${url}"
|
|
log_info "Target directory: ${kernel_dir}"
|
|
|
|
# Clean existing kernel directory
|
|
if [[ -d "$kernel_dir" ]]; then
|
|
log_info "Cleaning existing kernel directory"
|
|
safe_rmdir "$kernel_dir"
|
|
fi
|
|
safe_mkdir "$kernel_dir"
|
|
|
|
# Download kernel source
|
|
if [[ ! -f "$temp_file" ]]; then
|
|
log_info "Downloading kernel source: ${version}"
|
|
safe_execute wget --progress=dot:giga -O "$temp_file" "$url"
|
|
else
|
|
log_info "Using cached kernel source: ${temp_file}"
|
|
fi
|
|
|
|
# Verify download
|
|
local file_size=$(get_file_size "$temp_file")
|
|
log_info "Kernel source size: ${file_size}"
|
|
|
|
# Extract kernel source
|
|
log_info "Extracting kernel source"
|
|
safe_execute tar -xJf "$temp_file" -C "$kernel_dir"
|
|
|
|
# Verify extraction
|
|
if [[ ! -d "$source_dir" ]]; then
|
|
log_error "Kernel source extraction failed"
|
|
return 1
|
|
fi
|
|
|
|
# Create symlink for easier access
|
|
safe_execute ln -sf "linux-${version}" "${kernel_dir}/current"
|
|
|
|
# Cleanup download
|
|
safe_execute rm "$temp_file"
|
|
|
|
log_info "Kernel source download complete: ${source_dir}"
|
|
}
|
|
|
|
# Apply kernel configuration with embedded initramfs
|
|
function kernel_apply_config() {
|
|
local kernel_dir="$1"
|
|
local initramfs_path="$2"
|
|
local config_source="${3:-$KERNEL_CONFIG_SOURCE}"
|
|
|
|
section_header "Applying Kernel Configuration"
|
|
|
|
local source_dir="${kernel_dir}/current"
|
|
|
|
if [[ ! -d "$source_dir" ]]; then
|
|
log_error "Kernel source directory not found: ${source_dir}"
|
|
return 1
|
|
fi
|
|
|
|
if [[ ! -f "$config_source" ]]; then
|
|
log_error "Kernel config source not found: ${config_source}"
|
|
return 1
|
|
fi
|
|
|
|
if [[ ! -f "$initramfs_path" ]]; then
|
|
log_error "Initramfs file not found: ${initramfs_path}"
|
|
return 1
|
|
fi
|
|
|
|
safe_execute cd "$source_dir"
|
|
|
|
# Copy base configuration
|
|
log_info "Copying kernel configuration from: ${config_source}"
|
|
safe_copy "$config_source" ".config"
|
|
|
|
# Update configuration for embedded initramfs
|
|
log_info "Updating configuration for embedded initramfs"
|
|
log_info "Initramfs path: ${initramfs_path}"
|
|
|
|
# Resolve absolute path for initramfs
|
|
local abs_initramfs_path=$(resolve_path "$initramfs_path")
|
|
|
|
# Modify config for embedded initramfs
|
|
kernel_modify_config_for_initramfs "$abs_initramfs_path"
|
|
|
|
# Run olddefconfig to apply defaults for any new options
|
|
log_info "Running olddefconfig to finalize configuration"
|
|
safe_execute make olddefconfig
|
|
|
|
log_info "Kernel configuration applied successfully"
|
|
}
|
|
|
|
# Modify kernel config for embedded initramfs
|
|
function kernel_modify_config_for_initramfs() {
|
|
local initramfs_path="$1"
|
|
|
|
log_info "Modifying kernel config for embedded initramfs"
|
|
|
|
# Use sed to update configuration (execute directly to avoid quote issues)
|
|
log_info "Setting INITRAMFS_SOURCE to: ${initramfs_path}"
|
|
if ! sed -i "s|^CONFIG_INITRAMFS_SOURCE=.*|CONFIG_INITRAMFS_SOURCE=\"${initramfs_path}\"|" .config; then
|
|
log_error "Failed to set INITRAMFS_SOURCE in kernel config"
|
|
return 1
|
|
fi
|
|
|
|
# Ensure XZ compression is enabled for initramfs
|
|
log_info "Enabling XZ decompression support"
|
|
if ! sed -i 's|^# CONFIG_RD_XZ is not set|CONFIG_RD_XZ=y|' .config; then
|
|
log_warn "Could not enable CONFIG_RD_XZ (may already be set)"
|
|
fi
|
|
|
|
log_info "Setting initramfs compression to XZ"
|
|
if ! sed -i 's|^CONFIG_INITRAMFS_COMPRESSION_NONE=y|# CONFIG_INITRAMFS_COMPRESSION_NONE is not set|' .config; then
|
|
log_warn "Could not disable INITRAMFS_COMPRESSION_NONE (may already be disabled)"
|
|
fi
|
|
|
|
if ! sed -i 's|^# CONFIG_INITRAMFS_COMPRESSION_XZ is not set|CONFIG_INITRAMFS_COMPRESSION_XZ=y|' .config; then
|
|
log_warn "Could not enable INITRAMFS_COMPRESSION_XZ (may already be set)"
|
|
fi
|
|
|
|
# Verify critical settings
|
|
if ! grep -q "CONFIG_INITRAMFS_SOURCE=\"${initramfs_path}\"" .config; then
|
|
log_error "Failed to set INITRAMFS_SOURCE in kernel config"
|
|
return 1
|
|
fi
|
|
|
|
# Check if XZ support is available (either RD_XZ or built-in)
|
|
if grep -q "CONFIG_RD_XZ=y" .config || grep -q "CONFIG_KERNEL_XZ=y" .config; then
|
|
log_info "✓ XZ decompression support confirmed"
|
|
else
|
|
log_warn "XZ decompression support not confirmed, kernel may not boot"
|
|
fi
|
|
|
|
log_info "Kernel config updated for embedded initramfs"
|
|
}
|
|
|
|
# Build kernel with embedded initramfs
|
|
function kernel_build_with_initramfs() {
|
|
local kernel_config="$1"
|
|
local initramfs_path="$2"
|
|
local output_file="$3"
|
|
local kernel_dir="${4:-${PROJECT_ROOT}/kernel}"
|
|
|
|
section_header "Building Kernel with Embedded Initramfs"
|
|
|
|
# Download kernel source if needed
|
|
if [[ ! -d "${kernel_dir}/current" ]]; then
|
|
kernel_download_source "$kernel_dir"
|
|
fi
|
|
|
|
# Apply configuration
|
|
kernel_apply_config "$kernel_dir" "$initramfs_path" "$kernel_config"
|
|
|
|
local source_dir="${kernel_dir}/current"
|
|
safe_execute cd "$source_dir"
|
|
|
|
# Determine number of cores for parallel build
|
|
local cores=$(nproc)
|
|
local jobs=$((cores > 1 ? cores - 1 : 1)) # Leave one core free
|
|
log_info "Building with ${jobs} parallel jobs"
|
|
|
|
# Build kernel
|
|
log_info "Building kernel (this may take a while)..."
|
|
safe_execute make -j${jobs} bzImage
|
|
|
|
# Verify kernel was built
|
|
local kernel_image="arch/x86/boot/bzImage"
|
|
if [[ ! -f "$kernel_image" ]]; then
|
|
log_error "Kernel build failed - bzImage not found"
|
|
return 1
|
|
fi
|
|
|
|
# Resolve output path to absolute BEFORE cd side-effects influence relative paths
|
|
local output_abs="$output_file"
|
|
if [[ "$output_abs" != /* ]]; then
|
|
if [[ -n "${PROJECT_ROOT:-}" ]]; then
|
|
output_abs="${PROJECT_ROOT}/${output_abs#./}"
|
|
log_debug "kernel_build_with_initramfs: output path anchored to PROJECT_ROOT: ${output_abs}"
|
|
else
|
|
output_abs="$(pwd)/${output_abs}"
|
|
log_warn "kernel_build_with_initramfs: PROJECT_ROOT unset; output anchored to PWD: ${output_abs}"
|
|
fi
|
|
fi
|
|
|
|
# Copy to output location
|
|
local output_dir
|
|
output_dir=$(dirname "$output_abs")
|
|
safe_mkdir "$output_dir"
|
|
safe_copy "$kernel_image" "$output_abs"
|
|
|
|
# Also copy with versioned filename including kernel version and zinit hash
|
|
local full_kernel_version="${FULL_KERNEL_VERSION:-unknown}"
|
|
local zinit_hash="unknown"
|
|
local zinit_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}/zinit"
|
|
|
|
if [[ -d "$zinit_dir/.git" ]]; then
|
|
zinit_hash=$(get_git_commit_hash "$zinit_dir")
|
|
else
|
|
log_warn "zinit git directory not found at ${zinit_dir}, using 'unknown' for hash"
|
|
fi
|
|
|
|
# Create versioned filename: vmlinuz-{VERSION}-{ZINIT_HASH}.efi
|
|
local versioned_name="vmlinuz-${full_kernel_version}-${zinit_hash}.efi"
|
|
local versioned_output="${output_dir}/${versioned_name}"
|
|
safe_copy "$kernel_image" "$versioned_output"
|
|
|
|
# Verify final kernel
|
|
local kernel_size
|
|
kernel_size=$(get_file_size "$output_abs")
|
|
local versioned_size
|
|
versioned_size=$(get_file_size "$versioned_output")
|
|
log_info "Kernel build complete:"
|
|
log_info " Output file: ${output_abs}"
|
|
log_info " Versioned: ${versioned_output}"
|
|
log_info " Kernel size: ${kernel_size}"
|
|
log_info " Version: ${full_kernel_version}"
|
|
log_info " zinit hash: ${zinit_hash}"
|
|
|
|
# Verify initramfs is embedded
|
|
if strings "$output_file" | grep -q "initramfs"; then
|
|
log_info "✓ Initramfs appears to be embedded in kernel"
|
|
else
|
|
log_warn "Initramfs embedding verification inconclusive"
|
|
fi
|
|
|
|
# Upload versioned kernel to S3 if enabled
|
|
kernel_upload_to_s3 "$versioned_output" "$full_kernel_version" "$zinit_hash"
|
|
}
|
|
|
|
# Upload versioned kernel to S3 using MinIO client (mcli/mc)
|
|
function kernel_upload_to_s3() {
|
|
local kernel_file="$1"
|
|
local kernel_version="$2"
|
|
local zinit_hash="$3"
|
|
|
|
section_header "Uploading Kernel to S3"
|
|
|
|
# Check if upload is enabled
|
|
if [[ "${UPLOAD_KERNEL:-false}" != "true" ]]; then
|
|
log_info "UPLOAD_KERNEL not enabled; skipping kernel upload"
|
|
return 0
|
|
fi
|
|
|
|
# Verify kernel file exists
|
|
if [[ ! -f "$kernel_file" ]]; then
|
|
log_error "Kernel file not found: ${kernel_file}"
|
|
return 1
|
|
fi
|
|
|
|
# Load S3 configuration from rfs.conf
|
|
local rfs_conf="${PROJECT_ROOT}/config/rfs.conf"
|
|
local rfs_conf_example="${PROJECT_ROOT}/config/rfs.conf.example"
|
|
|
|
if [[ -f "$rfs_conf" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "$rfs_conf"
|
|
log_info "Loaded S3 config from: ${rfs_conf}"
|
|
elif [[ -f "$rfs_conf_example" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "$rfs_conf_example"
|
|
log_warn "Using example S3 config: ${rfs_conf_example}"
|
|
else
|
|
log_error "No S3 config found (config/rfs.conf or config/rfs.conf.example)"
|
|
return 1
|
|
fi
|
|
|
|
# Validate required S3 variables
|
|
for var in S3_ENDPOINT S3_BUCKET S3_PREFIX S3_ACCESS_KEY S3_SECRET_KEY; do
|
|
if [[ -z "${!var}" ]]; then
|
|
log_error "Missing required S3 variable: ${var}"
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
# Detect MinIO client binary (mcli or mc)
|
|
local mcli_bin=""
|
|
if command -v mcli >/dev/null 2>&1; then
|
|
mcli_bin="mcli"
|
|
elif command -v mc >/dev/null 2>&1; then
|
|
mcli_bin="mc"
|
|
else
|
|
log_warn "MinIO Client not found (expected mcli or mc); skipping kernel upload"
|
|
return 0
|
|
fi
|
|
|
|
log_info "Using MinIO client: ${mcli_bin}"
|
|
|
|
# Setup S3 alias
|
|
log_info "Configuring S3 alias..."
|
|
safe_execute "${mcli_bin}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
|
|
|
|
# Construct destination path: rfs/{bucket}/{prefix}/kernel/{versioned_filename}
|
|
local kernel_filename
|
|
kernel_filename=$(basename "$kernel_file")
|
|
local kernel_subpath="${KERNEL_SUBPATH:-kernel}"
|
|
local mcli_dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${kernel_subpath%/}/${kernel_filename}"
|
|
|
|
# Upload kernel
|
|
log_info "Uploading: ${kernel_file} -> ${mcli_dst}"
|
|
safe_execute "${mcli_bin}" cp "${kernel_file}" "${mcli_dst}"
|
|
|
|
log_info "✓ Kernel uploaded successfully"
|
|
log_info " Version: ${kernel_version}"
|
|
log_info " zinit: ${zinit_hash}"
|
|
log_info " S3 path: ${mcli_dst}"
|
|
|
|
# Generate and upload kernel index
|
|
kernel_generate_index "${mcli_bin}" "${S3_BUCKET}" "${S3_PREFIX}" "${kernel_subpath}"
|
|
}
|
|
|
|
# Generate kernel index file from S3 listing and upload it
|
|
function kernel_generate_index() {
|
|
local mcli_bin="$1"
|
|
local bucket="$2"
|
|
local prefix="$3"
|
|
local kernel_subpath="$4"
|
|
|
|
section_header "Generating Kernel Index"
|
|
|
|
# Construct S3 path for listing
|
|
local s3_path="rfs/${bucket}/${prefix%/}/${kernel_subpath%/}/"
|
|
|
|
log_info "Listing kernels from: ${s3_path}"
|
|
|
|
# List all files in the kernel directory
|
|
local ls_output
|
|
if ! ls_output=$("${mcli_bin}" ls "${s3_path}" 2>&1); then
|
|
log_warn "Failed to list S3 kernel directory, index not generated"
|
|
log_debug "mcli ls output: ${ls_output}"
|
|
return 0
|
|
fi
|
|
|
|
# Parse output and extract kernel filenames matching vmlinuz-*
|
|
local kernels=()
|
|
while IFS= read -r line; do
|
|
# mcli ls output format: [DATE TIME TZ] SIZE FILENAME
|
|
# Extract filename (last field)
|
|
local filename
|
|
filename=$(echo "$line" | awk '{print $NF}')
|
|
|
|
# Filter for vmlinuz files (both .efi and without extension)
|
|
if [[ "$filename" =~ ^vmlinuz-.* ]]; then
|
|
kernels+=("$filename")
|
|
fi
|
|
done <<< "$ls_output"
|
|
|
|
if [[ ${#kernels[@]} -eq 0 ]]; then
|
|
log_warn "No kernels found in S3 path: ${s3_path}"
|
|
return 0
|
|
fi
|
|
|
|
log_info "Found ${#kernels[@]} kernel(s)"
|
|
|
|
# Create index files in dist directory
|
|
local index_dir="${DIST_DIR:-${PROJECT_ROOT}/dist}"
|
|
local text_index="${index_dir}/kernels.txt"
|
|
local json_index="${index_dir}/kernels.json"
|
|
|
|
# Generate text index (one kernel per line, sorted)
|
|
printf "%s\n" "${kernels[@]}" | sort -r > "$text_index"
|
|
log_info "Created text index: ${text_index}"
|
|
|
|
# Generate JSON index (array of kernel filenames)
|
|
{
|
|
echo "{"
|
|
echo " \"kernels\": ["
|
|
local first=true
|
|
for kernel in $(printf "%s\n" "${kernels[@]}" | sort -r); do
|
|
if [[ "$first" == "true" ]]; then
|
|
first=false
|
|
else
|
|
echo ","
|
|
fi
|
|
printf " \"%s\"" "$kernel"
|
|
done
|
|
echo ""
|
|
echo " ],"
|
|
echo " \"updated\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","
|
|
echo " \"count\": ${#kernels[@]}"
|
|
echo "}"
|
|
} > "$json_index"
|
|
log_info "Created JSON index: ${json_index}"
|
|
|
|
# Upload both index files to S3
|
|
log_info "Uploading kernel index files to S3..."
|
|
|
|
local text_dst="${s3_path}kernels.txt"
|
|
local json_dst="${s3_path}kernels.json"
|
|
|
|
if safe_execute "${mcli_bin}" cp "$text_index" "$text_dst"; then
|
|
log_info "✓ Uploaded text index: ${text_dst}"
|
|
else
|
|
log_warn "Failed to upload text index"
|
|
fi
|
|
|
|
if safe_execute "${mcli_bin}" cp "$json_index" "$json_dst"; then
|
|
log_info "✓ Uploaded JSON index: ${json_dst}"
|
|
else
|
|
log_warn "Failed to upload JSON index"
|
|
fi
|
|
|
|
log_info "Kernel index generation complete"
|
|
}
|
|
|
|
# Build and install modules in container for proper dependency resolution
|
|
function kernel_build_modules() {
|
|
local kernel_dir="$1"
|
|
local initramfs_dir="$2"
|
|
local base_version="${3:-$KERNEL_VERSION}"
|
|
|
|
section_header "Building Kernel Modules in Container"
|
|
|
|
local source_dir="${kernel_dir}/current"
|
|
|
|
if [[ ! -d "$source_dir" ]]; then
|
|
log_error "Kernel source directory not found: ${source_dir}"
|
|
return 1
|
|
fi
|
|
|
|
safe_execute cd "$source_dir"
|
|
|
|
# Get the full kernel version including LOCALVERSION
|
|
local full_version=$(kernel_get_full_version "$base_version" "${PROJECT_ROOT}/config/kernel.config")
|
|
log_info "Base kernel version: ${base_version}"
|
|
log_info "Full kernel version: ${full_version}"
|
|
|
|
# Build modules
|
|
local cores=$(nproc)
|
|
local jobs=$((cores > 1 ? cores - 1 : 1))
|
|
|
|
log_info "Building kernel modules with ${jobs} parallel jobs"
|
|
safe_execute make -j${jobs} modules
|
|
|
|
# Install modules in container for proper modinfo/depmod access
|
|
local container_modules_dir="/lib/modules"
|
|
log_info "Installing modules in container: ${container_modules_dir}"
|
|
safe_execute make modules_install INSTALL_MOD_PATH=/
|
|
|
|
# Run depmod in container context for proper dependency resolution
|
|
log_info "Running depmod in container for ${full_version}"
|
|
safe_execute depmod -a "$full_version"
|
|
|
|
# Verify module installation in container
|
|
if [[ -d "/lib/modules/${full_version}" ]]; then
|
|
local module_count=$(find "/lib/modules/${full_version}" -name "*.ko*" | wc -l)
|
|
log_info "Container modules installed: ${module_count} modules in /lib/modules/${full_version}"
|
|
|
|
# Export the container modules path for dependency resolution
|
|
export CONTAINER_MODULES_PATH="/lib/modules/${full_version}"
|
|
export KERNEL_FULL_VERSION="$full_version"
|
|
log_info "Module dependency resolution will use: ${CONTAINER_MODULES_PATH}"
|
|
else
|
|
log_error "Module installation in container failed"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Kernel modules build and container installation complete"
|
|
}
|
|
|
|
# Clean kernel build artifacts
|
|
function kernel_cleanup() {
|
|
local kernel_dir="$1"
|
|
local keep_source="${2:-false}"
|
|
|
|
section_header "Cleaning Kernel Build Artifacts"
|
|
|
|
if [[ "$keep_source" == "true" ]]; then
|
|
log_info "Keeping source, cleaning build artifacts only"
|
|
local source_dir="${kernel_dir}/current"
|
|
if [[ -d "$source_dir" ]]; then
|
|
safe_execute cd "$source_dir"
|
|
safe_execute make clean
|
|
fi
|
|
else
|
|
log_info "Removing entire kernel directory"
|
|
safe_rmdir "$kernel_dir"
|
|
fi
|
|
|
|
log_info "Kernel cleanup complete"
|
|
}
|
|
|
|
# Export functions
|
|
export -f kernel_download_source kernel_apply_config kernel_modify_config_for_initramfs
|
|
export -f kernel_build_with_initramfs kernel_build_modules kernel_cleanup kernel_get_full_version
|