From 947d156921fc7e3ae0e492570645494cc34ed2c1 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Tue, 11 Nov 2025 20:49:36 +0100 Subject: [PATCH] Added youki build and fromatting of scripts --- Dockerfile | 3 +- README.md | 70 ++- claude.md | 51 +- config/build.conf | 1 + config/sources.conf | 5 +- scripts/build.sh | 7 + scripts/dev-container.sh | 109 ++-- scripts/lib/common.sh | 20 + scripts/lib/components.sh | 1167 +++++++++++++++++++------------------ scripts/lib/kernel.sh | 203 ++++++- 10 files changed, 1013 insertions(+), 623 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67f31fe..b0d3c07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ RUN apk add --no-cache \ musl-utils \ pkgconfig \ openssl openssl-dev \ + libseccomp libseccomp-dev \ perl \ shadow \ bash \ @@ -55,4 +56,4 @@ RUN chown builder:builder /workspace # Set environment variables - rustup handles everything ENV PATH="/root/.cargo/bin:${PATH}" -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/README.md b/README.md index 8771191..7d8abae 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,9 @@ Services are migrated from existing `configs/zinit/` directory with proper initi ### Phase 6: Packaging 1. Create `initramfs.cpio.xz` with XZ compression 2. Build kernel with embedded initramfs -3. Generate `vmlinuz.efi` +3. Generate `vmlinuz.efi` (default kernel) +4. Generate versioned kernel: `vmlinuz-{VERSION}-{ZINIT_HASH}.efi` +5. Optionally upload versioned kernel to S3 (set `UPLOAD_KERNEL=true`) ## Testing @@ -357,6 +359,72 @@ function build_myapp() { } ``` +### S3 Uploads (Kernel & RFS Flists) + +Automatically upload build artifacts to S3-compatible storage: + +#### Configuration + +Create `config/rfs.conf`: + +```bash +S3_ENDPOINT="https://s3.example.com:9000" +S3_REGION="us-east-1" +S3_BUCKET="zos" +S3_PREFIX="flists/zosbuilder" +S3_ACCESS_KEY="YOUR_ACCESS_KEY" +S3_SECRET_KEY="YOUR_SECRET_KEY" +``` + +#### Upload Kernel + +```bash +# Enable kernel upload +UPLOAD_KERNEL=true ./scripts/build.sh + +# Custom kernel subpath (default: kernel) +KERNEL_SUBPATH=kernels UPLOAD_KERNEL=true ./scripts/build.sh +``` + +**Uploaded files:** +- `s3://{bucket}/{prefix}/kernel/vmlinuz-{VERSION}-{ZINIT_HASH}.efi` - Versioned kernel +- `s3://{bucket}/{prefix}/kernel/kernels.txt` - Text index (one kernel per line) +- `s3://{bucket}/{prefix}/kernel/kernels.json` - JSON index with metadata + +**Index files:** +The build automatically generates and uploads index files listing all available kernels in the S3 bucket. This enables: +- Easy kernel selection in web UIs (dropdown menus) +- Programmatic access without S3 API listing +- Metadata like upload timestamp and kernel count (JSON format) + +**JSON index format:** +```json +{ + "kernels": [ + "vmlinuz-6.12.44-Zero-OS-abc1234.efi", + "vmlinuz-6.12.44-Zero-OS-def5678.efi" + ], + "updated": "2025-01-04T12:00:00Z", + "count": 2 +} +``` + +#### Upload RFS Flists + +```bash +# Enable flist uploads +UPLOAD_MANIFESTS=true ./scripts/build.sh +``` + +Uploaded as: +- `s3://{bucket}/{prefix}/manifests/modules-{VERSION}.fl` +- `s3://{bucket}/{prefix}/manifests/firmware-{TAG}.fl` + +#### Requirements + +- MinIO Client (`mcli` or `mc`) must be installed +- Valid S3 credentials in `config/rfs.conf` + ### Container Builds Build in isolated container: diff --git a/claude.md b/claude.md index a38bcc4..5d248b7 100644 --- a/claude.md +++ b/claude.md @@ -160,6 +160,47 @@ release:corex:https://github.com/threefoldtech/corex/releases/download/2.1.4/cor **Service definitions:** YAML files in `config/zinit/` with `after:`, `needs:`, `wants:` dependencies +### 6. Kernel Versioning and S3 Upload + +**Versioned Kernel Output:** +- Standard kernel: `dist/vmlinuz.efi` (for compatibility) +- Versioned kernel: `dist/vmlinuz-{VERSION}-{ZINIT_HASH}.efi` +- Example: `vmlinuz-6.12.44-Zero-OS-a1b2c3d.efi` + +**Version components:** +- `{VERSION}`: Full kernel version from `KERNEL_VERSION` + `CONFIG_LOCALVERSION` +- `{ZINIT_HASH}`: Short git commit hash from `components/zinit/.git` + +**S3 Upload (optional):** +- Controlled by `UPLOAD_KERNEL=true` environment variable +- Uses MinIO client (`mcli` or `mc`) to upload to S3-compatible storage +- Uploads versioned kernel to: `s3://{bucket}/{prefix}/kernel/{versioned_filename}` + +**Kernel Index Generation:** +After uploading, automatically generates and uploads index files: +- `kernels.txt` - Plain text, one kernel per line, sorted reverse chronologically +- `kernels.json` - JSON format with metadata (timestamp, count) + +**Why index files?** +- S3 web interfaces often don't support directory listings +- Enables dropdown menus in web UIs without S3 API access +- Provides kernel discovery for deployment tools + +**JSON index structure:** +```json +{ + "kernels": ["vmlinuz-6.12.44-Zero-OS-abc1234.efi", ...], + "updated": "2025-01-04T12:00:00Z", + "count": 2 +} +``` + +**Key functions:** +- `get_git_commit_hash()` in `scripts/lib/common.sh` - Extracts git hash +- `kernel_build_with_initramfs()` in `scripts/lib/kernel.sh` - Creates versioned kernel +- `kernel_upload_to_s3()` in `scripts/lib/kernel.sh` - Uploads to S3 +- `kernel_generate_index()` in `scripts/lib/kernel.sh` - Generates and uploads index + ## Critical Conventions ### Path Normalization @@ -394,8 +435,14 @@ rm -rf .build-stages/ # Reset stage markers **Firmware tagging:** - `FIRMWARE_TAG=20250908` - Firmware flist version tag -**RFS configuration:** +**S3 upload control:** +- `UPLOAD_KERNEL=true` - Upload versioned kernel to S3 (default: false) +- `UPLOAD_MANIFESTS=true` - Upload RFS flists to S3 (default: false) +- `KERNEL_SUBPATH=kernel` - S3 subpath for kernel uploads (default: kernel) + +**S3 configuration:** - See `config/rfs.conf` for S3 endpoint, credentials, paths +- Used by both RFS flist uploads and kernel uploads ## Documentation Hierarchy @@ -466,6 +513,8 @@ Co-Authored-By: Claude - **Size too large:** Check cleanup stage, strip/UPX execution, package list - **Container issues:** Rootless setup, subuid/subgid, podman vs docker - **RFS mount fails:** S3 credentials, network readiness, flist manifest paths +- **Kernel upload:** `UPLOAD_KERNEL=true`, requires `config/rfs.conf`, MinIO client (`mcli`/`mc`) +- **Kernel index:** Auto-generated `kernels.txt`/`kernels.json` for dropdown UIs, updated on upload --- diff --git a/config/build.conf b/config/build.conf index 3276c70..a9c4a1d 100644 --- a/config/build.conf +++ b/config/build.conf @@ -61,6 +61,7 @@ ENABLE_STRIP="true" ENABLE_UPX="true" ENABLE_AGGRESSIVE_CLEANUP="true" ENABLE_2STAGE_MODULES="true" +UPLOAD_KERNEL=true # Debug and development DEBUG_DEFAULT="0" diff --git a/config/sources.conf b/config/sources.conf index 501970d..d9dd1fd 100644 --- a/config/sources.conf +++ b/config/sources.conf @@ -4,8 +4,9 @@ # Git repositories to clone and build git zinit https://github.com/threefoldtech/zinit master build_zinit git mycelium https://github.com/threefoldtech/mycelium v0.6.1 build_mycelium -git zosstorage git@git.ourworld.tf:delandtj/zosstorage main build_zosstorage +git zosstorage https://git.ourworld.tf/delandtj/zosstorage main build_zosstorage +git youki git@github.com:youki-dev/youki.git v0.5.7 build_youki git rfs https://github.com/threefoldtech/rfs development build_rfs # Pre-built releases to download # release rfs https://github.com/threefoldtech/rfs/releases/download/v2.0.6/rfs v2.0.6 install_rfs -release corex https://github.com/threefoldtech/corex/releases/download/2.1.4/corex-2.1.4-amd64-linux-static 2.1.4 install_corex rename=corex \ No newline at end of file +release corex https://github.com/threefoldtech/corex/releases/download/2.1.4/corex-2.1.4-amd64-linux-static 2.1.4 install_corex rename=corex diff --git a/scripts/build.sh b/scripts/build.sh index 22b7b01..876bd87 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -401,6 +401,13 @@ function main_build_process() { log_debug "stage_kernel_build: defaulting INITRAMFS_ARCHIVE=${INITRAMFS_ARCHIVE}" fi + # Ensure FULL_KERNEL_VERSION is set for versioned output filename + if [[ -z "${FULL_KERNEL_VERSION:-}" ]]; then + FULL_KERNEL_VERSION=$(kernel_get_full_version "$KERNEL_VERSION" "$KERNEL_CONFIG") + export FULL_KERNEL_VERSION + log_debug "stage_kernel_build: resolved FULL_KERNEL_VERSION=${FULL_KERNEL_VERSION}" + fi + kernel_build_with_initramfs "$KERNEL_CONFIG" "$INITRAMFS_ARCHIVE" "$kernel_output" export KERNEL_OUTPUT="$kernel_output" } diff --git a/scripts/dev-container.sh b/scripts/dev-container.sh index 7209a24..c14591d 100755 --- a/scripts/dev-container.sh +++ b/scripts/dev-container.sh @@ -18,7 +18,7 @@ export DEBUG="${DEBUG:-1}" source "${SCRIPT_DIR}/lib/common.sh" function show_usage() { - cat << EOF + cat </dev/null; then if podman container inspect "$CONTAINER_NAME" --format '{{.State.Status}}' | grep -q "running"; then @@ -85,15 +85,16 @@ function dev_container_start() { return 0 fi fi - + log_info "Creating new development container: ${CONTAINER_NAME}" - + # Create persistent container with all necessary mounts and environment local podman_args=( run -d --name "$CONTAINER_NAME" --privileged -v "${PROJECT_ROOT}:/workspace" + -v "$HOME/.ssh:root/.ssh" -w /workspace -e DEBUG=1 -e ALPINE_VERSION=3.22 @@ -115,7 +116,7 @@ function dev_container_start() { ) safe_execute podman "${podman_args[@]}" - + log_info "Development container started successfully" log_info "Container name: ${CONTAINER_NAME}" log_info "Access with: $0 shell" @@ -123,7 +124,7 @@ function dev_container_start() { function dev_container_stop() { section_header "Stopping Development Container" - + if podman container exists "$CONTAINER_NAME" 2>/dev/null; then log_info "Stopping development container: ${CONTAINER_NAME}" safe_execute podman stop "$CONTAINER_NAME" @@ -135,17 +136,17 @@ function dev_container_stop() { function dev_container_shell() { section_header "Entering Development Container Shell" - + if ! podman container exists "$CONTAINER_NAME" 2>/dev/null; then log_info "Development container not found, starting..." dev_container_start fi - + if ! podman container inspect "$CONTAINER_NAME" --format '{{.State.Status}}' | grep -q "running"; then log_info "Starting stopped development container" safe_execute podman start "$CONTAINER_NAME" fi - + log_info "Entering container shell (exit with 'exit' or Ctrl+D)" # Use direct execution for interactive shell (don't use safe_execute) exec podman exec -it "$CONTAINER_NAME" /bin/bash @@ -153,56 +154,56 @@ function dev_container_shell() { function dev_container_build() { section_header "Running Build in Development Container" - + if ! podman container exists "$CONTAINER_NAME" 2>/dev/null; then log_info "Development container not found, starting..." dev_container_start fi - + if ! podman container inspect "$CONTAINER_NAME" --format '{{.State.Status}}' | grep -q "running"; then log_info "Starting stopped development container" safe_execute podman start "$CONTAINER_NAME" fi - + log_info "Running build in persistent container (real-time output)" log_info "Command: podman exec $CONTAINER_NAME ./scripts/build.sh $*" - + # Use direct execution to show real-time output (bypass safe_execute) podman exec "$CONTAINER_NAME" ./scripts/build.sh "$@" local exit_code=$? - + if [[ $exit_code -eq 0 ]]; then log_info "Build completed successfully in container" else log_error "Build failed in container with exit code: $exit_code" fi - + return $exit_code } function dev_container_clean() { section_header "Cleaning Development Container" - + if podman container exists "$CONTAINER_NAME" 2>/dev/null; then log_info "Removing existing development container" safe_execute podman rm -f "$CONTAINER_NAME" fi - + log_info "Starting fresh development container" dev_container_start } function dev_container_status() { section_header "Development Container Status" - + if podman container exists "$CONTAINER_NAME" 2>/dev/null; then local status=$(podman container inspect "$CONTAINER_NAME" --format '{{.State.Status}}') local created=$(podman container inspect "$CONTAINER_NAME" --format '{{.Created}}') - + log_info "Container: ${CONTAINER_NAME}" log_info "Status: ${status}" log_info "Created: ${created}" - + if [[ "$status" == "running" ]]; then log_info "✓ Ready for development" else @@ -216,7 +217,7 @@ function dev_container_status() { function dev_container_logs() { section_header "Development Container Logs" - + if podman container exists "$CONTAINER_NAME" 2>/dev/null; then safe_execute podman logs "$CONTAINER_NAME" else @@ -228,39 +229,39 @@ function dev_container_logs() { # Main function function main() { local command="${1:-help}" - + case "$command" in - start) - dev_container_start - ;; - stop) - dev_container_stop - ;; - shell) - dev_container_shell - ;; - build) - shift - dev_container_build "$@" - ;; - clean) - dev_container_clean - ;; - status) - dev_container_status - ;; - logs) - dev_container_logs - ;; - help|--help|-h) - show_usage - ;; - *) - log_error "Unknown command: $command" - show_usage - exit 1 - ;; + start) + dev_container_start + ;; + stop) + dev_container_stop + ;; + shell) + dev_container_shell + ;; + build) + shift + dev_container_build "$@" + ;; + clean) + dev_container_clean + ;; + status) + dev_container_status + ;; + logs) + dev_container_logs + ;; + help | --help | -h) + show_usage + ;; + *) + log_error "Unknown command: $command" + show_usage + exit 1 + ;; esac } -main "$@" \ No newline at end of file +main "$@" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 58aa08b..5da9bfb 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -195,6 +195,26 @@ function get_file_size() { fi } +# Get short git commit hash from a git repository directory +function get_git_commit_hash() { + local repo_dir="$1" + local short="${2:-true}" # Default to short hash + + if [[ ! -d "$repo_dir/.git" ]]; then + echo "unknown" + return 1 + fi + + local hash + if [[ "$short" == "true" ]]; then + hash=$(cd "$repo_dir" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") + else + hash=$(cd "$repo_dir" && git rev-parse HEAD 2>/dev/null || echo "unknown") + fi + + echo "$hash" +} + # Wait for file to exist with timeout function wait_for_file() { local file="$1" diff --git a/scripts/lib/components.sh b/scripts/lib/components.sh index 76dbab3..e6a5e18 100644 --- a/scripts/lib/components.sh +++ b/scripts/lib/components.sh @@ -11,642 +11,685 @@ CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" # Parse and process all components from sources.conf function components_parse_sources_conf() { - local sources_file="$1" - local components_dir="$2" - local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}" - - section_header "Parsing Sources Configuration" - - if [[ ! -f "$sources_file" ]]; then - log_error "Sources file not found: ${sources_file}" - return 1 - fi - - # Ensure components directory exists - safe_mkdir "$components_dir" - - # Export install directory for build functions - export INSTALL_DIR="$install_dir" - - log_info "Processing components from: ${sources_file}" - log_info "Components directory: ${components_dir}" - log_info "Install directory: ${install_dir}" - - local component_count=0 + local sources_file="$1" + local components_dir="$2" + local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}" - # Read entries from sources.conf (TYPE NAME URL VERSION BUILD_FUNCTION [EXTRA]) - while IFS= read -r _raw || [[ -n "$_raw" ]]; do - # Strip comments and trim whitespace - local line="${_raw%%#*}" - line="${line#"${line%%[![:space:]]*}"}" - line="${line%"${line##*[![:space:]]}"}" - [[ -z "$line" ]] && continue + section_header "Parsing Sources Configuration" - local type name url version build_func extra - # shellcheck disable=SC2086 - read -r type name url version build_func extra <<< "$line" + if [[ ! -f "$sources_file" ]]; then + log_error "Sources file not found: ${sources_file}" + return 1 + fi - if [[ -z "${type:-}" || -z "${name:-}" || -z "${url:-}" || -z "${version:-}" || -z "${build_func:-}" ]]; then - log_warn "Skipping malformed entry: ${_raw}" - continue - fi + # Ensure components directory exists + safe_mkdir "$components_dir" - component_count=$((component_count + 1)) - log_info "Processing component ${component_count}: ${name} (${type})" + # Export install directory for build functions + export INSTALL_DIR="$install_dir" - case "$type" in - git) - components_download_git "$name" "$url" "$version" "$components_dir" - ;; - release) - components_download_release "$name" "$url" "$version" "$components_dir" "$extra" - ;; - *) - log_error "Unknown component type in sources.conf: ${type}" - return 1 - ;; - esac + log_info "Processing components from: ${sources_file}" + log_info "Components directory: ${components_dir}" + log_info "Install directory: ${install_dir}" - components_build_component "$name" "$build_func" "$components_dir" - done < "$sources_file" - - if [[ $component_count -eq 0 ]]; then - log_warn "No components found in sources configuration" - else - log_info "Processed ${component_count} components successfully" - fi + local component_count=0 + + # Read entries from sources.conf (TYPE NAME URL VERSION BUILD_FUNCTION [EXTRA]) + while IFS= read -r _raw || [[ -n "$_raw" ]]; do + # Strip comments and trim whitespace + local line="${_raw%%#*}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "$line" ]] && continue + + local type name url version build_func extra + # shellcheck disable=SC2086 + read -r type name url version build_func extra <<<"$line" + + if [[ -z "${type:-}" || -z "${name:-}" || -z "${url:-}" || -z "${version:-}" || -z "${build_func:-}" ]]; then + log_warn "Skipping malformed entry: ${_raw}" + continue + fi + + component_count=$((component_count + 1)) + log_info "Processing component ${component_count}: ${name} (${type})" + + case "$type" in + git) + components_download_git "$name" "$url" "$version" "$components_dir" + ;; + release) + components_download_release "$name" "$url" "$version" "$components_dir" "$extra" + ;; + *) + log_error "Unknown component type in sources.conf: ${type}" + return 1 + ;; + esac + + components_build_component "$name" "$build_func" "$components_dir" + done <"$sources_file" + + if [[ $component_count -eq 0 ]]; then + log_warn "No components found in sources configuration" + else + log_info "Processed ${component_count} components successfully" + fi } # Download Git repository (reuse tree; only reclone if invalid or version not reachable) function components_download_git() { - local name="$1" - local url="$2" - local version="$3" - local components_dir="$4" + local name="$1" + local url="$2" + local version="$3" + local components_dir="$4" - section_header "Downloading Git Component: ${name}" + section_header "Downloading Git Component: ${name}" - local target_dir="${components_dir}/${name}" + local target_dir="${components_dir}/${name}" - log_info "Repository: ${url}" - log_info "Version/Branch/Tag: ${version}" - log_info "Target directory: ${target_dir}" + log_info "Repository: ${url}" + log_info "Version/Branch/Tag: ${version}" + log_info "Target directory: ${target_dir}" - # Ensure parent exists - safe_mkdir "$components_dir" + # Ensure parent exists + safe_mkdir "$components_dir" - # Decide whether we can reuse the existing working tree - local need_fresh_clone="0" - if [[ -d "$target_dir/.git" ]]; then - if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - log_warn "Existing ${name} directory is not a valid git repo; will reclone" - need_fresh_clone="1" - fi - elif [[ -d "$target_dir" ]]; then - log_warn "Existing ${name} directory without .git; will reclone" - need_fresh_clone="1" - fi + # Decide whether we can reuse the existing working tree + local need_fresh_clone="0" + if [[ -d "$target_dir/.git" ]]; then + if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + log_warn "Existing ${name} directory is not a valid git repo; will reclone" + need_fresh_clone="1" + fi + elif [[ -d "$target_dir" ]]; then + log_warn "Existing ${name} directory without .git; will reclone" + need_fresh_clone="1" + fi - if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then - log_info "Cloning ${name} (fresh) from ${url}" - safe_execute git clone "$url" "$target_dir" - fi + if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then + log_info "Cloning ${name} (fresh) from ${url}" + safe_execute git clone "$url" "$target_dir" + fi - # Ensure origin URL is correct (do not delete the tree if URL changed) - local current_url - current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "") - if [[ -n "$current_url" && "$current_url" != "$url" ]]; then - log_info "Updating origin URL: ${current_url} -> ${url}" - safe_execute git -C "$target_dir" remote set-url origin "$url" - elif [[ -z "$current_url" ]]; then - log_info "Setting origin URL to ${url}" - safe_execute git -C "$target_dir" remote add origin "$url" || true - fi + # Ensure origin URL is correct (do not delete the tree if URL changed) + local current_url + current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "") + if [[ -n "$current_url" && "$current_url" != "$url" ]]; then + log_info "Updating origin URL: ${current_url} -> ${url}" + safe_execute git -C "$target_dir" remote set-url origin "$url" + elif [[ -z "$current_url" ]]; then + log_info "Setting origin URL to ${url}" + safe_execute git -C "$target_dir" remote add origin "$url" || true + fi - # Fetch updates and tags - safe_execute git -C "$target_dir" fetch --tags --prune origin + # Fetch updates and tags + safe_execute git -C "$target_dir" fetch --tags --prune origin - # Resolve desired commit for the requested version/branch/tag - local desired_rev="" - if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then - desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}") - elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then - desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}") - else - log_warn "Version '${version}' not directly resolvable; fetching explicitly" - if git -C "$target_dir" fetch origin "${version}" --depth 1; then - desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD) - fi - fi + # Resolve desired commit for the requested version/branch/tag + local desired_rev="" + if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}") + elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}") + else + log_warn "Version '${version}' not directly resolvable; fetching explicitly" + if git -C "$target_dir" fetch origin "${version}" --depth 1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD) + fi + fi - # Fallback: shallow clone at the requested ref if we still can't resolve - if [[ -z "$desired_rev" ]]; then - log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref." - safe_execute rm -rf "${target_dir}.tmp" - if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then - safe_execute rm -rf "$target_dir" - safe_execute mv "${target_dir}.tmp" "$target_dir" - desired_rev=$(git -C "$target_dir" rev-parse HEAD) - else - log_error "Failed to clone ${url} at '${version}'" - return 1 - fi - fi + # Fallback: shallow clone at the requested ref if we still can't resolve + if [[ -z "$desired_rev" ]]; then + log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref." + safe_execute rm -rf "${target_dir}.tmp" + if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then + safe_execute rm -rf "$target_dir" + safe_execute mv "${target_dir}.tmp" "$target_dir" + desired_rev=$(git -C "$target_dir" rev-parse HEAD) + else + log_error "Failed to clone ${url} at '${version}'" + return 1 + fi + fi - local current_rev - current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "") - log_info "Current commit: ${current_rev:-}" - log_info "Desired commit: ${desired_rev}" + local current_rev + current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "") + log_info "Current commit: ${current_rev:-}" + log_info "Desired commit: ${desired_rev}" - if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then - log_info "Repository already at requested version; reusing working tree" - else - log_info "Checking out requested version" - # Prefer named refs when available; otherwise detach to exact commit - if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then - safe_execute git -C "$target_dir" checkout -f "${version}" - elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then - safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}" - elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then - safe_execute git -C "$target_dir" checkout -f "tags/${version}" - else - safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}" - fi - # Initialize submodules if present (non-fatal) - safe_execute git -C "$target_dir" submodule update --init --recursive || true - fi + if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then + log_info "Repository already at requested version; reusing working tree" + else + log_info "Checking out requested version" + # Prefer named refs when available; otherwise detach to exact commit + if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then + safe_execute git -C "$target_dir" checkout -f "${version}" + elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then + safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}" + elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then + safe_execute git -C "$target_dir" checkout -f "tags/${version}" + else + safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}" + fi + # Initialize submodules if present (non-fatal) + safe_execute git -C "$target_dir" submodule update --init --recursive || true + fi - log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)" + log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)" } # Download release binary/archive function components_download_release() { - local name="$1" - local url="$2" - local version="$3" - local components_dir="$4" - local extra="$5" - - section_header "Downloading Release Component: ${name}" - - local target_dir="${components_dir}/${name}" - local filename=$(basename "$url") - - log_info "Release URL: ${url}" - log_info "Version: ${version}" - log_info "Target directory: ${target_dir}" - - safe_mkdir "$target_dir" - - # Download release - log_info "Downloading release: ${filename}" - safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url" - - # Verify download - if [[ ! -f "${target_dir}/${filename}" ]]; then - log_error "Failed to download release: ${filename}" - return 1 - fi - - local file_size=$(get_file_size "${target_dir}/${filename}") - log_info "Downloaded file size: ${file_size}" - - # Handle extra options (like rename) - if [[ -n "$extra" ]]; then - components_process_extra_options "$target_dir" "$filename" "$extra" - fi - - log_info "Release component download complete: ${name}" + local name="$1" + local url="$2" + local version="$3" + local components_dir="$4" + local extra="$5" + + section_header "Downloading Release Component: ${name}" + + local target_dir="${components_dir}/${name}" + local filename=$(basename "$url") + + log_info "Release URL: ${url}" + log_info "Version: ${version}" + log_info "Target directory: ${target_dir}" + + safe_mkdir "$target_dir" + + # Download release + log_info "Downloading release: ${filename}" + safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url" + + # Verify download + if [[ ! -f "${target_dir}/${filename}" ]]; then + log_error "Failed to download release: ${filename}" + return 1 + fi + + local file_size=$(get_file_size "${target_dir}/${filename}") + log_info "Downloaded file size: ${file_size}" + + # Handle extra options (like rename) + if [[ -n "$extra" ]]; then + components_process_extra_options "$target_dir" "$filename" "$extra" + fi + + log_info "Release component download complete: ${name}" } # Process extra options for components function components_process_extra_options() { - local target_dir="$1" - local filename="$2" - local extra="$3" - - log_info "Processing extra options: ${extra}" - - # Handle rename option - if [[ "$extra" =~ rename=(.+) ]]; then - local new_name="${BASH_REMATCH[1]}" - log_info "Renaming ${filename} to ${new_name}" - safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}" - fi - - # Handle extract option for archives - if [[ "$extra" =~ extract ]]; then - log_info "Extracting archive: ${filename}" - safe_execute cd "$target_dir" - case "$filename" in - *.tar.gz|*.tgz) - safe_execute tar -xzf "$filename" - ;; - *.tar.bz2|*.tbz2) - safe_execute tar -xjf "$filename" - ;; - *.tar.xz|*.txz) - safe_execute tar -xJf "$filename" - ;; - *.zip) - safe_execute unzip "$filename" - ;; - *) - log_warn "Unknown archive format: ${filename}" - ;; - esac - fi + local target_dir="$1" + local filename="$2" + local extra="$3" + + log_info "Processing extra options: ${extra}" + + # Handle rename option + if [[ "$extra" =~ rename=(.+) ]]; then + local new_name="${BASH_REMATCH[1]}" + log_info "Renaming ${filename} to ${new_name}" + safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}" + fi + + # Handle extract option for archives + if [[ "$extra" =~ extract ]]; then + log_info "Extracting archive: ${filename}" + safe_execute cd "$target_dir" + case "$filename" in + *.tar.gz | *.tgz) + safe_execute tar -xzf "$filename" + ;; + *.tar.bz2 | *.tbz2) + safe_execute tar -xjf "$filename" + ;; + *.tar.xz | *.txz) + safe_execute tar -xJf "$filename" + ;; + *.zip) + safe_execute unzip "$filename" + ;; + *) + log_warn "Unknown archive format: ${filename}" + ;; + esac + fi } # Build component using specified build function function components_build_component() { - local name="$1" - local build_func="$2" - local components_dir="$3" - - section_header "Building Component: ${name}" - - local component_dir="${components_dir}/${name}" - - if [[ ! -d "$component_dir" ]]; then - log_error "Component directory not found: ${component_dir}" - return 1 - fi - - # Change to component directory - safe_execute cd "$component_dir" - - log_info "Build function: ${build_func}" - log_info "Working directory: $(pwd)" - - # Check if build function exists - if ! declare -f "$build_func" >/dev/null; then - log_error "Build function not found: ${build_func}" - return 1 - fi - - # Call the specific build function - log_info "Executing build function: ${build_func}" - "$build_func" "$name" "$component_dir" - - log_info "Component build complete: ${name}" + local name="$1" + local build_func="$2" + local components_dir="$3" + + section_header "Building Component: ${name}" + + local component_dir="${components_dir}/${name}" + + if [[ ! -d "$component_dir" ]]; then + log_error "Component directory not found: ${component_dir}" + return 1 + fi + + # Change to component directory + safe_execute cd "$component_dir" + + log_info "Build function: ${build_func}" + log_info "Working directory: $(pwd)" + + # Check if build function exists + if ! declare -f "$build_func" >/dev/null; then + log_error "Build function not found: ${build_func}" + return 1 + fi + + # Call the specific build function + log_info "Executing build function: ${build_func}" + "$build_func" "$name" "$component_dir" + + log_info "Component build complete: ${name}" } # Setup Rust environment for musl builds function components_setup_rust_env() { - section_header "Setting Up Rust Environment" - - # Source cargo environment if available - if [[ -f /root/.cargo/env ]]; then - log_info "Sourcing cargo environment from /root/.cargo/env" - source /root/.cargo/env - fi - - # Check if we have rustup (should be available now) - if command_exists "rustup"; then - log_info "Using rustup for Rust toolchain management" - - # Ensure musl target is installed - if ! rustup target list --installed | grep -q "$RUST_TARGET"; then - log_info "Installing Rust target: ${RUST_TARGET}" - safe_execute rustup target add "$RUST_TARGET" - else - log_info "Rust target already installed: ${RUST_TARGET}" - fi - - # Set environment variables for rustup (clean and simple) - export RUSTFLAGS="-C target-feature=+crt-static" - else - log_error "rustup not found after setup" - return 1 - fi - - log_info "Rust environment configured for musl builds" - log_info "RUST_TARGET: ${RUST_TARGET}" - log_info "RUSTFLAGS: ${RUSTFLAGS}" - log_info "CC: ${CC:-system-default}" + section_header "Setting Up Rust Environment" + + # Source cargo environment if available + if [[ -f /root/.cargo/env ]]; then + log_info "Sourcing cargo environment from /root/.cargo/env" + source /root/.cargo/env + fi + + # Check if we have rustup (should be available now) + if command_exists "rustup"; then + log_info "Using rustup for Rust toolchain management" + + # Ensure musl target is installed + if ! rustup target list --installed | grep -q "$RUST_TARGET"; then + log_info "Installing Rust target: ${RUST_TARGET}" + safe_execute rustup target add "$RUST_TARGET" + else + log_info "Rust target already installed: ${RUST_TARGET}" + fi + + # Set environment variables for rustup (clean and simple) + export RUSTFLAGS="-C target-feature=+crt-static" + else + log_error "rustup not found after setup" + return 1 + fi + + log_info "Rust environment configured for musl builds" + log_info "RUST_TARGET: ${RUST_TARGET}" + log_info "RUSTFLAGS: ${RUSTFLAGS}" + log_info "CC: ${CC:-system-default}" } # Build function for zinit (standard Rust build) function build_zinit() { - local name="$1" - local component_dir="$2" - - section_header "Building zinit with musl target" - - components_setup_rust_env - - log_info "Building zinit from: ${component_dir}" - - # Ensure we're in the correct directory - if [[ ! -d "$component_dir" ]]; then - log_error "Component directory not found: ${component_dir}" - return 1 - fi - - # Don't use safe_execute for cd - it runs in subshell - log_info "Executing: cd $component_dir" - cd "$component_dir" || { - log_error "Failed to change to directory: $component_dir" - return 1 - } - - local current_dir=$(pwd) - log_info "Current directory: ${current_dir}" - - # Verify Cargo.toml exists - if [[ ! -f "Cargo.toml" ]]; then - log_error "Cargo.toml not found in: ${current_dir}" - return 1 - fi - - # Build with musl target (rustup properly configured) - safe_execute cargo build --release --target "$RUST_TARGET" - - # Find and install binary - local binary_path="target/${RUST_TARGET}/release/zinit" - if [[ ! -f "$binary_path" ]]; then - log_error "zinit binary not found at: ${binary_path}" - return 1 - fi - - local binary_size=$(get_file_size "$binary_path") - log_info "Built zinit binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Building zinit with musl target" + + components_setup_rust_env + + log_info "Building zinit from: ${component_dir}" + + # Ensure we're in the correct directory + if [[ ! -d "$component_dir" ]]; then + log_error "Component directory not found: ${component_dir}" + return 1 + fi + + # Don't use safe_execute for cd - it runs in subshell + log_info "Executing: cd $component_dir" + cd "$component_dir" || { + log_error "Failed to change to directory: $component_dir" + return 1 + } + + local current_dir=$(pwd) + log_info "Current directory: ${current_dir}" + + # Verify Cargo.toml exists + if [[ ! -f "Cargo.toml" ]]; then + log_error "Cargo.toml not found in: ${current_dir}" + return 1 + fi + + # Build with musl target (rustup properly configured) + safe_execute cargo build --release --target "$RUST_TARGET" + + # Find and install binary + local binary_path="target/${RUST_TARGET}/release/zinit" + if [[ ! -f "$binary_path" ]]; then + log_error "zinit binary not found at: ${binary_path}" + return 1 + fi + + local binary_size=$(get_file_size "$binary_path") + log_info "Built zinit binary (${binary_size}) at: ${binary_path}" } # Build function for rfs (standard Rust build) function build_rfs() { - local name="$1" - local component_dir="$2" - - section_header "Building rfs with musl target" - - components_setup_rust_env - - log_info "Building rfs from: ${component_dir}" - - # Ensure we're in the correct directory - if [[ ! -d "$component_dir" ]]; then - log_error "Component directory not found: ${component_dir}" - return 1 - fi - - # Don't use safe_execute for cd - it runs in subshell - log_info "Executing: cd $component_dir" - cd "$component_dir" || { - log_error "Failed to change to directory: $component_dir" - return 1 - } - - local current_dir=$(pwd) - log_info "Current directory: ${current_dir}" - - # Verify Cargo.toml exists - if [[ ! -f "Cargo.toml" ]]; then - log_error "Cargo.toml not found in: ${current_dir}" - return 1 - fi - # remove rust-toolchain.toml, as not needed with latest release - # Build with musl target - safe_execute cargo build --release --target "$RUST_TARGET" --features build-binary - - # Find and install binary - local binary_path="target/${RUST_TARGET}/release/rfs" - if [[ ! -f "$binary_path" ]]; then - log_error "rfs binary not found at: ${binary_path}" - return 1 - fi - - local binary_size=$(get_file_size "$binary_path") - log_info "Built rfs binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Building rfs with musl target" + + components_setup_rust_env + + log_info "Building rfs from: ${component_dir}" + + # Ensure we're in the correct directory + if [[ ! -d "$component_dir" ]]; then + log_error "Component directory not found: ${component_dir}" + return 1 + fi + + # Don't use safe_execute for cd - it runs in subshell + log_info "Executing: cd $component_dir" + cd "$component_dir" || { + log_error "Failed to change to directory: $component_dir" + return 1 + } + + local current_dir=$(pwd) + log_info "Current directory: ${current_dir}" + + # Verify Cargo.toml exists + if [[ ! -f "Cargo.toml" ]]; then + log_error "Cargo.toml not found in: ${current_dir}" + return 1 + fi + # remove rust-toolchain.toml, as not needed with latest release + # Build with musl target + safe_execute cargo build --release --target "$RUST_TARGET" --features build-binary + + # Find and install binary + local binary_path="target/${RUST_TARGET}/release/rfs" + if [[ ! -f "$binary_path" ]]; then + log_error "rfs binary not found at: ${binary_path}" + return 1 + fi + + local binary_size=$(get_file_size "$binary_path") + log_info "Built rfs binary (${binary_size}) at: ${binary_path}" } # Build function for zosstorage (standard Rust build) function build_zosstorage() { - local name="$1" - local component_dir="$2" - - section_header "Building zosstorage with musl target" - - components_setup_rust_env - - log_info "Building zosstorage from: ${component_dir}" - - if [[ ! -d "$component_dir" ]]; then - log_error "Component directory not found: ${component_dir}" - return 1 - fi - - log_info "Executing: cd $component_dir" - cd "$component_dir" || { - log_error "Failed to change to directory: $component_dir" - return 1 - } - - local current_dir - current_dir=$(pwd) - log_info "Current directory: ${current_dir}" - - if [[ ! -f "Cargo.toml" ]]; then - log_error "Cargo.toml not found in: ${current_dir}" - return 1 - fi - - safe_execute cargo build --release --target "$RUST_TARGET" - - local binary_path="target/${RUST_TARGET}/release/zosstorage" - if [[ ! -f "$binary_path" ]]; then - log_error "zosstorage binary not found at: ${binary_path}" - return 1 - fi - - local binary_size - binary_size=$(get_file_size "$binary_path") - log_info "Built zosstorage binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Building zosstorage with musl target" + + components_setup_rust_env + + log_info "Building zosstorage from: ${component_dir}" + + if [[ ! -d "$component_dir" ]]; then + log_error "Component directory not found: ${component_dir}" + return 1 + fi + + log_info "Executing: cd $component_dir" + cd "$component_dir" || { + log_error "Failed to change to directory: $component_dir" + return 1 + } + + local current_dir + current_dir=$(pwd) + log_info "Current directory: ${current_dir}" + + if [[ ! -f "Cargo.toml" ]]; then + log_error "Cargo.toml not found in: ${current_dir}" + return 1 + fi + + safe_execute cargo build --release --target "$RUST_TARGET" + + local binary_path="target/${RUST_TARGET}/release/zosstorage" + if [[ ! -f "$binary_path" ]]; then + log_error "zosstorage binary not found at: ${binary_path}" + return 1 + fi + + local binary_size + binary_size=$(get_file_size "$binary_path") + log_info "Built zosstorage binary (${binary_size}) at: ${binary_path}" +} + +# Build function for youki (standard Rust build) +function build_youki() { + local name="$1" + local component_dir="$2" + + section_header "Building youki with musl target" + + components_setup_rust_env + + log_info "Building youki from: ${component_dir}" + + if [[ ! -d "$component_dir" ]]; then + log_error "Component directory not found: ${component_dir}" + return 1 + fi + + log_info "Executing: cd $component_dir" + cd "$component_dir" || { + log_error "Failed to change to directory: $component_dir" + return 1 + } + + local current_dir + current_dir=$(pwd) + log_info "Current directory: ${current_dir}" + + if [[ ! -f "Cargo.toml" ]]; then + log_error "Cargo.toml not found in: ${current_dir}" + return 1 + fi + + safe_execute cargo build --release --target "$RUST_TARGET" + + local binary_path="target/${RUST_TARGET}/release/youki" + if [[ ! -f "$binary_path" ]]; then + log_error "youki binary not found at: ${binary_path}" + return 1 + fi + + local binary_size + binary_size=$(get_file_size "$binary_path") + log_info "Built youki binary (${binary_size}) at: ${binary_path}" } - # Build function for mycelium (special subdirectory build) function build_mycelium() { - local name="$1" - local component_dir="$2" - - section_header "Building mycelium with musl target (special directory)" - - components_setup_rust_env - - log_info "Building mycelium from: ${component_dir}" - - # Change to myceliumd subdirectory (special requirement) - local myceliumd_dir="${component_dir}/myceliumd" - if [[ ! -d "$myceliumd_dir" ]]; then - log_error "myceliumd directory not found at: ${myceliumd_dir}" - return 1 - fi - - # Don't use safe_execute for cd - it runs in subshell - log_info "Executing: cd $myceliumd_dir" - cd "$myceliumd_dir" || { - log_error "Failed to change to myceliumd directory: $myceliumd_dir" - return 1 - } - log_info "Building in myceliumd subdirectory: $(pwd)" - - # Build with musl target - safe_execute cargo build --release --target "$RUST_TARGET" - - # Find and install binary (from target/x86.../release) - local binary_path="target/${RUST_TARGET}/release/mycelium" - if [[ ! -f "$binary_path" ]]; then - log_error "mycelium binary not found at: ${binary_path}" - return 1 - fi - - local binary_size=$(get_file_size "$binary_path") - log_info "Built mycelium binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Building mycelium with musl target (special directory)" + + components_setup_rust_env + + log_info "Building mycelium from: ${component_dir}" + + # Change to myceliumd subdirectory (special requirement) + local myceliumd_dir="${component_dir}/myceliumd" + if [[ ! -d "$myceliumd_dir" ]]; then + log_error "myceliumd directory not found at: ${myceliumd_dir}" + return 1 + fi + + # Don't use safe_execute for cd - it runs in subshell + log_info "Executing: cd $myceliumd_dir" + cd "$myceliumd_dir" || { + log_error "Failed to change to myceliumd directory: $myceliumd_dir" + return 1 + } + log_info "Building in myceliumd subdirectory: $(pwd)" + + # Build with musl target + safe_execute cargo build --release --target "$RUST_TARGET" + + # Find and install binary (from target/x86.../release) + local binary_path="target/${RUST_TARGET}/release/mycelium" + if [[ ! -f "$binary_path" ]]; then + log_error "mycelium binary not found at: ${binary_path}" + return 1 + fi + + local binary_size=$(get_file_size "$binary_path") + log_info "Built mycelium binary (${binary_size}) at: ${binary_path}" } # Install function for rfs (pre-built binary) function install_rfs() { - local name="$1" - local component_dir="$2" - - section_header "Installing rfs binary" - - log_info "Installing rfs from: ${component_dir}" - - # Find the rfs binary - local binary_path="${component_dir}/rfs" - if [[ ! -f "$binary_path" ]]; then - log_error "rfs binary not found at: ${binary_path}" - return 1 - fi - - # Make executable - safe_execute chmod +x "$binary_path" - - local binary_size=$(get_file_size "$binary_path") - log_info "Prepared rfs binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Installing rfs binary" + + log_info "Installing rfs from: ${component_dir}" + + # Find the rfs binary + local binary_path="${component_dir}/rfs" + if [[ ! -f "$binary_path" ]]; then + log_error "rfs binary not found at: ${binary_path}" + return 1 + fi + + # Make executable + safe_execute chmod +x "$binary_path" + + local binary_size=$(get_file_size "$binary_path") + log_info "Prepared rfs binary (${binary_size}) at: ${binary_path}" } # Install function for corex (pre-built binary) function install_corex() { - local name="$1" - local component_dir="$2" - - section_header "Installing corex binary" - - log_info "Installing corex from: ${component_dir}" - - # Find the corex binary (may have been renamed) - local binary_path - if [[ -f "${component_dir}/corex" ]]; then - binary_path="${component_dir}/corex" - elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then - binary_path="${component_dir}/corex-2.1.4-amd64-linux-static" - else - log_error "corex binary not found in: ${component_dir}" - return 1 - fi - - # Make executable - safe_execute chmod +x "$binary_path" - - local binary_size=$(get_file_size "$binary_path") - log_info "Prepared corex binary (${binary_size}) at: ${binary_path}" + local name="$1" + local component_dir="$2" + + section_header "Installing corex binary" + + log_info "Installing corex from: ${component_dir}" + + # Find the corex binary (may have been renamed) + local binary_path + if [[ -f "${component_dir}/corex" ]]; then + binary_path="${component_dir}/corex" + elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then + binary_path="${component_dir}/corex-2.1.4-amd64-linux-static" + else + log_error "corex binary not found in: ${component_dir}" + return 1 + fi + + # Make executable + safe_execute chmod +x "$binary_path" + + local binary_size=$(get_file_size "$binary_path") + log_info "Prepared corex binary (${binary_size}) at: ${binary_path}" } # Verify all components are built (not copied to initramfs yet) function components_verify_installation() { - local components_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}" - - section_header "Verifying Component Build" - - local ok_count=0 - local missing_count=0 + local components_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}" - # zinit - local zinit_bin="${components_dir}/zinit/target/x86_64-unknown-linux-musl/release/zinit" - if [[ -x "$zinit_bin" ]]; then - log_info "✓ zinit ($(get_file_size "$zinit_bin")) at: ${zinit_bin#${components_dir}/}" - ((ok_count++)) - else - log_error "✗ zinit missing: ${zinit_bin#${components_dir}/}" - ((missing_count++)) - fi + section_header "Verifying Component Build" - # rfs: accept both built and prebuilt locations - local rfs_built="${components_dir}/rfs/target/x86_64-unknown-linux-musl/release/rfs" - local rfs_release="${components_dir}/rfs/rfs" - if [[ -x "$rfs_built" ]]; then - log_info "✓ rfs (built) ($(get_file_size "$rfs_built")) at: ${rfs_built#${components_dir}/}" - ((ok_count++)) - elif [[ -x "$rfs_release" ]]; then - log_info "✓ rfs (release) ($(get_file_size "$rfs_release")) at: ${rfs_release#${components_dir}/}" - ((ok_count++)) - else - log_error "✗ rfs missing: checked rfs/target/.../rfs and rfs/rfs" - ((missing_count++)) - fi + local ok_count=0 + local missing_count=0 - # mycelium - local mycelium_bin="${components_dir}/mycelium/myceliumd/target/x86_64-unknown-linux-musl/release/mycelium" - if [[ -x "$mycelium_bin" ]]; then - log_info "✓ mycelium ($(get_file_size "$mycelium_bin")) at: ${mycelium_bin#${components_dir}/}" - ((ok_count++)) - else - log_error "✗ mycelium missing: ${mycelium_bin#${components_dir}/}" - ((missing_count++)) - fi + # zinit + local zinit_bin="${components_dir}/zinit/target/x86_64-unknown-linux-musl/release/zinit" + if [[ -x "$zinit_bin" ]]; then + log_info "✓ zinit ($(get_file_size "$zinit_bin")) at: ${zinit_bin#${components_dir}/}" + ((ok_count++)) + else + log_error "✗ zinit missing: ${zinit_bin#${components_dir}/}" + ((missing_count++)) + fi - # zosstorage - local zosstorage_bin="${components_dir}/zosstorage/target/x86_64-unknown-linux-musl/release/zosstorage" - if [[ -x "$zosstorage_bin" ]]; then - log_info "✓ zosstorage ($(get_file_size "$zosstorage_bin")) at: ${zosstorage_bin#${components_dir}/}" - ((ok_count++)) - else - log_error "✗ zosstorage missing: ${zosstorage_bin#${components_dir}/}" - ((missing_count++)) - fi - - # corex - local corex_bin="${components_dir}/corex/corex" - if [[ -x "$corex_bin" ]]; then - log_info "✓ corex ($(get_file_size "$corex_bin")) at: ${corex_bin#${components_dir}/}" - ((ok_count++)) - else - log_error "✗ corex missing: ${corex_bin#${components_dir}/}" - ((missing_count++)) - fi - - if [[ $missing_count -eq 0 ]]; then - log_info "All components built/installed successfully" - return 0 - else - log_error "${missing_count} components missing or failed to build" - return 1 - fi + # rfs: accept both built and prebuilt locations + local rfs_built="${components_dir}/rfs/target/x86_64-unknown-linux-musl/release/rfs" + local rfs_release="${components_dir}/rfs/rfs" + if [[ -x "$rfs_built" ]]; then + log_info "✓ rfs (built) ($(get_file_size "$rfs_built")) at: ${rfs_built#${components_dir}/}" + ((ok_count++)) + elif [[ -x "$rfs_release" ]]; then + log_info "✓ rfs (release) ($(get_file_size "$rfs_release")) at: ${rfs_release#${components_dir}/}" + ((ok_count++)) + else + log_error "✗ rfs missing: checked rfs/target/.../rfs and rfs/rfs" + ((missing_count++)) + fi + + # mycelium + local mycelium_bin="${components_dir}/mycelium/myceliumd/target/x86_64-unknown-linux-musl/release/mycelium" + if [[ -x "$mycelium_bin" ]]; then + log_info "✓ mycelium ($(get_file_size "$mycelium_bin")) at: ${mycelium_bin#${components_dir}/}" + ((ok_count++)) + else + log_error "✗ mycelium missing: ${mycelium_bin#${components_dir}/}" + ((missing_count++)) + fi + + # zosstorage + local zosstorage_bin="${components_dir}/zosstorage/target/x86_64-unknown-linux-musl/release/zosstorage" + if [[ -x "$zosstorage_bin" ]]; then + log_info "✓ zosstorage ($(get_file_size "$zosstorage_bin")) at: ${zosstorage_bin#${components_dir}/}" + ((ok_count++)) + else + log_error "✗ zosstorage missing: ${zosstorage_bin#${components_dir}/}" + ((missing_count++)) + fi + + # corex + local corex_bin="${components_dir}/corex/corex" + if [[ -x "$corex_bin" ]]; then + log_info "✓ corex ($(get_file_size "$corex_bin")) at: ${corex_bin#${components_dir}/}" + ((ok_count++)) + else + log_error "✗ corex missing: ${corex_bin#${components_dir}/}" + ((missing_count++)) + fi + + if [[ $missing_count -eq 0 ]]; then + log_info "All components built/installed successfully" + return 0 + else + log_error "${missing_count} components missing or failed to build" + return 1 + fi } # Clean component build artifacts function components_cleanup() { - local components_dir="$1" - local keep_sources="${2:-false}" - - section_header "Cleaning Component Build Artifacts" - - if [[ "$keep_sources" == "true" ]]; then - log_info "Keeping source directories, cleaning build artifacts only" - - # Clean Rust build artifacts - find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true - find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true - - else - log_info "Removing all component directories" - safe_rmdir "$components_dir" - fi - - log_info "Component cleanup complete" + local components_dir="$1" + local keep_sources="${2:-false}" + + section_header "Cleaning Component Build Artifacts" + + if [[ "$keep_sources" == "true" ]]; then + log_info "Keeping source directories, cleaning build artifacts only" + + # Clean Rust build artifacts + find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true + find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true + + else + log_info "Removing all component directories" + safe_rmdir "$components_dir" + fi + + log_info "Component cleanup complete" } # Export functions diff --git a/scripts/lib/kernel.sh b/scripts/lib/kernel.sh index 9d03101..bfdd7d1 100644 --- a/scripts/lib/kernel.sh +++ b/scripts/lib/kernel.sh @@ -223,20 +223,219 @@ function kernel_build_with_initramfs() { output_dir=$(dirname "$output_abs") safe_mkdir "$output_dir" safe_copy "$kernel_image" "$output_abs" - + + # Also copy with versioned filename including kernel version and zinit hash + local full_kernel_version="${FULL_KERNEL_VERSION:-unknown}" + local zinit_hash="unknown" + local zinit_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}/zinit" + + if [[ -d "$zinit_dir/.git" ]]; then + zinit_hash=$(get_git_commit_hash "$zinit_dir") + else + log_warn "zinit git directory not found at ${zinit_dir}, using 'unknown' for hash" + fi + + # Create versioned filename: vmlinuz-{VERSION}-{ZINIT_HASH}.efi + local versioned_name="vmlinuz-${full_kernel_version}-${zinit_hash}.efi" + local versioned_output="${output_dir}/${versioned_name}" + safe_copy "$kernel_image" "$versioned_output" + # Verify final kernel local kernel_size kernel_size=$(get_file_size "$output_abs") + local versioned_size + versioned_size=$(get_file_size "$versioned_output") log_info "Kernel build complete:" log_info " Output file: ${output_abs}" + log_info " Versioned: ${versioned_output}" log_info " Kernel size: ${kernel_size}" - + log_info " Version: ${full_kernel_version}" + log_info " zinit hash: ${zinit_hash}" + # Verify initramfs is embedded if strings "$output_file" | grep -q "initramfs"; then log_info "✓ Initramfs appears to be embedded in kernel" else log_warn "Initramfs embedding verification inconclusive" fi + + # Upload versioned kernel to S3 if enabled + kernel_upload_to_s3 "$versioned_output" "$full_kernel_version" "$zinit_hash" +} + +# Upload versioned kernel to S3 using MinIO client (mcli/mc) +function kernel_upload_to_s3() { + local kernel_file="$1" + local kernel_version="$2" + local zinit_hash="$3" + + section_header "Uploading Kernel to S3" + + # Check if upload is enabled + if [[ "${UPLOAD_KERNEL:-false}" != "true" ]]; then + log_info "UPLOAD_KERNEL not enabled; skipping kernel upload" + return 0 + fi + + # Verify kernel file exists + if [[ ! -f "$kernel_file" ]]; then + log_error "Kernel file not found: ${kernel_file}" + return 1 + fi + + # Load S3 configuration from rfs.conf + local rfs_conf="${PROJECT_ROOT}/config/rfs.conf" + local rfs_conf_example="${PROJECT_ROOT}/config/rfs.conf.example" + + if [[ -f "$rfs_conf" ]]; then + # shellcheck source=/dev/null + source "$rfs_conf" + log_info "Loaded S3 config from: ${rfs_conf}" + elif [[ -f "$rfs_conf_example" ]]; then + # shellcheck source=/dev/null + source "$rfs_conf_example" + log_warn "Using example S3 config: ${rfs_conf_example}" + else + log_error "No S3 config found (config/rfs.conf or config/rfs.conf.example)" + return 1 + fi + + # Validate required S3 variables + for var in S3_ENDPOINT S3_BUCKET S3_PREFIX S3_ACCESS_KEY S3_SECRET_KEY; do + if [[ -z "${!var}" ]]; then + log_error "Missing required S3 variable: ${var}" + return 1 + fi + done + + # Detect MinIO client binary (mcli or mc) + local mcli_bin="" + if command -v mcli >/dev/null 2>&1; then + mcli_bin="mcli" + elif command -v mc >/dev/null 2>&1; then + mcli_bin="mc" + else + log_warn "MinIO Client not found (expected mcli or mc); skipping kernel upload" + return 0 + fi + + log_info "Using MinIO client: ${mcli_bin}" + + # Setup S3 alias + log_info "Configuring S3 alias..." + safe_execute "${mcli_bin}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}" + + # Construct destination path: rfs/{bucket}/{prefix}/kernel/{versioned_filename} + local kernel_filename + kernel_filename=$(basename "$kernel_file") + local kernel_subpath="${KERNEL_SUBPATH:-kernel}" + local mcli_dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${kernel_subpath%/}/${kernel_filename}" + + # Upload kernel + log_info "Uploading: ${kernel_file} -> ${mcli_dst}" + safe_execute "${mcli_bin}" cp "${kernel_file}" "${mcli_dst}" + + log_info "✓ Kernel uploaded successfully" + log_info " Version: ${kernel_version}" + log_info " zinit: ${zinit_hash}" + log_info " S3 path: ${mcli_dst}" + + # Generate and upload kernel index + kernel_generate_index "${mcli_bin}" "${S3_BUCKET}" "${S3_PREFIX}" "${kernel_subpath}" +} + +# Generate kernel index file from S3 listing and upload it +function kernel_generate_index() { + local mcli_bin="$1" + local bucket="$2" + local prefix="$3" + local kernel_subpath="$4" + + section_header "Generating Kernel Index" + + # Construct S3 path for listing + local s3_path="rfs/${bucket}/${prefix%/}/${kernel_subpath%/}/" + + log_info "Listing kernels from: ${s3_path}" + + # List all files in the kernel directory + local ls_output + if ! ls_output=$("${mcli_bin}" ls "${s3_path}" 2>&1); then + log_warn "Failed to list S3 kernel directory, index not generated" + log_debug "mcli ls output: ${ls_output}" + return 0 + fi + + # Parse output and extract kernel filenames matching vmlinuz-* + local kernels=() + while IFS= read -r line; do + # mcli ls output format: [DATE TIME TZ] SIZE FILENAME + # Extract filename (last field) + local filename + filename=$(echo "$line" | awk '{print $NF}') + + # Filter for vmlinuz files (both .efi and without extension) + if [[ "$filename" =~ ^vmlinuz-.* ]]; then + kernels+=("$filename") + fi + done <<< "$ls_output" + + if [[ ${#kernels[@]} -eq 0 ]]; then + log_warn "No kernels found in S3 path: ${s3_path}" + return 0 + fi + + log_info "Found ${#kernels[@]} kernel(s)" + + # Create index files in dist directory + local index_dir="${DIST_DIR:-${PROJECT_ROOT}/dist}" + local text_index="${index_dir}/kernels.txt" + local json_index="${index_dir}/kernels.json" + + # Generate text index (one kernel per line, sorted) + printf "%s\n" "${kernels[@]}" | sort -r > "$text_index" + log_info "Created text index: ${text_index}" + + # Generate JSON index (array of kernel filenames) + { + echo "{" + echo " \"kernels\": [" + local first=true + for kernel in $(printf "%s\n" "${kernels[@]}" | sort -r); do + if [[ "$first" == "true" ]]; then + first=false + else + echo "," + fi + printf " \"%s\"" "$kernel" + done + echo "" + echo " ]," + echo " \"updated\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"," + echo " \"count\": ${#kernels[@]}" + echo "}" + } > "$json_index" + log_info "Created JSON index: ${json_index}" + + # Upload both index files to S3 + log_info "Uploading kernel index files to S3..." + + local text_dst="${s3_path}kernels.txt" + local json_dst="${s3_path}kernels.json" + + if safe_execute "${mcli_bin}" cp "$text_index" "$text_dst"; then + log_info "✓ Uploaded text index: ${text_dst}" + else + log_warn "Failed to upload text index" + fi + + if safe_execute "${mcli_bin}" cp "$json_index" "$json_dst"; then + log_info "✓ Uploaded JSON index: ${json_dst}" + else + log_warn "Failed to upload JSON index" + fi + + log_info "Kernel index generation complete" } # Build and install modules in container for proper dependency resolution