From 6193d241ea35ed7caa59e12f112d5bd018cc785f Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 1 Oct 2025 17:47:51 +0200 Subject: [PATCH] components: reuse existing git tree in components_download_git; config: update packages.list --- config/packages.list | 6 +- config/rfs.conf.example | 10 ++-- config/zinit/init/network.sh | 2 +- config/zinit/mycelium.yaml | 3 +- config/zinit/network.yaml | 3 +- scripts/lib/components.sh | 112 ++++++++++++++++++++++++++++------- scripts/lib/initramfs.sh | 3 +- 7 files changed, 104 insertions(+), 35 deletions(-) diff --git a/config/packages.list b/config/packages.list index 8f456e9..3b520fb 100644 --- a/config/packages.list +++ b/config/packages.list @@ -17,6 +17,8 @@ eudev-netifnames kmod fuse3 pciutils +efitools +efibootmgr # Console/terminal management util-linux @@ -47,7 +49,3 @@ haveged openssh-server zellij -# Essential debugging and monitoring tools included -# NO development tools, NO curl/wget, NO python, NO redis -# NO massive linux-firmware package -# Other tools will be loaded with RFS after network connectivity \ No newline at end of file diff --git a/config/rfs.conf.example b/config/rfs.conf.example index dd81d55..861a068 100644 --- a/config/rfs.conf.example +++ b/config/rfs.conf.example @@ -8,12 +8,12 @@ S3_ENDPOINT="https://hub.grid.tf" # AWS region string expected by the S3-compatible API -S3_REGION="us-east-1" +S3_REGION="garage" # Bucket and key prefix used for RFS store (content-addressed blobs) # The RFS store path will be: s3://...// S3_BUCKET="zos" -S3_PREFIX="zosbuilder/store" +S3_PREFIX="zos/store" # Access credentials (required by rfs pack to push blobs) S3_ACCESS_KEY="REPLACE_ME" @@ -36,10 +36,10 @@ MANIFESTS_SUBPATH="manifests" # Behavior flags (can be overridden by CLI flags or env) # Whether to keep s3:// store as a fallback entry in the .fl after adding WEB_ENDPOINT -KEEP_S3_FALLBACK="false" +KEEP_S3_FALLBACK="true" # Whether to attempt uploading .fl manifests to S3 (requires MinIO Client: mc) -UPLOAD_MANIFESTS="false" +UPLOAD_MANIFESTS="true" # Read-only credentials for route URL in manifest (optional; defaults to write keys above) # These will be embedded into the flist 'route.url' so runtime mounts can read directly from Garage. @@ -53,5 +53,5 @@ READ_SECRET_KEY="REPLACE_ME_READ" # - ROUTE_PATH: path to the blob route (default: /blobs) # - ROUTE_REGION: region string for Garage (default: garage) ROUTE_ENDPOINT="https://hub.grid.tf" -ROUTE_PATH="/blobs" +ROUTE_PATH="/zos/store" ROUTE_REGION="garage" \ No newline at end of file diff --git a/config/zinit/init/network.sh b/config/zinit/init/network.sh index 8644c43..adec0a2 100755 --- a/config/zinit/init/network.sh +++ b/config/zinit/init/network.sh @@ -5,4 +5,4 @@ if ! getent group dhcpcd >/dev/null 2>&1; then addgroup -S dhcpcd 2>/dev/null || if ! getent passwd dhcpcd >/dev/null 2>&1; then adduser -S -D -s /sbin/nologin -G dhcpcd dhcpcd 2>/dev/null || true; fi # Exec dhcpcd (will run as root if it cannot drop to dhcpcd user) interfaces=$(ip -br l | awk '!/lo/&&!/my0/{print $1}') -exec dhcpcd -B $interfaces \ No newline at end of file +exec dhcpcd -p -B $interfaces \ No newline at end of file diff --git a/config/zinit/mycelium.yaml b/config/zinit/mycelium.yaml index e61252a..595b8c0 100644 --- a/config/zinit/mycelium.yaml +++ b/config/zinit/mycelium.yaml @@ -3,4 +3,5 @@ exec: /usr/bin/mycelium --key-file /tmp/mycelium_priv_key.bin tcp://185.69.166.7:9651 tcp://185.69.166.8:9651 tcp://65.21.231.58:9651 tcp://65.109.18.113:9651 tcp://209.159.146.190:9651 tcp://5.78.122.16:9651 tcp://5.223.43.251:9651 tcp://142.93.217.194:9651 after: - - network \ No newline at end of file + - network + - udev-rfs \ No newline at end of file diff --git a/config/zinit/network.yaml b/config/zinit/network.yaml index b2d43eb..ab294c7 100644 --- a/config/zinit/network.yaml +++ b/config/zinit/network.yaml @@ -1,5 +1,6 @@ -exec: sh /etc/zinit/init/network.sh eth0 +exec: sh /etc/zinit/init/network.sh after: - depmod - udevd - udev-trigger +test: ping www.google.com diff --git a/scripts/lib/components.sh b/scripts/lib/components.sh index c93a38c..1caf3e7 100644 --- a/scripts/lib/components.sh +++ b/scripts/lib/components.sh @@ -46,7 +46,7 @@ function components_parse_sources_conf() { # Component 2: mycelium component_count=$((component_count + 1)) log_info "Processing component ${component_count}: mycelium (git)" - components_download_git "mycelium" "https://github.com/threefoldtech/mycelium" "v0.6.1" "$components_dir" + components_download_git "mycelium" "https://github.com/threefoldtech/mycelium" "v0.6.2" "$components_dir" components_build_component "mycelium" "build_mycelium" "$components_dir" # Component 3: rfs (pre-built release) @@ -68,36 +68,106 @@ function components_parse_sources_conf() { fi } -# Download Git repository +# Download Git repository (reuse tree; only reclone if invalid or version not reachable) function components_download_git() { local name="$1" local url="$2" local version="$3" local components_dir="$4" - + section_header "Downloading Git Component: ${name}" - + local target_dir="${components_dir}/${name}" - + log_info "Repository: ${url}" - log_info "Version/Branch: ${version}" + log_info "Version/Branch/Tag: ${version}" log_info "Target directory: ${target_dir}" - - # Always do fresh clone to avoid git state issues - if [[ -d "$target_dir" ]]; then - log_info "Removing existing ${name} directory for fresh clone" - safe_execute rm -rf "$target_dir" + + # Ensure parent exists + safe_mkdir "$components_dir" + + # Decide whether we can reuse the existing working tree + local need_fresh_clone="0" + if [[ -d "$target_dir/.git" ]]; then + if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + log_warn "Existing ${name} directory is not a valid git repo; will reclone" + need_fresh_clone="1" + fi + elif [[ -d "$target_dir" ]]; then + log_warn "Existing ${name} directory without .git; will reclone" + need_fresh_clone="1" fi - - log_info "Cloning ${name} from ${url}" - safe_execute git clone --depth 1 --branch "$version" "$url" "$target_dir" - - # Verify checkout - safe_execute cd "$target_dir" - local current_ref=$(git rev-parse HEAD) - log_info "Current commit: ${current_ref}" - - log_info "Git component download complete: ${name}" + + if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then + log_info "Cloning ${name} (fresh) from ${url}" + safe_execute git clone "$url" "$target_dir" + fi + + # Ensure origin URL is correct (do not delete the tree if URL changed) + local current_url + current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "") + if [[ -n "$current_url" && "$current_url" != "$url" ]]; then + log_info "Updating origin URL: ${current_url} -> ${url}" + safe_execute git -C "$target_dir" remote set-url origin "$url" + elif [[ -z "$current_url" ]]; then + log_info "Setting origin URL to ${url}" + safe_execute git -C "$target_dir" remote add origin "$url" || true + fi + + # Fetch updates and tags + safe_execute git -C "$target_dir" fetch --tags --prune origin + + # Resolve desired commit for the requested version/branch/tag + local desired_rev="" + if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}") + elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}") + else + log_warn "Version '${version}' not directly resolvable; fetching explicitly" + if git -C "$target_dir" fetch origin "${version}" --depth 1; then + desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD) + fi + fi + + # Fallback: shallow clone at the requested ref if we still can't resolve + if [[ -z "$desired_rev" ]]; then + log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref." + safe_execute rm -rf "${target_dir}.tmp" + if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then + safe_execute rm -rf "$target_dir" + safe_execute mv "${target_dir}.tmp" "$target_dir" + desired_rev=$(git -C "$target_dir" rev-parse HEAD) + else + log_error "Failed to clone ${url} at '${version}'" + return 1 + fi + fi + + local current_rev + current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "") + log_info "Current commit: ${current_rev:-}" + log_info "Desired commit: ${desired_rev}" + + if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then + log_info "Repository already at requested version; reusing working tree" + else + log_info "Checking out requested version" + # Prefer named refs when available; otherwise detach to exact commit + if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then + safe_execute git -C "$target_dir" checkout -f "${version}" + elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then + safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}" + elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then + safe_execute git -C "$target_dir" checkout -f "tags/${version}" + else + safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}" + fi + # Initialize submodules if present (non-fatal) + safe_execute git -C "$target_dir" submodule update --init --recursive || true + fi + + log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)" } # Download release binary/archive diff --git a/scripts/lib/initramfs.sh b/scripts/lib/initramfs.sh index 533b2d0..d765477 100644 --- a/scripts/lib/initramfs.sh +++ b/scripts/lib/initramfs.sh @@ -617,7 +617,6 @@ EOF log_info "Branding enabled: updating /etc/issue to Zero-OS branding" cat > "${initramfs_dir}/etc/issue" << 'EOF' Zero-OS \r \m -Built on \l EOF else @@ -780,7 +779,7 @@ function initramfs_create_cpio() { case "$compression" in "xz") log_info "Creating XZ compressed CPIO archive" - safe_execute find . -print0 | cpio -o -H newc -0 | xz -${XZ_COMPRESSION_LEVEL} --check=crc32 > "$output_file_abs" + safe_execute find . -print0 | cpio -o -H newc -0 | xz -T 8 -${XZ_COMPRESSION_LEVEL} --check=crc32 > "$output_file_abs" ;; "gzip"|"gz") log_info "Creating gzip compressed CPIO archive"