rfs: add pack-tree.sh to pack arbitrary directory to flist using config/rfs.conf; enable --debug on rfs pack for verbose diagnostics

scripts/rfs/pack-tree.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# Pack an arbitrary directory tree into an RFS flist and upload blobs to S3 (Garage)
+# - Uses config from config/rfs.conf (or rfs.conf.example fallback)
+# - Packs the specified directory (default: repository root)
+# - Patches manifest route URL with read-only S3 creds
+# - Optionally patches stores to WEB_ENDPOINT and uploads the .fl via MinIO client
+#
+# Usage:
+#   scripts/rfs/pack-tree.sh [-p PATH] [-n MANIFEST_BASENAME] [--web-endpoint URL] [--keep-s3-fallback] [--no-upload]
+# Examples:
+#   scripts/rfs/pack-tree.sh
+#   scripts/rfs/pack-tree.sh -p ./components/zinit -n zinit-src
+#   WEB_ENDPOINT=https://hub.grid.tf/zos/zosbuilder/store scripts/rfs/pack-tree.sh -p dist --keep-s3-fallback
+
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${HERE}/common.sh"
+
+section() { echo -e "\n==== $* ====\n"; }
+
+print_help() {
+  cat <<EOF
+Pack a directory to an RFS flist and upload blobs to S3, using config/rfs.conf.
+
+Options:
+  -p, --path PATH        Source directory to pack (default: repository root)
+  -n, --name NAME        Manifest base name (without .fl). Default: tree-<basename(PATH)>-<YYYYMMDDHHMMSS>
+      --web-endpoint URL Override WEB_ENDPOINT for stores patching (default from rfs.conf)
+      --keep-s3-fallback Keep s3:// store rows alongside HTTPS store in manifest
+      --no-upload        Do not upload the .fl manifest via MinIO client even if enabled in config
+  -h, --help             Show this help
+
+Environment/config:
+  Reads S3 and route settings from config/rfs.conf (or rfs.conf.example).
+  Honors: WEB_ENDPOINT, KEEP_S3_FALLBACK, UPLOAD_MANIFESTS.
+EOF
+}
+
+# Defaults
+SRC_PATH=""
+MANIFEST_NAME=""
+ARG_WEB_ENDPOINT=""
+ARG_KEEP_S3="false"
+ARG_NO_UPLOAD="false"
+
+# Parse args
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -p|--path)
+      SRC_PATH="$2"; shift 2;;
+    -n|--name)
+      MANIFEST_NAME="$2"; shift 2;;
+    --web-endpoint)
+      ARG_WEB_ENDPOINT="$2"; shift 2;;
+    --keep-s3-fallback)
+      ARG_KEEP_S3="true"; shift 1;;
+    --no-upload)
+      ARG_NO_UPLOAD="true"; shift 1;;
+    -h|--help)
+      print_help; exit 0;;
+    *)
+      echo "Unknown argument: $1" >&2; print_help; exit 1;;
+  esac
+done
+
+# Determine PROJECT_ROOT and defaults
+PROJECT_ROOT="${PROJECT_ROOT:-$(rfs_common_project_root)}"
+if [[ -z "${SRC_PATH}" ]]; then
+  SRC_PATH="${PROJECT_ROOT}"
+fi
+
+# Normalize SRC_PATH to absolute
+if [[ "${SRC_PATH}" != /* ]]; then
+  SRC_PATH="$(cd "${SRC_PATH}" && pwd)"
+fi
+if [[ ! -d "${SRC_PATH}" ]]; then
+  log_error "Source path is not a directory: ${SRC_PATH}"
+  exit 1
+fi
+
+# Compute default manifest name if not given
+if [[ -z "${MANIFEST_NAME}" ]]; then
+  base="$(basename "${SRC_PATH}")"
+  ts="$(date -u +%Y%m%d%H%M%S)"
+  MANIFEST_NAME="tree-${base}-${ts}"
+fi
+MANIFEST_FILE="${MANIFEST_NAME%.fl}.fl"
+
+section "Loading RFS and kernel configuration"
+# Kernel version for consistent logs (not strictly required for generic pack)
+rfs_common_load_build_kernel_version
+rfs_common_load_rfs_s3_config
+# Allow CLI override for WEB_ENDPOINT and KEEP_S3_FALLBACK
+if [[ -n "${ARG_WEB_ENDPOINT}" ]]; then
+  WEB_ENDPOINT="${ARG_WEB_ENDPOINT}"
+fi
+if [[ "${ARG_KEEP_S3}" == "true" ]]; then
+  KEEP_S3_FALLBACK="true"
+fi
+rfs_common_build_s3_store_uri
+rfs_common_locate_rfs
+
+# Prepare output
+MANIFEST_PATH="$(rfs_common_prepare_output "${MANIFEST_FILE}")"
+
+section "Packing directory to flist"
+log_info "Source path: ${SRC_PATH}"
+log_info "Manifest: ${MANIFEST_PATH}"
+log_info "Store: ${RFS_S3_STORE_URI}"
+safe_execute "${RFS_BIN}" pack --debug -m "${MANIFEST_PATH}" -s "${RFS_S3_STORE_URI}" "${SRC_PATH}"
+
+section "Patching route.url in manifest to S3 read-only endpoint"
+rfs_common_build_route_url
+rfs_common_patch_flist_route_url "${MANIFEST_PATH}"
+
+if [[ -n "${WEB_ENDPOINT:-}" ]]; then
+  section "Patching stores to HTTPS web endpoint"
+  log_info "WEB_ENDPOINT=${WEB_ENDPOINT}"
+  log_info "KEEP_S3_FALLBACK=${KEEP_S3_FALLBACK:-false}"
+  rfs_common_patch_flist_stores "${MANIFEST_PATH}" "${WEB_ENDPOINT}" "${KEEP_S3_FALLBACK:-false}"
+else
+  log_warn "WEB_ENDPOINT not set; leaving manifest stores as-is (s3:// only)"
+fi
+
+# Optional manifest upload via MinIO client
+UPLOAD="${UPLOAD_MANIFESTS:-false}"
+if [[ "${ARG_NO_UPLOAD}" == "true" ]]; then
+  UPLOAD="false"
+fi
+
+if [[ "${UPLOAD}" == "true" ]]; then
+  section "Uploading manifest .fl via MinIO client (mcli/mc)"
+  if command -v mcli >/dev/null 2>&1; then
+    MCLI_BIN="mcli"
+  elif command -v mc >/dev/null 2>&1; then
+    MCLI_BIN="mc"
+  else
+    log_warn "MinIO Client not found (expected mcli or mc); skipping manifest upload"
+    MCLI_BIN=""
+  fi
+  if [[ -n "${MCLI_BIN}" ]]; then
+    local_subpath="${MANIFESTS_SUBPATH:-manifests}"
+    safe_execute "${MCLI_BIN}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
+    dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${local_subpath%/}/${MANIFEST_FILE}"
+    log_info "${MCLI_BIN} cp ${MANIFEST_PATH} ${dst}"
+    safe_execute "${MCLI_BIN}" cp "${MANIFEST_PATH}" "${dst}"
+  fi
+else
+  log_info "UPLOAD_MANIFESTS=false; skipping manifest upload"
+fi
+
+section "Done"
+log_info "Packed: ${MANIFEST_PATH}"