zosbuilder/scripts/rfs/pack-tree.sh

#!/bin/bash
# Pack an arbitrary directory tree into an RFS flist and upload blobs to S3 (Garage)
# - Uses config from config/rfs.conf (or rfs.conf.example fallback)
# - Packs the specified directory (default: repository root)
# - Patches manifest route URL with read-only S3 creds
# - Optionally patches stores to WEB_ENDPOINT and uploads the .fl via MinIO client
#
# Usage:
#   scripts/rfs/pack-tree.sh [-p PATH] [-n MANIFEST_BASENAME] [--web-endpoint URL] [--keep-s3-fallback] [--no-upload]
# Examples:
#   scripts/rfs/pack-tree.sh
#   scripts/rfs/pack-tree.sh -p ./components/zinit -n zinit-src
#   WEB_ENDPOINT=https://hub.grid.tf/zos/zosbuilder/store scripts/rfs/pack-tree.sh -p dist --keep-s3-fallback

set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${HERE}/common.sh"

section() { echo -e "\n==== $* ====\n"; }

print_help() {
  cat <<EOF
Pack a directory to an RFS flist and upload blobs to S3, using config/rfs.conf.

Options:
  -p, --path PATH        Source directory to pack (default: repository root)
  -n, --name NAME        Manifest base name (without .fl). Default: tree-<basename(PATH)>-<YYYYMMDDHHMMSS>
      --web-endpoint URL Override WEB_ENDPOINT for stores patching (default from rfs.conf)
      --keep-s3-fallback Keep s3:// store rows alongside HTTPS store in manifest
      --no-upload        Do not upload the .fl manifest via MinIO client even if enabled in config
  -h, --help             Show this help

Environment/config:
  Reads S3 and route settings from config/rfs.conf (or rfs.conf.example).
  Honors: WEB_ENDPOINT, KEEP_S3_FALLBACK, UPLOAD_MANIFESTS.
EOF
}

# Defaults
SRC_PATH=""
MANIFEST_NAME=""
ARG_WEB_ENDPOINT=""
ARG_KEEP_S3="false"
ARG_NO_UPLOAD="false"

# Parse args
while [[ $# -gt 0 ]]; do
  case "$1" in
    -p|--path)
      SRC_PATH="$2"; shift 2;;
    -n|--name)
      MANIFEST_NAME="$2"; shift 2;;
    --web-endpoint)
      ARG_WEB_ENDPOINT="$2"; shift 2;;
    --keep-s3-fallback)
      ARG_KEEP_S3="true"; shift 1;;
    --no-upload)
      ARG_NO_UPLOAD="true"; shift 1;;
    -h|--help)
      print_help; exit 0;;
    *)
      echo "Unknown argument: $1" >&2; print_help; exit 1;;
  esac
done

# Determine PROJECT_ROOT and defaults
PROJECT_ROOT="${PROJECT_ROOT:-$(rfs_common_project_root)}"
if [[ -z "${SRC_PATH}" ]]; then
  SRC_PATH="${PROJECT_ROOT}"
fi

# Normalize SRC_PATH to absolute
if [[ "${SRC_PATH}" != /* ]]; then
  SRC_PATH="$(cd "${SRC_PATH}" && pwd)"
fi
if [[ ! -d "${SRC_PATH}" ]]; then
  log_error "Source path is not a directory: ${SRC_PATH}"
  exit 1
fi

# Compute default manifest name if not given
if [[ -z "${MANIFEST_NAME}" ]]; then
  base="$(basename "${SRC_PATH}")"
  ts="$(date -u +%Y%m%d%H%M%S)"
  MANIFEST_NAME="tree-${base}-${ts}"
fi
MANIFEST_FILE="${MANIFEST_NAME%.fl}.fl"

section "Loading RFS and kernel configuration"
# Kernel version for consistent logs (not strictly required for generic pack)
rfs_common_load_build_kernel_version
rfs_common_load_rfs_s3_config
# Allow CLI override for WEB_ENDPOINT and KEEP_S3_FALLBACK
if [[ -n "${ARG_WEB_ENDPOINT}" ]]; then
  WEB_ENDPOINT="${ARG_WEB_ENDPOINT}"
fi
if [[ "${ARG_KEEP_S3}" == "true" ]]; then
  KEEP_S3_FALLBACK="true"
fi
rfs_common_build_s3_store_uri
rfs_common_locate_rfs

# Prepare output
MANIFEST_PATH="$(rfs_common_prepare_output "${MANIFEST_FILE}")"

section "Packing directory to flist"
log_info "Source path: ${SRC_PATH}"
log_info "Manifest: ${MANIFEST_PATH}"
log_info "Store: ${RFS_S3_STORE_URI}"
safe_execute "${RFS_BIN}" --debug pack  -m "${MANIFEST_PATH}" -s "${RFS_S3_STORE_URI}" "${SRC_PATH}"

section "Patching route.url in manifest to S3 read-only endpoint"
rfs_common_build_route_url
rfs_common_patch_flist_route_url "${MANIFEST_PATH}"

if [[ -n "${WEB_ENDPOINT:-}" ]]; then
  section "Patching stores to HTTPS web endpoint"
  log_info "WEB_ENDPOINT=${WEB_ENDPOINT}"
  log_info "KEEP_S3_FALLBACK=${KEEP_S3_FALLBACK:-false}"
  rfs_common_patch_flist_stores "${MANIFEST_PATH}" "${WEB_ENDPOINT}" "${KEEP_S3_FALLBACK:-false}"
else
  log_warn "WEB_ENDPOINT not set; leaving manifest stores as-is (s3:// only)"
fi

# Optional manifest upload via MinIO client
UPLOAD="${UPLOAD_MANIFESTS:-false}"
if [[ "${ARG_NO_UPLOAD}" == "true" ]]; then
  UPLOAD="false"
fi

if [[ "${UPLOAD}" == "true" ]]; then
  section "Uploading manifest .fl via MinIO client (mcli/mc)"
  if command -v mcli >/dev/null 2>&1; then
    MCLI_BIN="mcli"
  elif command -v mc >/dev/null 2>&1; then
    MCLI_BIN="mc"
  else
    log_warn "MinIO Client not found (expected mcli or mc); skipping manifest upload"
    MCLI_BIN=""
  fi
  if [[ -n "${MCLI_BIN}" ]]; then
    local_subpath="${MANIFESTS_SUBPATH:-manifests}"
    safe_execute "${MCLI_BIN}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
    dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${local_subpath%/}/${MANIFEST_FILE}"
    log_info "${MCLI_BIN} cp ${MANIFEST_PATH} ${dst}"
    safe_execute "${MCLI_BIN}" cp "${MANIFEST_PATH}" "${dst}"
  fi
else
  log_info "UPLOAD_MANIFESTS=false; skipping manifest upload"
fi

section "Done"
log_info "Packed: ${MANIFEST_PATH}"