forked from tfgrid/zosbuilder
rfs: add pack-tree.sh to pack arbitrary directory to flist using config/rfs.conf; enable --debug on rfs pack for verbose diagnostics
This commit is contained in:
155
scripts/rfs/pack-tree.sh
Executable file
155
scripts/rfs/pack-tree.sh
Executable file
@@ -0,0 +1,155 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Pack an arbitrary directory tree into an RFS flist and upload blobs to S3 (Garage)
|
||||||
|
# - Uses config from config/rfs.conf (or rfs.conf.example fallback)
|
||||||
|
# - Packs the specified directory (default: repository root)
|
||||||
|
# - Patches manifest route URL with read-only S3 creds
|
||||||
|
# - Optionally patches stores to WEB_ENDPOINT and uploads the .fl via MinIO client
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# scripts/rfs/pack-tree.sh [-p PATH] [-n MANIFEST_BASENAME] [--web-endpoint URL] [--keep-s3-fallback] [--no-upload]
|
||||||
|
# Examples:
|
||||||
|
# scripts/rfs/pack-tree.sh
|
||||||
|
# scripts/rfs/pack-tree.sh -p ./components/zinit -n zinit-src
|
||||||
|
# WEB_ENDPOINT=https://hub.grid.tf/zos/zosbuilder/store scripts/rfs/pack-tree.sh -p dist --keep-s3-fallback
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${HERE}/common.sh"
|
||||||
|
|
||||||
|
section() { echo -e "\n==== $* ====\n"; }
|
||||||
|
|
||||||
|
print_help() {
|
||||||
|
cat <<EOF
|
||||||
|
Pack a directory to an RFS flist and upload blobs to S3, using config/rfs.conf.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-p, --path PATH Source directory to pack (default: repository root)
|
||||||
|
-n, --name NAME Manifest base name (without .fl). Default: tree-<basename(PATH)>-<YYYYMMDDHHMMSS>
|
||||||
|
--web-endpoint URL Override WEB_ENDPOINT for stores patching (default from rfs.conf)
|
||||||
|
--keep-s3-fallback Keep s3:// store rows alongside HTTPS store in manifest
|
||||||
|
--no-upload Do not upload the .fl manifest via MinIO client even if enabled in config
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Environment/config:
|
||||||
|
Reads S3 and route settings from config/rfs.conf (or rfs.conf.example).
|
||||||
|
Honors: WEB_ENDPOINT, KEEP_S3_FALLBACK, UPLOAD_MANIFESTS.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
# Defaults
|
||||||
|
SRC_PATH=""
|
||||||
|
MANIFEST_NAME=""
|
||||||
|
ARG_WEB_ENDPOINT=""
|
||||||
|
ARG_KEEP_S3="false"
|
||||||
|
ARG_NO_UPLOAD="false"
|
||||||
|
|
||||||
|
# Parse args
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
-p|--path)
|
||||||
|
SRC_PATH="$2"; shift 2;;
|
||||||
|
-n|--name)
|
||||||
|
MANIFEST_NAME="$2"; shift 2;;
|
||||||
|
--web-endpoint)
|
||||||
|
ARG_WEB_ENDPOINT="$2"; shift 2;;
|
||||||
|
--keep-s3-fallback)
|
||||||
|
ARG_KEEP_S3="true"; shift 1;;
|
||||||
|
--no-upload)
|
||||||
|
ARG_NO_UPLOAD="true"; shift 1;;
|
||||||
|
-h|--help)
|
||||||
|
print_help; exit 0;;
|
||||||
|
*)
|
||||||
|
echo "Unknown argument: $1" >&2; print_help; exit 1;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Determine PROJECT_ROOT and defaults
|
||||||
|
PROJECT_ROOT="${PROJECT_ROOT:-$(rfs_common_project_root)}"
|
||||||
|
if [[ -z "${SRC_PATH}" ]]; then
|
||||||
|
SRC_PATH="${PROJECT_ROOT}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Normalize SRC_PATH to absolute
|
||||||
|
if [[ "${SRC_PATH}" != /* ]]; then
|
||||||
|
SRC_PATH="$(cd "${SRC_PATH}" && pwd)"
|
||||||
|
fi
|
||||||
|
if [[ ! -d "${SRC_PATH}" ]]; then
|
||||||
|
log_error "Source path is not a directory: ${SRC_PATH}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Compute default manifest name if not given
|
||||||
|
if [[ -z "${MANIFEST_NAME}" ]]; then
|
||||||
|
base="$(basename "${SRC_PATH}")"
|
||||||
|
ts="$(date -u +%Y%m%d%H%M%S)"
|
||||||
|
MANIFEST_NAME="tree-${base}-${ts}"
|
||||||
|
fi
|
||||||
|
MANIFEST_FILE="${MANIFEST_NAME%.fl}.fl"
|
||||||
|
|
||||||
|
section "Loading RFS and kernel configuration"
|
||||||
|
# Kernel version for consistent logs (not strictly required for generic pack)
|
||||||
|
rfs_common_load_build_kernel_version
|
||||||
|
rfs_common_load_rfs_s3_config
|
||||||
|
# Allow CLI override for WEB_ENDPOINT and KEEP_S3_FALLBACK
|
||||||
|
if [[ -n "${ARG_WEB_ENDPOINT}" ]]; then
|
||||||
|
WEB_ENDPOINT="${ARG_WEB_ENDPOINT}"
|
||||||
|
fi
|
||||||
|
if [[ "${ARG_KEEP_S3}" == "true" ]]; then
|
||||||
|
KEEP_S3_FALLBACK="true"
|
||||||
|
fi
|
||||||
|
rfs_common_build_s3_store_uri
|
||||||
|
rfs_common_locate_rfs
|
||||||
|
|
||||||
|
# Prepare output
|
||||||
|
MANIFEST_PATH="$(rfs_common_prepare_output "${MANIFEST_FILE}")"
|
||||||
|
|
||||||
|
section "Packing directory to flist"
|
||||||
|
log_info "Source path: ${SRC_PATH}"
|
||||||
|
log_info "Manifest: ${MANIFEST_PATH}"
|
||||||
|
log_info "Store: ${RFS_S3_STORE_URI}"
|
||||||
|
safe_execute "${RFS_BIN}" pack --debug -m "${MANIFEST_PATH}" -s "${RFS_S3_STORE_URI}" "${SRC_PATH}"
|
||||||
|
|
||||||
|
section "Patching route.url in manifest to S3 read-only endpoint"
|
||||||
|
rfs_common_build_route_url
|
||||||
|
rfs_common_patch_flist_route_url "${MANIFEST_PATH}"
|
||||||
|
|
||||||
|
if [[ -n "${WEB_ENDPOINT:-}" ]]; then
|
||||||
|
section "Patching stores to HTTPS web endpoint"
|
||||||
|
log_info "WEB_ENDPOINT=${WEB_ENDPOINT}"
|
||||||
|
log_info "KEEP_S3_FALLBACK=${KEEP_S3_FALLBACK:-false}"
|
||||||
|
rfs_common_patch_flist_stores "${MANIFEST_PATH}" "${WEB_ENDPOINT}" "${KEEP_S3_FALLBACK:-false}"
|
||||||
|
else
|
||||||
|
log_warn "WEB_ENDPOINT not set; leaving manifest stores as-is (s3:// only)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Optional manifest upload via MinIO client
|
||||||
|
UPLOAD="${UPLOAD_MANIFESTS:-false}"
|
||||||
|
if [[ "${ARG_NO_UPLOAD}" == "true" ]]; then
|
||||||
|
UPLOAD="false"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${UPLOAD}" == "true" ]]; then
|
||||||
|
section "Uploading manifest .fl via MinIO client (mcli/mc)"
|
||||||
|
if command -v mcli >/dev/null 2>&1; then
|
||||||
|
MCLI_BIN="mcli"
|
||||||
|
elif command -v mc >/dev/null 2>&1; then
|
||||||
|
MCLI_BIN="mc"
|
||||||
|
else
|
||||||
|
log_warn "MinIO Client not found (expected mcli or mc); skipping manifest upload"
|
||||||
|
MCLI_BIN=""
|
||||||
|
fi
|
||||||
|
if [[ -n "${MCLI_BIN}" ]]; then
|
||||||
|
local_subpath="${MANIFESTS_SUBPATH:-manifests}"
|
||||||
|
safe_execute "${MCLI_BIN}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
|
||||||
|
dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${local_subpath%/}/${MANIFEST_FILE}"
|
||||||
|
log_info "${MCLI_BIN} cp ${MANIFEST_PATH} ${dst}"
|
||||||
|
safe_execute "${MCLI_BIN}" cp "${MANIFEST_PATH}" "${dst}"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log_info "UPLOAD_MANIFESTS=false; skipping manifest upload"
|
||||||
|
fi
|
||||||
|
|
||||||
|
section "Done"
|
||||||
|
log_info "Packed: ${MANIFEST_PATH}"
|
||||||
Reference in New Issue
Block a user