From 120e01441b9abf4f58f12969aeddc8b158861b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 10:55:57 +0200 Subject: [PATCH 1/7] determine $EESSI_VERSIONS_SUBPATH based on type of build, and pass it to create_tarball.sh --- bot/build.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 4c91f19b..460c1325 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -149,6 +149,8 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # "source init/eessi_defaults" via sourcing init/minimal_eessi_env # Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we # append the project subdirectory to ${EESSI_CVMFS_REPO_OVERRIDE} +# TODO: is appending the dev project name to EESSI_CVMFS_REPO_OVERRIDE still required? +# Probably not for creating tarballs, but maybe still in checking the easystack file location in the repo in EESSI-install-software.sh? export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" @@ -172,6 +174,17 @@ if [[ "${EESSI_CVMFS_REPO_OVERRIDE}" != "/cvmfs/software.eessi.io" && "${REPOSIT # Make sure that the compatibility layer is still used from software.eessi.io export EESSI_CVMFS_COMPAT_REPO=/cvmfs/software.eessi.io echo "EESSI_CVMFS_COMPAT_REPO=${EESSI_CVMFS_COMPAT_REPO}" + + # Determine the relative path to the versions subdir for site installations + # by removing the /cvmfs/some.repo.tld part and appending /versions at the end + # E.g. EESSI_SITE_SOFTWARE_PREFIX=/cvmfs/my.site.tld/eessi/builds would result in eessi/builds/versions + export EESSI_VERSIONS_SUBPATH="${EESSI_SITE_SOFTWARE_PREFIX#/cvmfs/${REPOSITORY_NAME}}/versions" +elif [[ "${REPOSITORY_NAME}" != "dev.eessi.io" ]]; then + # For dev.eessi.io the versions directory is a subdir of the project directory, e.g. /cvmfs/dev.eessi.io/example/versions + export EESSI_VERSIONS_SUBPATH=${EESSI_DEV_PROJECT}/versions +else + # For software.eessi.io the versions dir is always in the root of the repository + export EESSI_VERSIONS_SUBPATH=versions fi # determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} @@ -371,8 +384,8 @@ TMP_IN_CONTAINER=/tmp tarball_accelerators=$(IFS=+; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}") echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"$tarball_accelerators\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSIONS_SUBPATH} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"$tarball_accelerators\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$tarball_accelerators" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSIONS_SUBPATH} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$tarball_accelerators" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From e99ce302c96502fc5582cc292c6438692f3b6bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 10:56:49 +0200 Subject: [PATCH 2/7] make script more generic for builds in subdirs of the repo --- create_tarball.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 2472ac94..6a0c50ea 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -4,15 +4,16 @@ set -e base_dir=$(dirname $(realpath $0)) -if [ $# -ne 5 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ $# -ne 6 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 -eessi_version=$2 -cpu_arch_subdir=$3 -accel_subdirs="$4" -target_tarball=$5 +eessi_versions_subpath=$2 +eessi_version=$3 +cpu_arch_subdir=$4 +accel_subdirs="$5" +target_tarball=$6 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -20,7 +21,7 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +software_dir="${cvmfs_repo}/${eessi_versions_subpath}/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -29,18 +30,18 @@ fi # Need to extract the cvmfs_repo_name from the cvmfs_repo variable # - remove /${EESSI_DEV_PROJECT} from the end (if it exists) # - remove /cvmfs/ from the beginning -cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} +cvmfs_repo_name=${cvmfs_repo%"/${eessi_versions_subpath}"} cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${eessi_versions_subpath:+/$eessi_versions_subpath}" -software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" +software_dir_overlay="${overlay_upper_dir}/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 fi current_workdir=${PWD} -cd ${overlay_upper_dir}/versions/ +cd ${overlay_upper_dir} echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." files_list=${tmpdir}/files.list.txt @@ -115,7 +116,7 @@ if [ -r ${module_files_list} ]; then cp ${module_files_list} ${current_workdir}/module_files.list.txt fi -topdir=${cvmfs_repo}/versions/ +topdir=${cvmfs_repo}/${eessi_versions_subpath} echo ">> Creating tarball ${target_tarball} from ${topdir}..." tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list} From 6376429d4534a6ca69d0cc97d8f274efb4a7a36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 11:07:19 +0200 Subject: [PATCH 3/7] don't append EESSI_DEV_PROJECT to EESSI_CVMFS_REPO_OVERRIDE --- bot/build.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 460c1325..72e20c30 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -147,11 +147,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we -# append the project subdirectory to ${EESSI_CVMFS_REPO_OVERRIDE} -# TODO: is appending the dev project name to EESSI_CVMFS_REPO_OVERRIDE still required? -# Probably not for creating tarballs, but maybe still in checking the easystack file location in the repo in EESSI-install-software.sh? -export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # If we're not building for software.eessi.io, then consider this a site install From 99f717b574ab0483e36fad1e047c441bc9cfaac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 11:10:45 +0200 Subject: [PATCH 4/7] don't run basename on EESSI_CVMFS_REPO_OVERRIDE, we no longer append dev project name to the override --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 8dc0663e..964d8d83 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -413,10 +413,10 @@ else # make sure that easystack file being picked up is for EESSI version that we're building for... # Preferentially check EESSI_CVMFS_REPO_OVERRIDE (since that is set for site builds), otherwise default to EESSI_CVMFS_REPO - echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})/${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}/" + echo "${easystack_file}" | grep -q "^easystacks/${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}/${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}/" if [ $? -ne 0 ]; then # Check if this was even an easystack file for the right repository - echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})" + echo "${easystack_file}" | grep -q "^easystacks/${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}" if [ $? -ne 0 ]; then echo_yellow "Easystack file ${easystack_file} is not intended for the repository ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}, skipping it..." else From 2f78c0578c0d07f6db2214044815c6a43d584ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 15:02:11 +0200 Subject: [PATCH 5/7] remove all dev.eessi.io-specific code, rely on site installation functionality --- EESSI-install-software.sh | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 964d8d83..409ae63d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -185,9 +185,9 @@ fi pr_diff=$(ls [0-9]*.diff | head -n 1) export PR_DIFF="$PWD/$pr_diff" -# Only run install_scripts.sh if not in dev.eessi.io (for security) -# Also skip installing scripts for site-installs -if [[ -z ${EESSI_DEV_PROJECT} && -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then +# Only run install_scripts.sh for software.eessi.io builds, +# i.e. skip it for site-installs (which also includes builds for dev.eessi.io) +if [[ -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION} --eessi-version ${EESSI_VERSION} fi @@ -294,16 +294,6 @@ fi export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EASYBUILD_INSTALLPATH set to $EASYBUILD_INSTALLPATH" -# If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - # We keep track of the old install path for settings paths to SitePackage.lua and .lmodrc later - EASYBUILD_INSTALLPATH_STANDARD=${EASYBUILD_INSTALLPATH} - # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL - unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} - echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" -fi - # If we have EESSI_ACCELERATOR_TARGET_OVERRIDE set (and non-empty), then this implies building for a GPU target # (this must be set _before_ we load EESSI-extend). # We also make sure that EESSI_ACCELERATOR_TARGET is also set as EESSI_ACCELERATOR_TARGET_OVERRIDE must @@ -469,14 +459,8 @@ else fi echo "DEBUG: before creating/updating lmod config files // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - # Make sure .lmod files are not checked for dev.eeessi.io - export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" - export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" -else - export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" - export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" -fi +export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" +export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" # If this is a site install, the old method of checking if lmodrc.lua was updated doesn't work # We simply skip that step for now - it's hardly ever changed anyway From 3af0ba191b3a66a18a3dff253d3a68403982530f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 15:02:38 +0200 Subject: [PATCH 6/7] remove all dev.eessi.io-specific code, rely on site installation functionality --- bot/build.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 72e20c30..a60054fe 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -151,7 +151,7 @@ export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # If we're not building for software.eessi.io, then consider this a site install -if [[ "${EESSI_CVMFS_REPO_OVERRIDE}" != "/cvmfs/software.eessi.io" && "${REPOSITORY_NAME}" != "dev.eessi.io" ]]; then +if [[ "${EESSI_CVMFS_REPO_OVERRIDE}" != "/cvmfs/software.eessi.io" ]]; then # To build on top of EESSI, we need the software.eessi.io repository to be mounted next to the target repository # The bot/build.sh script does this when the EESSI_SITE_INSTALL_FORCE environment variable is set # Other build scripts will also respect this variable where needed in order to make sure that 'building on top' @@ -174,10 +174,8 @@ if [[ "${EESSI_CVMFS_REPO_OVERRIDE}" != "/cvmfs/software.eessi.io" && "${REPOSIT # Determine the relative path to the versions subdir for site installations # by removing the /cvmfs/some.repo.tld part and appending /versions at the end # E.g. EESSI_SITE_SOFTWARE_PREFIX=/cvmfs/my.site.tld/eessi/builds would result in eessi/builds/versions + # Note that this also works for dev.eessi.io builds, as these are configured as site installations export EESSI_VERSIONS_SUBPATH="${EESSI_SITE_SOFTWARE_PREFIX#/cvmfs/${REPOSITORY_NAME}}/versions" -elif [[ "${REPOSITORY_NAME}" != "dev.eessi.io" ]]; then - # For dev.eessi.io the versions directory is a subdir of the project directory, e.g. /cvmfs/dev.eessi.io/example/versions - export EESSI_VERSIONS_SUBPATH=${EESSI_DEV_PROJECT}/versions else # For software.eessi.io the versions dir is always in the root of the repository export EESSI_VERSIONS_SUBPATH=versions @@ -225,16 +223,17 @@ COMMON_ARGS+=("--mode" "exec") [[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID}") # Also expose software.eessi.io when building on top of EESSI (i.e. when EESSI_SITE_INSTALL_FORCE is set) -# or on top of the EESSI compat layer (i.e. for dev.eessi.io) -if [[ "${REPOSITORY_NAME}" == "dev.eessi.io" || -n "${EESSI_SITE_INSTALL_FORCE}" ]]; then +if [[ -n "${EESSI_SITE_INSTALL_FORCE}" ]]; then COMMON_ARGS+=("--repository" "software.eessi.io,access=ro") fi # Override the compat layer if EESSI_CVMFS_COMPAT_REPO is defined. This allows using a different repo for the # compatibility layer compared to the EESSI_CVMFS_REPO (in which things will be installed) -# (this is already done for dev.eessi.io through a customized SLURM script but that could probably be removed then) if [[ -n "${EESSI_CVMFS_COMPAT_REPO}" && -n "${EESSI_VERSION_OVERRIDE:-$EESSI_VERSION}" ]]; then - export EESSI_COMPAT_LAYER_DIR_OVERRIDE="${EESSI_CVMFS_COMPAT_REPO}/versions/${EESSI_VERSION_OVERRIDE:-$EESSI_VERSION}/compat/linux/$(uname -m)" + EESSI_COMPAT_VERSION=${EESSI_VERSION_OVERRIDE:-$EESSI_VERSION} + # Cut off any version suffix (2025.06-001 -> 2025.06) + EESSI_COMPAT_VERSION=${EESSI_COMPAT_VERSION%-*} + export EESSI_COMPAT_LAYER_DIR_OVERRIDE="${EESSI_CVMFS_COMPAT_REPO}/versions/${EESSI_COMPAT_VERSION}/compat/linux/$(uname -m)" msg="bot:build.sh: Set EESSI_COMPAT_LAYER_DIR_OVERRIDE to $EESSI_COMPAT_LAYER_DIR_OVERRIDE since both EESSI_CVMFS_COMPAT_REPO" msg="$msg (${EESSI_CVMFS_COMPAT_REPO}) and EESSI_VERSION_OVERRIDE (${EESSI_VERSION_OVERRIDE}) are defined" echo "$msg" @@ -368,10 +367,6 @@ else export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi -# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball -echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'" -export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} - # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh # TODO should we make this a configurable parameter of eessi_container.sh using From 2c96760be87c81f805c35a7746e4199854fdb128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 26 Aug 2026 15:33:44 +0200 Subject: [PATCH 7/7] readd basename command to find easystack repo --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 409ae63d..5a96f38e 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -403,10 +403,10 @@ else # make sure that easystack file being picked up is for EESSI version that we're building for... # Preferentially check EESSI_CVMFS_REPO_OVERRIDE (since that is set for site builds), otherwise default to EESSI_CVMFS_REPO - echo "${easystack_file}" | grep -q "^easystacks/${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}/${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}/" + echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})/${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}/" if [ $? -ne 0 ]; then # Check if this was even an easystack file for the right repository - echo "${easystack_file}" | grep -q "^easystacks/${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}" + echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})" if [ $? -ne 0 ]; then echo_yellow "Easystack file ${easystack_file} is not intended for the repository ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}, skipping it..." else