More Acorn+WCOSS2/NCO updates #1051
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ubuntu-ci-c6a-x86_64-oneapi-build | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| # DH* REVERT ME AFTER MERGE | |
| BUILD_CACHE_PATH: /home/ubuntu/spack-stack/build-cache-new-spack-v1 | |
| SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache | |
| jobs: | |
| ubuntu-ci-c6a-x86_64-oneapi-build: | |
| runs-on: [ubuntu-ci-c6a-x86_64] | |
| steps: | |
| - name: cleanup | |
| run: | | |
| pwd | |
| ls -lart | |
| set +e | |
| find ./* -type d -exec chmod u+xw {} \; | |
| set -e | |
| rm -fr * | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: prepare-directories | |
| run: | | |
| # DH* REVERT ME AFTER MERGE | |
| mkdir -p ${BUILD_CACHE_PATH} | |
| mkdir -p ${SOURCE_CACHE_PATH} | |
| - name: create-buildcache | |
| run: | | |
| # Get day of week to decide whether to use build caches or not | |
| DOW=$(date +%u) | |
| # Monday is 1 ... Sunday is 7 | |
| if [[ $DOW == 7 ]]; then | |
| export USE_BINARY_CACHE=false | |
| echo "Ignore existing binary cache for creating buildcache environment" | |
| else | |
| export USE_BINARY_CACHE=true | |
| echo "Use existing binary cache for creating buildcache environment" | |
| fi | |
| # Set up spack-stack | |
| source ./setup.sh | |
| declare -a TEMPLATES=("unified-dev" "skylab-dev" "neptune-dev") | |
| for TEMPLATE in "${TEMPLATES[@]}"; do | |
| if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then | |
| export ENVNAME=ue-oneapi-2024.2.0-buildcache | |
| elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then | |
| export ENVNAME=se-oneapi-2024.2.0-buildcache | |
| elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then | |
| export ENVNAME=ce-oneapi-2024.2.0-buildcache | |
| elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then | |
| export ENVNAME=ne-oneapi-2024.2.0-buildcache | |
| fi | |
| echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" | |
| export ENVDIR=$PWD/envs/${ENVNAME} | |
| spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi | |
| spack env activate ${ENVDIR} | |
| # Turn off variant "adp" for neptune-dev (NRL internal only) | |
| if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then | |
| sed -i 's/+adp/~adp/g' ${ENVDIR}/spack.yaml | |
| fi | |
| unset SPACK_DISABLE_LOCAL_CONFIG | |
| export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" | |
| # Find external packages | |
| spack external find --scope system \ | |
| --exclude bison --exclude openssl \ | |
| --exclude curl --exclude python \ | |
| --exclude meson --exclude gmake | |
| spack external find --scope system grep | |
| spack external find --scope system sed | |
| spack external find --scope system perl | |
| spack external find --scope system wget | |
| spack external find --scope system texlive | |
| spack external find --scope system mysql | |
| # Find compilers | |
| spack compiler find --scope system | |
| # For Intel oneAPI | |
| source /etc/profile.d/modules.sh | |
| module use /opt/intel/oneapi/modulefiles | |
| cat .github/workflows/ubuntu-ci-x86_64-oneapi.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml | |
| export SPACK_DISABLE_LOCAL_CONFIG=true | |
| unset SPACK_SYSTEM_CONFIG_PATH | |
| # For buildcaches | |
| spack config add config:install_tree:padded_length:200 | |
| # Set compiler and MPI specs | |
| spack config add "packages:mpi:buildable:False" | |
| spack config add "packages:mpi:require:['[email protected]']" | |
| spack config add "packages:all:prefer:['%oneapi']" | |
| # Add additional variants for MET packages, different from config/common/packages.yaml | |
| spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" | |
| ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) | |
| #spack config add "packages:gmake:require:'@:4.2'" | |
| ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI | |
| #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` | |
| # Don't generate ecflow module when using external package | |
| spack config add "modules:default:tcl:exclude:[ecflow]" | |
| # Concretize and check for duplicates | |
| spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} | |
| ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env | |
| # Add and update source cache | |
| spack mirror add local-source file://${SOURCE_CACHE_PATH}/ | |
| spack mirror create -a -d ${SOURCE_CACHE_PATH}/ | |
| # Add binary cache if requested | |
| if [ "$USE_BINARY_CACHE" = true ] ; then | |
| set +e | |
| spack mirror add local-binary file://${BUILD_CACHE_PATH}/ | |
| spack buildcache update-index local-binary | |
| set -e | |
| echo "Packages in spack binary cache:" | |
| spack buildcache list | |
| fi | |
| # Break installation up in pieces and create build caches in between | |
| # This allows us to "spin up" builds that altogether take longer than | |
| # six hours, and/or fail later in the build process. | |
| if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then | |
| # base-env | |
| echo "base-env ..." | |
| spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env | |
| spack buildcache create -u local-binary base-env | |
| # jedi-base-env | |
| echo "jedi-base-env ..." | |
| spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env | |
| spack buildcache create -u local-binary jedi-base-env | |
| # jedi-ufs-env | |
| echo "jedi-ufs-env ..." | |
| spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env | |
| spack buildcache create -u local-binary jedi-ufs-env | |
| elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then | |
| # base-env | |
| echo "base-env ..." | |
| spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env | |
| spack buildcache create -u local-binary base-env | |
| # neptune-python-env | |
| echo "neptune-env ..." | |
| spack install --fail-fast --source --no-check-signature neptune-python-env 2>&1 | tee log.install.${ENVNAME}.neptune-env | |
| spack buildcache create -u local-binary neptune-python-env | |
| fi | |
| # the rest | |
| echo "${TEMPLATE} ..." | |
| spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all | |
| spack buildcache create -u local-binary | |
| # Check for libirc.so in shared libraries (returns 1=error if found) | |
| ${SPACK_STACK_DIR}/util/check_libirc.sh | |
| # Remove binary cache for next round of concretization | |
| if [ "$USE_BINARY_CACHE" = true ] ; then | |
| spack mirror rm local-binary | |
| fi | |
| # Remove buildcache config settings | |
| spack config remove config:install_tree:padded_length | |
| # Next steps: synchronize source and build cache to a central/combined mirror? | |
| # Cleanup | |
| spack clean -a | |
| spack env deactivate | |
| done | |
| - name: create-env | |
| run: | | |
| # Set up spack-stack | |
| source ./setup.sh | |
| export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache | |
| export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} | |
| export ENVNAME=ue-oneapi-2024.2.0 | |
| export ENVDIR=$PWD/envs/${ENVNAME} | |
| rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ | |
| spack env activate ${ENVDIR} | |
| # Concretize and check for duplicates | |
| spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} | |
| ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env | |
| # Add binary cache back in | |
| spack mirror add local-binary file://${BUILD_CACHE_PATH}/ | |
| echo "Packages in combined spack build caches:" | |
| spack buildcache list | |
| # Install from cache | |
| spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all | |
| # For spack-built Intel oneAPI runtime to find libintlc.so.5 etc. | |
| export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2024.2/lib:$LD_LIBRARY_PATH" | |
| # Check shared libraries | |
| ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check | |
| # Check for libirc.so in shared libraries (returns 1=error if found) | |
| ${SPACK_STACK_DIR}/util/check_libirc.sh | |
| # Create modules | |
| spack clean -a | |
| spack module tcl refresh -y | |
| spack stack setup-meta-modules | |
| spack env deactivate | |
| - name: test-env | |
| run: | | |
| source /etc/profile.d/modules.sh | |
| module use /opt/intel/oneapi/modulefiles | |
| module use /home/ubuntu/spack-stack/modulefiles | |
| export ENVNAME=ue-oneapi-2024.2.0 | |
| export ENVDIR=$PWD/envs/${ENVNAME} | |
| ls -l ${ENVDIR}/modules/Core | |
| module use ${ENVDIR}/modules/Core | |
| module load stack-intel-oneapi-compilers/2024.2.0 | |
| module load stack-intel-oneapi-mpi/2021.13 | |
| module available | |
| module load jedi-ufs-env | |
| module load ewok-env | |
| module load soca-env | |
| module list |