Add weekly build/test scripts (#1213)

AlexanderRichert-NOAA · web-flow · commit b47798d87ceb · 2024-12-05T23:41:12.000Z
This PR will add scripting for weekly unit testing and build caching of the unified env through Jenkins, cron jobs, etc.

Also a few updates for Acorn, including a change to spack stack create env --compiler that allows compiler version to be specified (spack stack create env --compiler intel -&gt; envs/myenv-intel; spack stack create env --compiler intel@2022 -&gt; envs/myenv-intel-2022), which is needed to differentiate on platforms platform where more than one version of a compiler is installed.
diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -0,0 +1,18 @@
+name: Nightly 01 Directory Setup
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 01 Directory Setup
+    shell: bash
+    run: |
+      # Write RUNID to file so that following steps use the
+      # same date and don't pick up the next day's date
+      RUNID=`date +"%Y%m%d"`
+      echo "${RUNID}" > RUNID_SAVE.log
+      # Get day of week for later use and write to file
+      DOW=$(date +%u)
+      echo "${DOW}" > DOW_SAVE.log
+      #
+      cd util/weekly_build
+      ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 02 Get Spack-Stack
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 02 Get Spack-Stack
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_03_SetupEnv/action.yaml b/.github/actions/Nightly_03_SetupEnv/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 03 Setup Env
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 03 Setup Env
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./03_SetupEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -0,0 +1,20 @@
+name: Nightly 04 Spack Install
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 04 Spack Install
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      DOW=$(<DOW_SAVE.log)
+      # Build everything from scratch on Sundays, otherwise
+      # use build cache to speed up the installations.
+      # Monday is 1 ... Sunday is 7
+      if [[ $DOW == 7 ]]; then
+        export REUSE_BUILD_CACHE="NO"
+      else
+        export REUSE_BUILD_CACHE="YES"
+      fi
+      cd util/weekly_build
+      ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_05_BuildCache/action.yaml b/.github/actions/Nightly_05_BuildCache/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 05 Build Cache
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 05 Build Cache
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_07_Cleanup/action.yaml b/.github/actions/Nightly_07_Cleanup/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 07 Cleanup
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 07 Cleanup
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml
@@ -6,10 +6,7 @@ compilers:
       cxx: CC
       f77: ftn
       fc: ftn
-    flags:
-      cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
-      cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
-      fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
+    flags: {}
     operating_system: sles15
     modules:
     - PrgEnv-intel/8.5.0
@@ -30,10 +27,7 @@ compilers:
       cxx: CC
       f77: ftn
       fc: ftn
-    flags:
-      cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
-      cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
-      fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread
+    flags: {}
     operating_system: sles15
     modules:
     - PrgEnv-intel/8.5.0
diff --git a/configs/sites/tier1/acorn/mirrors.yaml b/configs/sites/tier1/acorn/mirrors.yaml
@@ -0,0 +1,3 @@
+mirrors:
+  local-source: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache
+  local-binary: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache
diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///neptune_diagnostics/spack-stack/source-cache
+  local-binary: file:///neptune_diagnostics/spack-stack/build-cache
diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
diff --git a/doc/source/AutomatedBuilds.rst b/doc/source/AutomatedBuilds.rst
@@ -0,0 +1,54 @@
+================
+Automated builds
+================
+
+.. _AutomatedBuildWorkflow:
+
+Workflow
+*************************
+
+The spack-stack repository now contains scripts for implementing scheduled, automated building, unit testing, and build caching of packages. These scripts live under utils/weekly_build/. They download their own copy of spack-stack (the develop branch, by default) and build one or more environments under that directory structure, which may be transient or reused.
+
+Script failures are detected by non-zero exit codes.
+
+The list of packages to be unit tested is stored in the ``$PACKAGES_TO_TEST`` variable in ShellSetup.sh. As unit testing needs are identified, this list should be updated to be as comprehensive as possible based on https://github.com/JCSDA/spack-stack/wiki/Automated-testing-and-building#unit-testing-requirements.
+
+.. _AutomatedBuildAddPlatform:
+
+Overview of implementing on a new platform
+******************************************
+
+Use a clean environment. Do NOT run setup.sh in the directory containing the build scripts (util/weekly_build/) to be run. Nothing in spack-stack outside of the build scripts is needed to get the automated build workflow running, as the workflow will take care of download a copy of spack-stack to be used for doing the automated builds themselves.
+
+-------------------------------
+Platform-specific configuration
+-------------------------------
+
+The ShellSetup.sh script points to a script in the sites/ subdirectory based on the site name as set by the third argument of each build script (stored and used as ``$PLATFORM`` throughout the scripts). Note that a number of variables and functions can be set or overridden here:
+ - ``$COMPILERS``: Space-delimited list of compilers to be built against on a given system.
+ - ``$TEMPLATES``: Space-delimited list of templates to install. Default is 'unified-env'.
+ - ``$PACKAGES_TO_TEST``: Space-delimited list of packages for which units tests should be run.
+ - ``$PACKAGES_TO_INSTALL``: Space-delimited list of packages to be installed. For many systems it can be left empty so as to install the entire Unified Environment.
+ - ``alert_cmd()``: This function should be set for each system. It is invoked when any step fails (non-zero exit code somewhere in the workflow), and so can be used to send emails, post to Slack or GitHub, etc.
+ - ``spack_wrapper()``: This function runs the Spack command, where the first argument is a log file name. It should not typically need to be redefined by platform.
+ - ``spack_install_wrapper()``: This function determines the means by which the ``spack`` command is invoked for the install steps, and can be overridden for each platform. If testing and installation will be performed outside of a batch scheduler, i.e., directly on a login node, then the default can probably be used. However, overriding this function allows, for instance, for job schedulers to be invoked (srun, qsub, etc.), as well as for parallel installations to be performed. Note that the first argument is a log file name, so custom versions of this function will most likely need to use ``shift`` to properly access the remaining arguments.
+ - ``$INSTALL_OPTS``: Additional ``spack install`` flags to be supplied to ``spack_install_wrapper()``.
+ - ``$TEST_<APPNAME>``: Set this variable to ON in order to enable the corresponding test under util/weekly_build/apptests/.
+ - ``$BUILD_CACHE``: Name of or path to (beginning with '\file://') build cache directory. Defaults to 'local-binary' mirror defined in site's mirrors.yaml.
+ - ``$SOURCE_CACHE``: Name of or path to (beginning with '\file://') source cache directory. Defaults to 'local-source' mirror defined in site's mirrors.yaml.
+ - ``$PADDED_LENGTH``: Padded length setting for Spack build cache generation. It should be as long as possible without the build failing.
+ - ``$KEEP_WEEKLY_BUILD_DIR``: Set to 'YES' to use a persistent directory structure for automated builds.
+
+------------------------------
+Setting up weekly builds: cron
+------------------------------
+
+Prior to running, obtain a current copy of the build scripts by cloning the spack-stack repository to the platform of interest (/path/to/spack-stack-auto-build/).
+
+If /path/to/spack-stack is the local spack-stack root directory containing spack-stack-x.y.z installations, run ``mkdir -p /path/to/spack-stack/weekly_build/logs``.
+
+In crontab, add:
+
+.. code-block:: console
+
+   0 1 * * SUN /path/to/spack-stack-auto-build/util/weekly_build/SpackStackBuildCache_AllSteps.sh $(date +\%y\%m\%d) /path/to/spack-stack/weekly_build acorn > /path/to/spack-stack/weekly_build/logs/cron.$(date +\%y\%m\%d).out 2>&1
diff --git a/doc/source/index.rst b/doc/source/index.rst
@@ -19,6 +19,7 @@ Table of contents
    Utilities
    KnownIssues
    MaintainersSection
+   AutomatedBuilds
    Documentation
    Acronyms
    Glossary
diff --git a/util/weekly_build/01_DirectorySetup.sh b/util/weekly_build/01_DirectorySetup.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+set -ex
+
+echo Base directory: ${RUNDIR:?}
+
+mkdir -p $RUNDIR
diff --git a/util/weekly_build/02_GetSpackStack.sh b/util/weekly_build/02_GetSpackStack.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+set -ex
+
+cd $RUNDIR
+if [ -d $RUNID ]; then
+  cd $RUNID
+  git pull
+else
+  git clone --depth 1 --recurse-submodules --shallow-submodules ${SPACK_STACK_URL:-https://github.com/JCSDA/spack-stack} -b ${SPACK_STACK_BRANCH:-develop} $RUNID
+fi
diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+set -ex
+
+cd $RUNDIR/$RUNID
+
+set +x
+. setup.sh
+set -x
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    echo "Setting up environment $envname in $envdir"
+    rm -rf $envdir
+    spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler
+    cd $envdir
+    spack env activate .
+    spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
+    # Check for duplicates and fail before doing the "real" concretization:
+    spack_wrapper log.concretize concretize --fresh
+    ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
+#   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
+#    spack concretize --force --fresh --test all | tee log.concretize_test
+  done
+done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -ex
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+cd $RUNDIR/${RUNID}
+
+set +x
+. setup.sh
+set -x
+
+if [ "$REUSE_BUILD_CACHE" == YES ]; then
+  cache_flag="--no-check-signature"
+else
+  cache_flag="--no-cache"
+fi
+
+INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS"
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    echo "Building environment $envname in $envdir"
+    cd $envdir
+    spack env activate .
+    if [ "${SOURCE_CACHE::7}" == "file://" ]; then
+      mirrorpath=${SOURCE_CACHE}
+    else
+      mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
+    fi
+    mirrorpath=${mirrorpath#file://}
+    spack_wrapper log.fetch mirror create --dependencies \
+        --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \
+        ${PACKAGES_TO_INSTALL:---all}
+    # Just install the packages we're testing (+dependencies):
+    if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
+      spack_install_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
+    fi
+    # Install the rest of the stack as usual:
+    spack_install_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL
+  done
+done
diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+cd $RUNDIR/$RUNID
+
+set +x
+. setup.sh
+set -x
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    cd $envdir
+    spack env activate .
+    spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL
+    spack buildcache rebuild-index ${BUILD_CACHE}
+  done
+done
diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -ex
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+cd $RUNDIR/$RUNID
+
+set +x
+. setup.sh
+set -x
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    cd $envdir
+    spack env activate .
+    spack module lmod refresh -y
+    spack stack setup-meta-modules
+  done
+done
+
+if [ "$TEST_UFSWM" == ON ]; then
+  . $(dirname $0)/apptests/test_ufswm.sh
+fi
diff --git a/util/weekly_build/07_Cleanup.sh b/util/weekly_build/07_Cleanup.sh
diff --git a/util/weekly_build/README b/util/weekly_build/README
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh
diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh
diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh
diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+mirrors:`
	`2`	`+ local-source: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache`
	`3`	`+ local-binary: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache`