[sanitizer] Extract build_step function

Author: vitalybuka

zorg/buildbot/builders/sanitizers/buildbot_android.sh

@@ -21,7 +21,7 @@ CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON"
 
 build_stage2_android
 
-echo @@@BUILD_STEP run cmake@@@
+build_step "run cmake"
 configure_android aarch64 aarch64-linux-android
 # Testing armv7 instead of plain arm to work around
 # https://code.google.com/p/android/issues/detail?id=68779

zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh

@@ -9,7 +9,7 @@ function download_android_tools {
   local FILE_NAME=${VERSION}-linux-x86_64.zip
   local NDK_URL=https://dl.google.com/android/repository/${FILE_NAME}
   if  [[ "$(cat ${NDK_DIR}/android_ndk_url)" != ${NDK_URL} ]] ; then
-    echo @@@BUILD_STEP downloading Android NDK@@@
+    build_step "downloading Android NDK"
     [[ -d ${NDK_DIR} ]] && rm -rf ${NDK_DIR}
     [[ -d ${VERSION} ]] && rm -rf ${VERSION}
     [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME}
@@ -20,7 +20,7 @@ function download_android_tools {
   fi
 
   if  [[ ! -d platform-tools ]] ; then
-    echo @@@BUILD_STEP downloading Android Platform Tools@@@
+    build_step "downloading Android Platform Tools"
     local FILE_NAME=platform-tools-latest-linux.zip
     [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME}
     wget -e dotbytes=10m https://dl.google.com/android/repository/${FILE_NAME}
@@ -34,7 +34,7 @@ function build_stage2_android() {
   local CMAKE_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON ${STAGE1_AS_COMPILER}"
   CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DCLANG_DEFAULT_RTLIB=libgcc"
 
-  echo @@@BUILD_STEP bootstrap clang@@@
+  build_step "bootstrap clang"
   mkdir -p llvm_build64
   if  [[ "$(cat llvm_build64/CMAKE_OPTIONS)" != "${CMAKE_OPTIONS}" ]] ; then
     (cd llvm_build64 && cmake ${CMAKE_OPTIONS} $LLVM && \
@@ -105,7 +105,7 @@ BUILD_RT_ERR=""
 function build_android {
   local _arch=$1
   wait
-  echo @@@BUILD_STEP build android/$_arch@@@
+  build_step "build android/$_arch"
   if ! ninja -C llvm_build_android_$_arch llvm-symbolizer ; then
     BUILD_RT_ERR="${BUILD_RT_ERR}|${_arch}|"
     build_failure
@@ -134,7 +134,7 @@ function patch_abilist { # IN OUT
 
 function restart_adb_server {
   ADB=adb
-  echo @@@BUILD_STEP restart adb server@@@
+  build_step "restart adb server"
   $ADB kill-server
   sleep 2
   $ADB start-server
@@ -184,7 +184,7 @@ function test_android {
   fi
 
   ADB=adb
-  echo @@@BUILD_STEP run all tests@@@
+  build_step "run all tests"
   ANDROID_DEVICES=$(${ADB} devices | grep 'device$' | sort -r | awk '{print $1}')
 
   rm -rf test_android_log_*
@@ -200,7 +200,7 @@ function test_android {
   for _arg in "$@"; do
     local _arch=${_arg%:*}
     if [[ ! -f tested_arch_$_arch ]]; then
-      echo @@@BUILD_STEP unavailable device android/$_arch@@@
+      build_step "unavailable device android/$_arch"
       echo @@@STEP_EXCEPTION@@@
     fi
   done
@@ -220,7 +220,7 @@ function run_tests_sharded {
 
   local NUM_SHARDS=4
   local _log_prefix=$(mktemp shards_XXXX_)
-  echo @@@BUILD_STEP run $_test_name tests [$DEVICE_DESCRIPTION]@@@
+  build_step "run $_test_name tests [$DEVICE_DESCRIPTION]"
   LOGS=
   for ((SHARD=0; SHARD < $NUM_SHARDS; SHARD++)); do
     LOG=${_log_prefix}_$SHARD
@@ -253,7 +253,7 @@ function test_arch_on_device {
   export ANDROID_SERIAL=$_serial
   echo "Serial $_serial"
 
-  echo @@@BUILD_STEP device setup [$DEVICE_DESCRIPTION]@@@
+  build_step "device setup [$DEVICE_DESCRIPTION]"
   $ADB wait-for-device
   $ADB devices
   $ADB shell getprop ro.build.version.release
@@ -276,7 +276,7 @@ function test_arch_on_device {
   done
   wait
 
-  echo @@@BUILD_STEP run lit tests [$DEVICE_DESCRIPTION]@@@
+  build_step "run lit tests [$DEVICE_DESCRIPTION]"
   (cd $COMPILER_RT_BUILD_DIR && ninja check-all) || build_failure
 
   run_tests_sharded sanitizer_common SanitizerTest ""

zorg/buildbot/builders/sanitizers/buildbot_bisect_run.sh

@@ -11,7 +11,7 @@ HERE="$(cd $(dirname $0) && pwd)"
 
 BUILDBOT_REVISION=origin/main buildbot_update
 
-echo @@@BUILD_STEP bisecting ${BUILDBOT_REVISION}@@@
+build_step "bisecting ${BUILDBOT_REVISION}"
 
 # Try to get them out from the bisect string in BUILDBOT_REVISION first.
 GOOD="${BUILDBOT_REVISION/:*/}"
@@ -34,7 +34,7 @@ cd "${LLVM}/.."
     git bisect run bash -c "cd $ROOT && $*"
   fi
 
-  echo @@@BUILD_STEP bisect result@@@
+  build_step "bisect result"
   git bisect log
 ) || true
 

zorg/buildbot/builders/sanitizers/buildbot_cmake.sh

@@ -59,7 +59,7 @@ function build {
     BUILD_DIR="build_${1}"
   fi
 
-  echo "@@@BUILD_STEP build compiler-rt ${1}@@@"
+  build_step "build compiler-rt ${1}"
   [[ ! -f "${BUILD_DIR}/delete_next_time" ]] || rm -rf "${BUILD_DIR}"
   mkdir -p ${BUILD_DIR}
 
@@ -76,7 +76,7 @@ function build {
 function build_and_test {
   build "${1}" "${2}"
 
-  echo "@@@BUILD_STEP test compiler-rt ${1}@@@"
+  build_step "test compiler-rt ${1}"
   ninja -C ${BUILD_DIR} check-compiler-rt || build_failure
 }
 
@@ -98,7 +98,7 @@ build_and_test "default" ""
 
 FRESH_CLANG_PATH=${ROOT}/${BUILD_DIR}/bin
 
-echo @@@BUILD_STEP build standalone compiler-rt@@@
+build_step "build standalone compiler-rt"
 mkdir -p compiler_rt_build
 # Standalone build as in https://compiler-rt.llvm.org/
 cmake -B compiler_rt_build -GNinja \
@@ -110,12 +110,12 @@ cmake -B compiler_rt_build -GNinja \
   $LLVM/../compiler-rt || build_failure
 ninja -C compiler_rt_build || build_failure
 
-echo @@@BUILD_STEP test standalone compiler-rt@@@
+build_step "test standalone compiler-rt"
 ninja -C compiler_rt_build check-all || build_failure
 
 if [ "$CHECK_TSAN" == "1" ]; then
   # FIXME: Convert to a LIT test.
-  echo @@@BUILD_STEP tsan analyze@@@
+  build_step "tsan analyze"
   BIN=tsan_bin
   echo "int main() {return 0;}" | ${FRESH_CLANG_PATH}/clang -x c++ - -fsanitize=thread -O2 -o ${BIN}
   $LLVM/../compiler-rt/lib/tsan/check_analyze.sh ${BIN} || build_failure

zorg/buildbot/builders/sanitizers/buildbot_fast.sh

@@ -5,7 +5,7 @@ HERE="$(cd $(dirname $0) && pwd)"
 
 # TODO: find a better way to disable cleanup.
 function cleanup() {
-  echo @@@BUILD_STEP cleanup@@@
+  build_step "cleanup"
   # Workaround the case when a new unittest was reverted, but incremental build continues to execute the leftover binary.
   find -executable -type f -wholename *unittests* -delete
   du -hs * | sort -h

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

@@ -24,6 +24,10 @@ fi
 
 SANITIZER_LOG_DIR=$ROOT/sanitizer_logs
 
+function build_step() {
+  echo "@@@BUILD_STEP ""$@""@@"
+}
+
 function include_config() {
   local P=.
   while true ; do
@@ -39,7 +43,7 @@ function include_config() {
 
 include_config
 
-echo @@@BUILD_STEP Info@@@
+build_step "Info"
 (
   set +e
   date
@@ -66,7 +70,7 @@ echo @@@BUILD_STEP Info@@@
   hostname -f
 )
 
-echo @@@BUILD_STEP Prepare@@@
+build_step "Prepare"
 
 export LIT_OPTS="--time-tests"
 # --timeout requires psutil missing on some bots.
@@ -89,7 +93,7 @@ function rm_dirs {
 
 function cleanup() {
   [[ -v BUILDBOT_BUILDERNAME ]] || return 0
-  echo @@@BUILD_STEP cleanup@@@
+  build_step "cleanup"
   rm_dirs llvm_build2_* llvm_build_* libcxx_build_* libcxx_install_* compiler_rt_build* symbolizer_build* "$@"
   if ccache -s >/dev/null ; then
     rm_dirs llvm_build64
@@ -101,7 +105,7 @@ function cleanup() {
 
 function clobber {
   if [[ "$BUILDBOT_CLOBBER" != "" ]]; then
-    echo @@@BUILD_STEP clobber@@@
+    build_step "clobber"
     if [[ ! -v BUILDBOT_BUILDERNAME ]]; then
       echo "Clobbering is supported on buildbot only!"
       exit 1
@@ -119,7 +123,7 @@ BUILDBOT_MONO_REPO_PATH=${BUILDBOT_MONO_REPO_PATH:-}
 
 function buildbot_update {
   if [[ "${BUILDBOT_BISECT_MODE:-}" == "1" ]]; then
-    echo "@@@BUILD_STEP bisect status@@@"
+    build_step "bisect status"
     (
       cd llvm-project
       git bisect visualize --oneline
@@ -129,7 +133,7 @@ function buildbot_update {
     LLVM=$ROOT/llvm-project/llvm
     return 0
   fi
-  echo "@@@BUILD_STEP update $BUILDBOT_REVISION@@@"
+  build_step "update $BUILDBOT_REVISION"
   if [[ -d "$BUILDBOT_MONO_REPO_PATH" ]]; then
     LLVM=$BUILDBOT_MONO_REPO_PATH/llvm
   else
@@ -163,7 +167,7 @@ function buildbot_update {
 
 function print_sanitizer_logs() {
   if compgen -G "${SANITIZER_LOG_DIR}"/* ; then
-    echo @@@BUILD_STEP sanitizer logs@@@
+    build_step "sanitizer logs"
     head -n -1 "${SANITIZER_LOG_DIR}"/*
     buildbot_build && rm -rf "${SANITIZER_LOG_DIR}"/*
     build_warning
@@ -207,7 +211,7 @@ function build_stage1_clang_impl {
 }
 
 function build_stage1_clang {
-  echo @@@BUILD_STEP stage1 build all@@@
+  build_step "stage1 build all"
   common_stage1_variables
   build_stage1_clang_impl
 }
@@ -218,7 +222,7 @@ function download_clang_from_chromium {
   curl -s https://raw.githubusercontent.com/chromium/chromium/main/tools/clang/scripts/update.py \
     | python3 - --output-dir=${STAGE1_DIR}
 
-  echo @@@BUILD_STEP using pre-built stage1 clang at $(cat ${STAGE1_DIR}/cr_build_revision)@@@
+  build_step "using pre-built stage1 clang at $(cat ${STAGE1_DIR}/cr_build_revision)"
 }
 
 function build_clang_at_release_tag {
@@ -235,12 +239,12 @@ function build_clang_at_release_tag {
   if  [ -r ${STAGE1_DIR}/host_clang_revision ] && \
       [ "$(cat ${STAGE1_DIR}/host_clang_revision)" == $host_clang_revision ]
   then
-    echo "@@@BUILD_STEP using pre-built stage1 clang at r${host_clang_revision}@@@"
+    build_step "using pre-built stage1 clang at r${host_clang_revision}"
   else
     SKIP_OLD=0 BUILDBOT_MONO_REPO_PATH="" BUILDBOT_REVISION="${host_clang_revision}" buildbot_update
 
     rm -rf ${STAGE1_DIR}
-    echo @@@BUILD_STEP build stage1 clang at $host_clang_revision@@@
+    build_step "build stage1 clang at $host_clang_revision"
     # PGO, can improve build time by 10%. However bots spend most of the time
     # running tests and compilation mostly incremental or CCCACH-ed.
     build_stage1_clang_impl && \
@@ -258,7 +262,7 @@ function common_stage2_variables {
 
 function build_stage2 {
   local sanitizer_name=$1
-  echo @@@BUILD_STEP stage2/$sanitizer_name build libcxx@@@
+  build_step "stage2/$sanitizer_name build libcxx"
 
   local libcxx_build_dir=libcxx_build_${sanitizer_name}
   local libcxx_install_dir=libcxx_install_${sanitizer_name}
@@ -348,7 +352,7 @@ function build_stage2 {
   local sanitizer_ldflags="-Wl,--rpath=${libcxx_runtime_path} -L${libcxx_runtime_path}"
   local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_install_dir}/include -isystem ${ROOT}/${libcxx_install_dir}/include/c++/v1 $fsanitize_flag"
 
-  echo @@@BUILD_STEP stage2/$sanitizer_name build@@@
+  build_step "stage2/$sanitizer_name build"
 
   # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264
   sanitizer_cflags+=" $sanitizer_ldflags -w"
@@ -434,7 +438,7 @@ function check_stage2 {
   if [[ "${STAGE2_SKIP_TEST_CXX:-}" != "1" ]] ; then
     (
       LIT_OPTS+=" --timeout=1500"
-      echo @@@BUILD_STEP stage2/$sanitizer_name check-cxx@@@
+      build_step "stage2/$sanitizer_name check-cxx"
       # Very slow.
       export LIT_FILTER_OUT="std/utilities/format/format.functions/format.locale.runtime_format.pass.cpp"
       LIT_FILTER_OUT+="|std/utilities/format/format.functions/format.runtime_format.pass.cpp"
@@ -466,7 +470,7 @@ function check_stage2 {
     )
   fi
 
-  echo @@@BUILD_STEP stage2/$sanitizer_name check@@@
+  build_step "stage2/$sanitizer_name check"
   run_ninja -C ${STAGE2_DIR} check-all
 }
 
@@ -496,7 +500,7 @@ function check_stage2_asan_ubsan {
 
 function build_stage3 {
   local sanitizer_name=$1
-  echo @@@BUILD_STEP build stage3/$sanitizer_name build@@@
+  build_step "build stage3/$sanitizer_name build"
 
   local build_dir=llvm_build2_${sanitizer_name}
 
@@ -547,7 +551,7 @@ function build_stage3_ubsan {
 
 function check_stage3 {
   local sanitizer_name=$1
-  echo @@@BUILD_STEP stage3/$sanitizer_name check@@@
+  build_step "stage3/$sanitizer_name check"
 
   local build_dir=llvm_build2_${sanitizer_name}
 

zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh

@@ -24,12 +24,12 @@ build_stage1_clang
 # export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}"
 export PATH="$(readlink -f ${STAGE1_DIR}/bin):$PATH"
 
-echo @@@BUILD_STEP get fuzzer-test-suite @@@
+build_step "get fuzzer-test-suite "
 [ ! -e fuzzer-test-suite ] && git clone https://github.com/google/fuzzer-test-suite.git
 (cd fuzzer-test-suite && git pull)
 
 RunFuzzerTest() {
-  echo @@@BUILD_STEP test "$1" fuzzer@@@
+  build_step "test "$1" fuzzer"
   ln -sf $LIBFUZZER .
   export FUZZING_ENGINE=fsanitize_fuzzer
   `pwd`/fuzzer-test-suite/build-and-test.sh "$1"