Improve temporary error handling in package building process

dev-tools/build-packages/base-packages-to-base/base-packages.Dockerfile

@@ -21,14 +21,16 @@ ARG WAZUH_DASHBOARD_REPORTING_BRANCH
 ENV OPENSEARCH_DASHBOARDS_VERSION=3.2.0
 ENV ENV_ARCHITECTURE=${ARCHITECTURE}
 USER root
-RUN apt-get update && apt-get install -y jq
+RUN apt-get update && apt-get install -y jq && mkdir -p /usr/local/lib/wazuh
+ADD ./common/run-with-retry.sh /usr/local/lib/wazuh/run-with-retry.sh
+RUN chown -R node:node /usr/local/lib/wazuh
 USER node
-ADD ./clone-plugins.sh /home/node/clone-plugins.sh
-ADD ./repositories/wazuh-dashboard.sh /home/node/repositories/wazuh-dashboard.sh
-ADD ./repositories/plugins/wazuh-dashboard-security-analytics.sh /home/node/repositories/plugins/wazuh-dashboard-security-analytics.sh
-ADD ./repositories/plugins/wazuh-security-dashboards-plugin.sh /home/node/repositories/plugins/wazuh-security-dashboards-plugin.sh
-ADD ./repositories/plugins/wazuh-dashboard-reporting.sh /home/node/repositories/plugins/wazuh-dashboard-reporting.sh
-ADD ./repositories/plugins/wazuh-dashboard-plugins.sh /home/node/repositories/plugins/wazuh-dashboard-plugins.sh
+ADD ./base-packages-to-base/clone-plugins.sh /home/node/clone-plugins.sh
+ADD ./base-packages-to-base/repositories/wazuh-dashboard.sh /home/node/repositories/wazuh-dashboard.sh
+ADD ./base-packages-to-base/repositories/plugins/wazuh-dashboard-security-analytics.sh /home/node/repositories/plugins/wazuh-dashboard-security-analytics.sh
+ADD ./base-packages-to-base/repositories/plugins/wazuh-security-dashboards-plugin.sh /home/node/repositories/plugins/wazuh-security-dashboards-plugin.sh
+ADD ./base-packages-to-base/repositories/plugins/wazuh-dashboard-reporting.sh /home/node/repositories/plugins/wazuh-dashboard-reporting.sh
+ADD ./base-packages-to-base/repositories/plugins/wazuh-dashboard-plugins.sh /home/node/repositories/plugins/wazuh-dashboard-plugins.sh
 RUN bash /home/node/clone-plugins.sh
 
 FROM node:${NODE_VERSION}

dev-tools/build-packages/base-packages-to-base/clone-plugins.sh

@@ -1,16 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 base_path_plugins="/home/node/app/plugins"
 base_path_repositories_scripts="/home/node/repositories"
 base_path_repositories_plugins_scripts="/home/node/repositories/plugins"
 plugins=$(ls $base_path_repositories_plugins_scripts)
 
-mkdir /home/node/packages
+mkdir -p /home/node/packages
 echo "Cloning Wazuh dashboard"
 source $base_path_repositories_scripts/wazuh-dashboard.sh
 
 for plugin in $plugins; do
   echo "Cloning $plugin"
   source $base_path_repositories_plugins_scripts/$plugin
 done
-
-
-

dev-tools/build-packages/base-packages-to-base/docker-compose.yml

@@ -2,8 +2,8 @@ services:
   base:
     image: base-${WAZUH_DASHBOARD_BRANCH}-${WAZUH_SECURITY_PLUGIN_BRANCH}-${WAZUH_DASHBOARD_PLUGINS_BRANCH}-${WAZUH_DASHBOARD_REPORTING_BRANCH}-${WAZUH_DASHBOARD_SECURITY_ANALYTICS_BRANCH}-${ARCHITECTURE}
     build:
-      context: .
-      dockerfile: base-packages.Dockerfile
+      context: ..
+      dockerfile: base-packages-to-base/base-packages.Dockerfile
       args:
         NODE_VERSION: ${NODE_VERSION}
         WAZUH_DASHBOARD_BRANCH: ${WAZUH_DASHBOARD_BRANCH}

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-dashboard-plugins.sh

@@ -1,14 +1,16 @@
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 base_path_plugins="/home/node/app/plugins"
 cd $base_path_plugins
-git clone --depth 1 --branch ${WAZUH_DASHBOARD_PLUGINS_BRANCH} https://github.com/wazuh/wazuh-dashboard-plugins.git
+run_with_retry git clone --depth 1 --branch ${WAZUH_DASHBOARD_PLUGINS_BRANCH} https://github.com/wazuh/wazuh-dashboard-plugins.git
 wazuh_dashboard_plugins=$(ls $base_path_plugins/wazuh-dashboard-plugins/plugins)
 mv wazuh-dashboard-plugins/plugins/* ./
 mkdir /home/node/packages/wazuh-dashboard-plugins
 for wazuh_dashboard_plugin in $wazuh_dashboard_plugins; do
   cd $base_path_plugins/$wazuh_dashboard_plugin
-  GIT_REF="${WAZUH_DASHBOARD_PLUGINS_BRANCH}" yarn install
+  run_with_retry env GIT_REF="${WAZUH_DASHBOARD_PLUGINS_BRANCH}" yarn install
   echo "Building $wazuh_dashboard_plugin"
-  yarn build
+  run_with_retry yarn build
   echo "Copying $wazuh_dashboard_plugin"
   package_name=$(jq -r '.id' ./opensearch_dashboards.json)
   cp $base_path_plugins/$wazuh_dashboard_plugin/build/$package_name-$OPENSEARCH_DASHBOARDS_VERSION.zip /home/node/packages/wazuh-dashboard-plugins/$package_name-$OPENSEARCH_DASHBOARDS_VERSION.zip

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-dashboard-reporting.sh

@@ -1,10 +1,12 @@
 # Clone the Wazuh security plugin
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 cd /home/node/app/plugins
-git clone --depth 1 --branch ${WAZUH_DASHBOARD_REPORTING_BRANCH} https://github.com/wazuh/wazuh-dashboards-reporting.git
+run_with_retry git clone --depth 1 --branch ${WAZUH_DASHBOARD_REPORTING_BRANCH} https://github.com/wazuh/wazuh-dashboards-reporting.git
 cd wazuh-dashboards-reporting
-yarn install
+run_with_retry yarn install
 echo "Building Wazuh reporting plugin"
-yarn build
+run_with_retry yarn build
 echo "Copying Wazuh reporting plugin"
 mkdir /home/node/packages/wazuh-dashboards-reporting
 cp -r build/* /home/node/packages/wazuh-dashboards-reporting

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-dashboard-security-analytics.sh

@@ -1,10 +1,12 @@
 # Clone the Wazuh Security Analytics plugin
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 cd /home/node/app/plugins
-git clone --depth 1 --branch ${WAZUH_DASHBOARD_SECURITY_ANALYTICS_BRANCH} https://github.com/wazuh/wazuh-dashboard-security-analytics.git
+run_with_retry git clone --depth 1 --branch ${WAZUH_DASHBOARD_SECURITY_ANALYTICS_BRANCH} https://github.com/wazuh/wazuh-dashboard-security-analytics.git
 cd wazuh-dashboard-security-analytics
-yarn install
+run_with_retry yarn install
 echo "Building Wazuh Security Analytics plugin"
-yarn build
+run_with_retry yarn build
 echo "Copying Wazuh Security Analytics plugin"
 mkdir /home/node/packages/wazuh-security-analytics-plugin
-cp -r build/* /home/node/packages/wazuh-security-analytics-plugin
+cp -r build/* /home/node/packages/wazuh-security-analytics-plugin

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-security-dashboards-plugin.sh

@@ -1,10 +1,12 @@
 # Clone the Wazuh security plugin
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 cd /home/node/app/plugins
-git clone --depth 1 --branch ${WAZUH_DASHBOARD_SECURITY_BRANCH} https://github.com/wazuh/wazuh-security-dashboards-plugin.git
+run_with_retry git clone --depth 1 --branch ${WAZUH_DASHBOARD_SECURITY_BRANCH} https://github.com/wazuh/wazuh-security-dashboards-plugin.git
 cd wazuh-security-dashboards-plugin
-yarn install
+run_with_retry yarn install
 echo "Building Wazuh security plugin"
-yarn build
+run_with_retry yarn build
 echo "Copying Wazuh security plugin"
 mkdir /home/node/packages/wazuh-security-dashboards-plugin
 cp -r build/* /home/node/packages/wazuh-security-dashboards-plugin

dev-tools/build-packages/base-packages-to-base/repositories/wazuh-dashboard.sh

@@ -1,11 +1,13 @@
-git clone --depth 1 --branch ${WAZUH_DASHBOARD_BRANCH} https://github.com/wazuh/wazuh-dashboard.git /home/node/app
+source /usr/local/lib/wazuh/run-with-retry.sh
+
+run_with_retry git clone --depth 1 --branch ${WAZUH_DASHBOARD_BRANCH} https://github.com/wazuh/wazuh-dashboard.git /home/node/app
 cd /home/node/app
-yarn osd bootstrap --production
+run_with_retry yarn osd bootstrap --production
 echo "Building Wazuh dashboard"
 if [ $ENV_ARCHITECTURE == "arm" ]; then
-  yarn build-platform --linux-arm --skip-os-packages --release
+  run_with_retry yarn build-platform --linux-arm --skip-os-packages --release
 else
-  yarn build-platform --linux --skip-os-packages --release
+  run_with_retry yarn build-platform --linux --skip-os-packages --release
 fi
 mkdir /home/node/packages/wazuh-dashboard
 echo "Copying Wazuh dashboard"

dev-tools/build-packages/base/Docker/Dockerfile

@@ -26,10 +26,13 @@ RUN git clone https://github.com/google/brotli.git -b v1.0.9
 RUN cd brotli && chmod +x ./bootstrap && ./bootstrap && ./configure --prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --libexecdir=/usr/lib64/brotli --libdir=/usr/lib64/brotli --datarootdir=/usr/share --mandir=/usr/share/man/man1 --docdir=/usr/share/doc \
     && make && make install
 
+RUN mkdir -p /usr/local/lib/wazuh
+
 # Add the scripts to build the RPM package
 ADD base-builder.sh /usr/local/bin/builder
 ADD VERSION.json /usr/local/bin/VERSION.json
 ADD plugins /usr/local/bin/plugins
+ADD run-with-retry.sh /usr/local/lib/wazuh/run-with-retry.sh
 RUN chmod +x /usr/local/bin/builder
 
 # Set the entrypoint

dev-tools/build-packages/base/base-builder.sh

@@ -16,6 +16,8 @@ revision="$2"
 architecture="$3"
 verbose="$4"
 
+source /usr/local/lib/wazuh/run-with-retry.sh
+
 if [ "$verbose" = "debug" ]; then
   set -x
 fi
@@ -92,7 +94,7 @@ for plugin in $plugins; do
     install=$plugin
   fi
   log "Installing ${plugin} plugin"
-  if ! bin/opensearch-dashboards-plugin install $install --allow-root 2>&1 >/dev/null; then
+  if ! run_with_retry bin/opensearch-dashboards-plugin install "${install}" --allow-root >/dev/null; then
     echo "Plugin ${plugin} installation failed"
     exit 1
   fi

dev-tools/build-packages/build-packages.sh

@@ -16,15 +16,19 @@ rpm="no"
 tar="no"
 architecture="x64"
 production="no"
-retry_max_attempts=3
-retry_delay_seconds=15
 commit_sha=$(git rev-parse --short HEAD)
 output_dir="${current_path}/output"
 tmp_dir="${current_path}/tmp"
 config_dir="${root_dir}/config"
 package_config_dir="${current_path}/config"
 verbose="info"
 
+RETRY_MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-3}"
+RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-15}"
+export RETRY_MAX_ATTEMPTS RETRY_DELAY_SECONDS
+
+source "${current_path}/common/run-with-retry.sh"
+
 trap clean INT
 trap clean EXIT
 
@@ -34,35 +38,6 @@ log() {
     fi
 }
 
-run_with_retry() {
-    local attempt=1
-    local exit_code=0
-
-    while [ "${attempt}" -le "${retry_max_attempts}" ]; do
-        "$@" # Run the command
-        exit_code=$?
-        if [ ${exit_code} -eq 0 ]; then
-            return 0
-        fi
-
-        if [ "${attempt}" -ge "${retry_max_attempts}" ]; then
-            echo "Command failed after ${retry_max_attempts} attempts (exit code ${exit_code}): $*" >&2
-            return ${exit_code}
-        fi
-
-        local next_attempt=$((attempt + 1))
-        if [ "$verbose" != "silent" ]; then
-            echo "Command failed (attempt ${attempt}/${retry_max_attempts}, exit ${exit_code}). Retrying in ${retry_delay_seconds}s (attempt ${next_attempt}/${retry_max_attempts})." >&2
-        fi
-
-        if [ "${retry_delay_seconds}" -gt 0 ]; then
-            sleep "${retry_delay_seconds}"
-        fi
-
-        attempt=$((attempt + 1))
-    done
-}
-
 clean() {
     exit_code=$?
     echo
@@ -72,6 +47,7 @@ clean() {
     rm -rf ${tmp_dir}
     rm -f ${current_path}/base/Docker/base-builder.sh
     rm -f ${current_path}/base/Docker/plugins
+    rm -f ${current_path}/base/Docker/run-with-retry.sh
     rm -f ${current_path}/rpm/Docker/rpm-builder.sh
     rm -f ${current_path}/rpm/Docker/wazuh-dashboard.spec
     rm -f ${current_path}/deb/Docker/deb-builder.sh
@@ -101,7 +77,7 @@ get_packages(){
 
     if [[ $package_url =~ $valid_url ]]; then
       if ! run_with_retry curl --output "packages/${package_var}.zip" --silent --show-error --fail "${package_url}"; then
-        echo "Failed to download ${package_name} after ${retry_max_attempts} attempts: ${package_url}"
+        echo "Failed to download ${package_name} after ${RETRY_MAX_ATTEMPTS} attempts: ${package_url}"
         clean 1
       fi
     else
@@ -128,8 +104,12 @@ build_tar() {
   cp ./base-builder.sh ${dockerfile_path}
   cp ./plugins ${dockerfile_path}
   cp ${root_dir}/VERSION.json ${dockerfile_path}
+  cp ${current_path}/common/run-with-retry.sh ${dockerfile_path}
   run_with_retry docker build -t "${container_name}" "${dockerfile_path}" || return 1
-  run_with_retry docker run -t --rm -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
+  run_with_retry docker run -t --rm \
+    -e "RETRY_MAX_ATTEMPTS=${RETRY_MAX_ATTEMPTS}" \
+    -e "RETRY_DELAY_SECONDS=${RETRY_DELAY_SECONDS}" \
+    -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
     "${container_name}" "${version}" "${revision}" "${architecture}" "${verbose}" || return 1
   cd ..
 }
@@ -143,7 +123,10 @@ build_rpm() {
   cp ./rpm-builder.sh ${dockerfile_path}
   cp ./wazuh-dashboard.spec ${dockerfile_path}
   run_with_retry docker build -t "${container_name}" "${dockerfile_path}" || return 1
-  run_with_retry docker run -t --rm -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
+  run_with_retry docker run -t --rm \
+    -e "RETRY_MAX_ATTEMPTS=${RETRY_MAX_ATTEMPTS}" \
+    -e "RETRY_DELAY_SECONDS=${RETRY_DELAY_SECONDS}" \
+    -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
     "${container_name}" "${version}" "${revision}" "${architecture}" \
     "${commit_sha}" "${production}" "${verbose}" || return 1
   cd ../
@@ -159,7 +142,10 @@ build_deb() {
   cp ./deb-builder.sh ${dockerfile_path}
   cp -r ./debian ${dockerfile_path}
   run_with_retry docker build -t "${container_name}" "${dockerfile_path}" || return 1
-  run_with_retry docker run -t --rm -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
+  run_with_retry docker run -t --rm \
+    -e "RETRY_MAX_ATTEMPTS=${RETRY_MAX_ATTEMPTS}" \
+    -e "RETRY_DELAY_SECONDS=${RETRY_DELAY_SECONDS}" \
+    -v "${tmp_dir}/:/tmp:Z" -v "${output_dir}/:/output:Z" \
     "${container_name}" "${version}" "${revision}" "${architecture}" \
     "${commit_sha}" "${production}" "${verbose}" || return 1
   cd ..
@@ -309,7 +295,7 @@ main() {
             ;;
         "--retry-attempts")
             if [ -n "${2}" ] && [[ "${2}" =~ ^[0-9]+$ ]] && [ "${2}" -gt 0 ]; then
-                retry_max_attempts="${2}"
+                RETRY_MAX_ATTEMPTS="${2}"
                 shift 2
             else
                 echo "Invalid value for --retry-attempts. It must be a positive integer."
@@ -318,7 +304,7 @@ main() {
             ;;
         "--retry-delay")
             if [ -n "${2}" ] && [[ "${2}" =~ ^[0-9]+$ ]]; then
-                retry_delay_seconds="${2}"
+                RETRY_DELAY_SECONDS="${2}"
                 shift 2
             else
                 echo "Invalid value for --retry-delay. It must be a non-negative integer."

dev-tools/build-packages/common/run-with-retry.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Shared retry helper for shell scripts that need to tolerate transient failures.
+#
+# Environment variables:
+#   RETRY_MAX_ATTEMPTS - number of times to retry (default: 3)
+#   RETRY_DELAY_SECONDS - seconds to wait between retries (default: 15)
+
+if ! declare -f run_with_retry >/dev/null 2>&1; then
+  run_with_retry() {
+    local max_attempts=${RETRY_MAX_ATTEMPTS:-3}
+    local delay=${RETRY_DELAY_SECONDS:-15}
+    local attempt=1
+    local exit_code=0
+
+    while [ "${attempt}" -le "${max_attempts}" ]; do
+      "$@"
+      exit_code=$?
+      if [ ${exit_code} -eq 0 ]; then
+        return 0
+      fi
+
+      if [ "${attempt}" -ge "${max_attempts}" ]; then
+        echo "Command failed after ${max_attempts} attempts (exit code ${exit_code}): $*" >&2
+        return ${exit_code}
+      fi
+
+      if [ "${delay}" -gt 0 ]; then
+        echo "Command failed (attempt ${attempt}/${max_attempts}, exit ${exit_code}). Retrying in ${delay}s..." >&2
+        sleep "${delay}"
+      else
+        echo "Command failed (attempt ${attempt}/${max_attempts}, exit ${exit_code}). Retrying..." >&2
+      fi
+
+      attempt=$((attempt + 1))
+    done
+  }
+fi