testing fix

Author: cjac

cloudbuild/Dockerfile

@@ -30,3 +30,7 @@ RUN update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-${bazel_ve
     update-alternatives                    --set bazel /usr/bin/bazel-${bazel_version}
 
 USER ia-tests
+
+# Install Python dependencies
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install -r /init-actions/requirements.txt

gpu/Dockerfile

@@ -40,6 +40,7 @@ RUN apt-get -y -qq install emacs-nox vim uuid-runtime > /dev/null 2>&1 && apt-ge
 WORKDIR /init-actions
 
 USER ia-tests
+COPY --chown=ia-tests:ia-tests "cloudbuild/key.json" /key.json
 COPY --chown=ia-tests:ia-tests . ${WORKDIR}
 
 ENTRYPOINT ["/bin/bash"]

gpu/install_gpu_driver.sh

@@ -2637,50 +2637,6 @@ function cache_fetched_package() {
   fi
 }
 
-function fix_nodemanager_init_script() {
-  local lsb_script="/etc/init.d/hadoop-yarn-nodemanager"
-  local service_name="hadoop-yarn-nodemanager"
-  local generated_unit_file="/run/systemd/generator.late/${service_name}.service"
-  local changed=0
-
-  if [[ ! -f "${lsb_script}" ]]; then
-    echo "WARN: LSB script ${lsb_script} not found."
-    return
-  fi
-
-  # 1. Fix the stop command
-  local broken_stop_cmd='start_daemon $EXEC_PATH --config "$CONF_DIR" stop $DAEMON_FLAGS'
-  local fixed_stop_cmd='start_daemon $EXEC_PATH --config "$CONF_DIR" --daemon stop $DAEMON_FLAGS'
-
-  if grep -qF "${broken_stop_cmd}" "${lsb_script}"; then
-    echo "Fixing stop command in ${lsb_script}..."
-    local sed_broken_stop_cmd=$(printf '%s\\n' "${broken_stop_cmd}" | sed 's/[][\\/.^$*]/\\\\&/g')
-    local sed_fixed_stop_cmd=$(printf '%s\\n' "${fixed_cmd}" | sed 's/[][\\/.^$*]/\\\\&/g')
-    sed -i "s|${sed_broken_stop_cmd}|${sed_fixed_stop_cmd}|" "${lsb_script}"
-    changed=1
-  fi
-
-  # 2. Prepend source commands to the 'su -c' line in the start function
-  local start_line_marker='--daemon start $DAEMON_FLAGS'
-  if grep -qF "${start_line_marker}" "${lsb_script}" && ! grep -qF "source /etc/environment && source /etc/default/hadoop-yarn-nodemanager" "${lsb_script}"; then
-    echo "Adding source commands to su -c in start() for ${lsb_script}"
-    sed -i '/su -s \/bin\/bash yarn -c "/s|yarn -c "|yarn -c "source /etc/environment && source /etc/default/hadoop-yarn-nodemanager && |' "${lsb_script}"
-    changed=1
-  fi
-
-  if [[ "${changed}" -eq 1 ]]; then
-    if [[ -f "${generated_unit_file}" ]]; then
-      echo "Removing old generated unit file: ${generated_unit_file}"
-      rm -f "${generated_unit_file}"
-    fi
-    echo "Reloading systemd daemon to regenerate units..."
-    systemctl daemon-reload
-    echo "Systemd daemon reloaded."
-  else
-    echo "No changes made to ${lsb_script}."
-  fi
-}
-
 function clean_up_sources_lists() {
   if ! is_debuntu; then return; fi
   #

integration_tests/dataproc_test_case.py

@@ -9,7 +9,7 @@
 import sys
 from threading import Timer
 
-from packaging import version
+import pkg_resources
 from absl import flags
 from absl.testing import parameterized
 
@@ -252,7 +252,7 @@ def getImageVersion():
         # Special case a 'preview' image versions and return a large number
         # instead to make it a higher image version in comparisons
         version = FLAGS.image_version
-        return version.parse('999') if version.startswith(
+        return pkg_resources.parse_version('999') if version.startswith(
             'preview') else pkg_resources.parse_version(version.split('-')[0])
 
     @staticmethod
@@ -307,21 +307,23 @@ def assert_dataproc_job(self,
 
         Args:
             cluster_name: cluster name to submit job to
-            job_type: type of the job, e.g. spark, hadoop, pyspark
-            job_params: job parameters
-            timeout_in_minutes: timeout in minutes
+            job_type: job type (hadoop, spark, etc)
+            job_params: job command parameters
+            timeout_in_minutes: timeout in minutes after which process that
+                                waits on job will be killed if job did not
+                                finish
+        Returns:
+            ret_code: the return code of the job
+            stdout: standard output of the job
+            stderr: error output of the job
         Raises:
             AssertionError: if job returned non-0 exit code.
         """
 
-        ret_code, stdout, stderr = DataprocTestCase.run_command(
+        ret_code, stdout, stderr = self.assert_command(
             'gcloud dataproc jobs submit {} --cluster={} --region={} {}'.
             format(job_type, cluster_name, self.REGION,
                    job_params), timeout_in_minutes)
-        self.assertEqual(
-            ret_code, 0,
-            "Job failed with code {}. stdout: {}. stderr: {}".format(
-                ret_code, stdout, stderr))
         return ret_code, stdout, stderr
 
     def assert_command(self, cmd, timeout_in_minutes=DEFAULT_TIMEOUT):

requirements.txt

@@ -0,0 +1,4 @@
+bootstrapping==0.1.2
+click==8.1.7
+packaging
+setuptools<70.0.0