From 17abc40899e86fe190ccdc6bfde178c9d4b3d513 Mon Sep 17 00:00:00 2001 From: Luis Perez Date: Mon, 9 Sep 2024 14:11:39 -0700 Subject: [PATCH 1/2] Stop checking if volume to mount in k8s exists locally It's entirely possible (and not an error) to create a podspec that contains paths to volume-mount that do not exist wherever service_configuration_lib is being invoked from. As long as we ensure that we don't double-add things (which is only a problem due to the way that the env var keys are constructed here), skipping the existence check should be fine. --- service_configuration_lib/spark_config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 5c11293..108f620 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -223,13 +223,12 @@ def _get_k8s_docker_volumes_conf( for volume in k8s_volumes: host_path, container_path, mode = volume['hostPath'], volume['containerPath'], volume['mode'] - if os.path.exists(host_path) and host_path not in mounted_volumes: + if host_path not in mounted_volumes: env.update(_get_k8s_volume(host_path, container_path, mode)) mounted_volumes.add(host_path) else: log.warning( - f'Path {host_path} does not exist on this host or it has already been mounted.' - ' Skipping this bindings.', + f'Path {host_path} has already been mounted. Skipping this binding.', ) return env From 85e8e3899358823f9e40fcf3028123aed61f5e00 Mon Sep 17 00:00:00 2001 From: Luis Perez Date: Mon, 9 Sep 2024 14:24:45 -0700 Subject: [PATCH 2/2] Remove another exists check --- tests/spark_config_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/spark_config_test.py b/tests/spark_config_test.py index 60150ed..6bb456e 100644 --- a/tests/spark_config_test.py +++ b/tests/spark_config_test.py @@ -1216,7 +1216,6 @@ def _get_k8s_base_volumes(self): """Helper needed to allow tests to pass in github CI checks.""" return [ volume for volume in spark_config.K8S_BASE_VOLUMES - if os.path.exists(volume['containerPath']) ] @pytest.fixture