From 48375039857a0f15112226df8ab8361a8049bff8 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 10:19:42 -0700 Subject: [PATCH 1/6] Fix R CI flakiness with Spark download retry logic and timeout --- .github/workflows/r.yml | 1 + R/tests/testthat/helper-initialize.R | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 55951c4036c..65a182f66b5 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -143,6 +143,7 @@ jobs: cd ./R/tests NOT_CRAN='true' Rscript testthat.R shell: bash + timeout-minutes: 30 - uses: actions/upload-artifact@v4 if: failure() with: diff --git a/R/tests/testthat/helper-initialize.R b/R/tests/testthat/helper-initialize.R index 84cadbff017..95260346498 100644 --- a/R/tests/testthat/helper-initialize.R +++ b/R/tests/testthat/helper-initialize.R @@ -22,7 +22,29 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { hadoop_version <- Sys.getenv("HADOOP_VERSION") spark_installed <- spark_installed_versions() if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { - spark_install(version, hadoop_version) + # Install Spark with retry logic to handle download failures + install_attempts <- 3 + install_delay <- 5 + for (attempt in seq(install_attempts)) { + install_success <- tryCatch( + { + spark_install(version, hadoop_version) + TRUE + }, + error = function(e) { + if (attempt < install_attempts) { + message(sprintf("Spark installation attempt %d failed: %s", attempt, e$message)) + message(sprintf("Retrying in %d seconds...", install_delay)) + Sys.sleep(install_delay) + install_delay <<- install_delay * 2 # Exponential backoff + FALSE + } else { + stop(sprintf("Failed to install Spark after %d attempts: %s", install_attempts, e$message)) + } + } + ) + if (install_success) break + } } conn_attempts <- 3 From e14e43ec6de2c8eb95be75b43cc7257312dbf345 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 15:41:13 -0700 Subject: [PATCH 2/6] test the same way python.yml download spark --- .github/workflows/r.yml | 35 +++++++++++++------ R/tests/testthat/helper-initialize.R | 50 +++++++++++++--------------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 65a182f66b5..e75855c64fc 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -109,24 +109,37 @@ jobs: distribution: 'temurin' java-version: '11' cache: 'maven' - - name: Get OS name - id: os-name - run: | - # `os_name` will be like "Ubuntu-20.04.1-LTS" - OS_NAME=$(lsb_release -ds | sed 's/\s/-/g') - echo "os-name=$OS_NAME" >> $GITHUB_OUTPUT - - name: Cache Spark installations - if: runner.os != 'Windows' - uses: actions/cache@master + - uses: actions/setup-python@v5 with: - path: ~/spark - key: apache.sedona-apache-spark-${{ steps.os-name.outputs.os-name }}-${{ env.SPARK_VERSION }} + python-version: '3.11' + - name: Install PySpark (same as python.yml) + run: | + # Follow python.yml approach exactly + pip3 install pyspark==${SPARK_VERSION} + - name: Download JAI libraries (same as python.yml) + run: | + # Follow python.yml JAI download pattern exactly + PYSPARK_PATH=$(python3 -c "import pyspark; print(pyspark.__path__[0])") + wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar + wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar + wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar + mv -v jai_core-${JAI_CORE_VERSION}.jar ${PYSPARK_PATH}/jars + mv -v jai_codec-${JAI_CODEC_VERSION}.jar ${PYSPARK_PATH}/jars + mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar ${PYSPARK_PATH}/jars + echo "PYSPARK_PATH=${PYSPARK_PATH}" >> $GITHUB_ENV - name: Build Sedona libraries run: | SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3} mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools + - name: Copy Sedona JARs to PySpark (same as python.yml) + run: | + # Follow python.yml Sedona JAR copying pattern exactly + find spark-shaded/target -name sedona-*.jar -exec cp {} ${PYSPARK_PATH}/jars/ \; - name: Run tests run: | + # Set SPARK_HOME to PySpark path (same as python.yml pattern) + export SPARK_HOME=${PYSPARK_PATH} + if [[ "${SPARK_VERSION:0:3}" < "3.3" ]]; then case "$HADOOP_VERSION" in 3) diff --git a/R/tests/testthat/helper-initialize.R b/R/tests/testthat/helper-initialize.R index 95260346498..96603273055 100644 --- a/R/tests/testthat/helper-initialize.R +++ b/R/tests/testthat/helper-initialize.R @@ -20,30 +20,18 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { if (!exists(conn_key, envir = .GlobalEnv)) { version <- Sys.getenv("SPARK_VERSION") hadoop_version <- Sys.getenv("HADOOP_VERSION") - spark_installed <- spark_installed_versions() - if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { - # Install Spark with retry logic to handle download failures - install_attempts <- 3 - install_delay <- 5 - for (attempt in seq(install_attempts)) { - install_success <- tryCatch( - { - spark_install(version, hadoop_version) - TRUE - }, - error = function(e) { - if (attempt < install_attempts) { - message(sprintf("Spark installation attempt %d failed: %s", attempt, e$message)) - message(sprintf("Retrying in %d seconds...", install_delay)) - Sys.sleep(install_delay) - install_delay <<- install_delay * 2 # Exponential backoff - FALSE - } else { - stop(sprintf("Failed to install Spark after %d attempts: %s", install_attempts, e$message)) - } - } - ) - if (install_success) break + spark_home <- Sys.getenv("SPARK_HOME") + + # Check if SPARK_HOME is set (from CI workflow) + if (spark_home != "") { + # Use pre-installed Spark from CI + message(sprintf("Using pre-installed Spark from: %s", spark_home)) + } else { + # Local development: install Spark if needed + spark_installed <- spark_installed_versions() + if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { + message("Installing Spark for local development...") + spark_install(version, hadoop_version) } } @@ -55,13 +43,21 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { config <- spark_config() config[["sparklyr.connect.timeout"]] <- 300 - sc <- spark_connect( + # Use SPARK_HOME if set, otherwise use version + connect_args <- list( master = "local", method = "shell", config = config, - app_name = paste0("testthat-", uuid::UUIDgenerate()), - version = version + app_name = paste0("testthat-", uuid::UUIDgenerate()) ) + + if (spark_home != "") { + connect_args$spark_home <- spark_home + } else { + connect_args$version <- version + } + + sc <- do.call(spark_connect, connect_args) assign(conn_key, sc, envir = .GlobalEnv) TRUE }, From 9b3a45b1f28d409feda65a2e8eff9b991bb938f9 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 15:49:12 -0700 Subject: [PATCH 3/6] add jai versions env --- .github/workflows/r.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index e75855c64fc..44a8134ac07 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -41,6 +41,9 @@ on: env: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=60 + JAI_CORE_VERSION: '1.1.3' + JAI_CODEC_VERSION: '1.1.3' + JAI_IMAGEIO_VERSION: '1.1' DO_NOT_TRACK: true concurrency: From 7109a825882737653373e57ccb88f95dbb94ad25 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 15:51:33 -0700 Subject: [PATCH 4/6] revert r install script --- R/tests/testthat/helper-initialize.R | 30 ++++++---------------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/R/tests/testthat/helper-initialize.R b/R/tests/testthat/helper-initialize.R index 96603273055..84cadbff017 100644 --- a/R/tests/testthat/helper-initialize.R +++ b/R/tests/testthat/helper-initialize.R @@ -20,19 +20,9 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { if (!exists(conn_key, envir = .GlobalEnv)) { version <- Sys.getenv("SPARK_VERSION") hadoop_version <- Sys.getenv("HADOOP_VERSION") - spark_home <- Sys.getenv("SPARK_HOME") - - # Check if SPARK_HOME is set (from CI workflow) - if (spark_home != "") { - # Use pre-installed Spark from CI - message(sprintf("Using pre-installed Spark from: %s", spark_home)) - } else { - # Local development: install Spark if needed - spark_installed <- spark_installed_versions() - if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { - message("Installing Spark for local development...") - spark_install(version, hadoop_version) - } + spark_installed <- spark_installed_versions() + if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { + spark_install(version, hadoop_version) } conn_attempts <- 3 @@ -43,21 +33,13 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { config <- spark_config() config[["sparklyr.connect.timeout"]] <- 300 - # Use SPARK_HOME if set, otherwise use version - connect_args <- list( + sc <- spark_connect( master = "local", method = "shell", config = config, - app_name = paste0("testthat-", uuid::UUIDgenerate()) + app_name = paste0("testthat-", uuid::UUIDgenerate()), + version = version ) - - if (spark_home != "") { - connect_args$spark_home <- spark_home - } else { - connect_args$version <- version - } - - sc <- do.call(spark_connect, connect_args) assign(conn_key, sc, envir = .GlobalEnv) TRUE }, From c568f5dbc6f053efa8afff8e10cc7ddd8c3f21f2 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 16:05:37 -0700 Subject: [PATCH 5/6] update R helper to check SPARK_HOME before downloading spark from apache archive --- R/tests/testthat/helper-initialize.R | 30 ++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/R/tests/testthat/helper-initialize.R b/R/tests/testthat/helper-initialize.R index 84cadbff017..41ec2e19e77 100644 --- a/R/tests/testthat/helper-initialize.R +++ b/R/tests/testthat/helper-initialize.R @@ -20,9 +20,19 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { if (!exists(conn_key, envir = .GlobalEnv)) { version <- Sys.getenv("SPARK_VERSION") hadoop_version <- Sys.getenv("HADOOP_VERSION") - spark_installed <- spark_installed_versions() - if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { - spark_install(version, hadoop_version) + spark_home <- Sys.getenv("SPARK_HOME") + + # Use pre-installed Spark from CI (via PySpark) + if (spark_home != "") { + message(sprintf("Using pre-installed Spark from: %s", spark_home)) + # No need to download - Spark is already available via PySpark + } else { + # For local development only (not CI) + spark_installed <- spark_installed_versions() + if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { + message("Installing Spark for local development...") + spark_install(version, hadoop_version) + } } conn_attempts <- 3 @@ -33,13 +43,21 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { config <- spark_config() config[["sparklyr.connect.timeout"]] <- 300 - sc <- spark_connect( + # Use spark_home if set (CI), otherwise use version (local dev) + connect_args <- list( master = "local", method = "shell", config = config, - app_name = paste0("testthat-", uuid::UUIDgenerate()), - version = version + app_name = paste0("testthat-", uuid::UUIDgenerate()) ) + + if (spark_home != "") { + connect_args$spark_home <- spark_home + } else { + connect_args$version <- version + } + + sc <- do.call(spark_connect, connect_args) assign(conn_key, sc, envir = .GlobalEnv) TRUE }, From 2f2c0f5c1523bb98d8eda152195433e98a2861f7 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Tue, 16 Sep 2025 16:09:47 -0700 Subject: [PATCH 6/6] clean up comments --- .github/workflows/r.yml | 11 ++++------- R/tests/testthat/helper-initialize.R | 3 --- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 44a8134ac07..236b30e7e35 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -115,13 +115,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install PySpark (same as python.yml) + - name: Install PySpark run: | - # Follow python.yml approach exactly pip3 install pyspark==${SPARK_VERSION} - - name: Download JAI libraries (same as python.yml) + - name: Download JAI libraries run: | - # Follow python.yml JAI download pattern exactly PYSPARK_PATH=$(python3 -c "import pyspark; print(pyspark.__path__[0])") wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar @@ -134,13 +132,12 @@ jobs: run: | SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3} mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools - - name: Copy Sedona JARs to PySpark (same as python.yml) + - name: Copy Sedona JARs to PySpark run: | - # Follow python.yml Sedona JAR copying pattern exactly find spark-shaded/target -name sedona-*.jar -exec cp {} ${PYSPARK_PATH}/jars/ \; - name: Run tests run: | - # Set SPARK_HOME to PySpark path (same as python.yml pattern) + # Set SPARK_HOME to PySpark path export SPARK_HOME=${PYSPARK_PATH} if [[ "${SPARK_VERSION:0:3}" < "3.3" ]]; then diff --git a/R/tests/testthat/helper-initialize.R b/R/tests/testthat/helper-initialize.R index 41ec2e19e77..9d1143d93f6 100644 --- a/R/tests/testthat/helper-initialize.R +++ b/R/tests/testthat/helper-initialize.R @@ -22,12 +22,9 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) { hadoop_version <- Sys.getenv("HADOOP_VERSION") spark_home <- Sys.getenv("SPARK_HOME") - # Use pre-installed Spark from CI (via PySpark) if (spark_home != "") { message(sprintf("Using pre-installed Spark from: %s", spark_home)) - # No need to download - Spark is already available via PySpark } else { - # For local development only (not CI) spark_installed <- spark_installed_versions() if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop == hadoop_version, ]) == 0) { message("Installing Spark for local development...")