Fix checksum flag and tests

Author: ashwiniitm

openml/_api_calls.py

@@ -344,6 +344,7 @@ def __read_url(
         url=url,
         data=data,
         md5_checksum=md5_checksum,
+        check_digest=config.check_digest,
     )
 
 
@@ -356,12 +357,14 @@ def __is_checksum_equal(downloaded_file_binary: bytes, md5_checksum: str | None
     return md5_checksum == md5_checksum_download
 
 
-def _send_request(  # noqa: C901, PLR0912
+def _send_request(  # noqa: C901, PLR0912, PLR0913
     request_method: str,
     url: str,
     data: DATA_TYPE,
+    *,
     files: FILE_ELEMENTS_TYPE | None = None,
     md5_checksum: str | None = None,
+    check_digest: bool = True,
 ) -> requests.Response:
     n_retries = max(1, config.connection_n_retries)
 
@@ -386,16 +389,18 @@ def _send_request(  # noqa: C901, PLR0912
 
                 __check_response(response=response, url=url, file_elements=files)
 
-                if request_method == "get" and not __is_checksum_equal(
-                    response.text.encode("utf-8"), md5_checksum
+                if (
+                    request_method == "get"
+                    and check_digest
+                    and not __is_checksum_equal(response.text.encode("utf-8"), md5_checksum)
                 ):
                     # -- Check if encoding is not UTF-8 perhaps
                     if __is_checksum_equal(response.content, md5_checksum):
                         raise OpenMLHashException(
-                            f"Checksum of downloaded file is unequal to the expected checksum"
+                            "Checksum of downloaded file is unequal to the expected checksum "
                             f"{md5_checksum} because the text encoding is not UTF-8 when "
-                            f"downloading {url}. There might be a sever-sided issue with the file, "
-                            "see: https://github.com/openml/openml-python/issues/1180.",
+                            f"downloading {url}. There might be a server-sided issue with the "
+                            "file, see issue #1180."
                         )
 
                     raise OpenMLHashException(

openml/config.py

@@ -34,6 +34,7 @@ class _Config(TypedDict):
     retry_policy: Literal["human", "robot"]
     connection_n_retries: int
     show_progress: bool
+    check_digest: bool
 
 
 def _create_log_handlers(create_file_handler: bool = True) -> None:  # noqa: FBT001, FBT002
@@ -154,6 +155,7 @@ def _resolve_default_cache_dir() -> Path:
     "retry_policy": "human",
     "connection_n_retries": 5,
     "show_progress": False,
+    "check_digest": True,  # Whether to check the md5 checksum of downloaded files
 }
 
 # Default values are actually added here in the _setup() function which is
@@ -183,6 +185,7 @@ def get_server_base_url() -> str:
 
 retry_policy: Literal["human", "robot"] = _defaults["retry_policy"]
 connection_n_retries: int = _defaults["connection_n_retries"]
+check_digest: bool = _defaults["check_digest"]
 
 
 def set_retry_policy(value: Literal["human", "robot"], n_retries: int | None = None) -> None:
@@ -340,6 +343,7 @@ def _setup(config: _Config | None = None) -> None:
     global _root_cache_directory  # noqa: PLW0603
     global avoid_duplicate_runs  # noqa: PLW0603
     global show_progress  # noqa: PLW0603
+    global check_digest  # noqa: PLW0603
 
     config_file = determine_config_file_path()
     config_dir = config_file.parent
@@ -361,6 +365,7 @@ def _setup(config: _Config | None = None) -> None:
     apikey = config["apikey"]
     server = config["server"]
     show_progress = config["show_progress"]
+    check_digest = config["check_digest"]
     n_retries = int(config["connection_n_retries"])
 
     set_retry_policy(config["retry_policy"], n_retries)
@@ -427,7 +432,7 @@ def _parse_config(config_file: str | Path) -> _Config:
     config_file_.seek(0)
     config.read_file(config_file_)
     configuration = dict(config.items("FAKE_SECTION"))
-    for boolean_field in ["avoid_duplicate_runs", "show_progress"]:
+    for boolean_field in ["avoid_duplicate_runs", "show_progress", "check_digest"]:
         if isinstance(config["FAKE_SECTION"][boolean_field], str):
             configuration[boolean_field] = config["FAKE_SECTION"].getboolean(boolean_field)  # type: ignore
     return configuration  # type: ignore
@@ -442,6 +447,7 @@ def get_config_as_dict() -> _Config:
         "connection_n_retries": connection_n_retries,
         "retry_policy": retry_policy,
         "show_progress": show_progress,
+        "check_digest": check_digest,
     }
 
 

tests/test_utils/test_checksum.py

@@ -0,0 +1,57 @@
+import pytest
+import openml
+from unittest.mock import patch, Mock
+from openml.exceptions import OpenMLHashException
+
+
+def _mock_response():
+    mock_response = Mock()
+    mock_response.text = "hello"
+    mock_response.content = b"hello"
+    mock_response.status_code = 200
+    mock_response.headers = {"Content-Encoding": "gzip"}  # Required by headers check
+    return mock_response
+
+
+@patch("requests.Session")
+def test_checksum_match(Session_class_mock):
+    Session_class_mock.return_value.__enter__.return_value.get.return_value = _mock_response()
+
+    with openml.config.overwrite_config_context({"connection_n_retries": 1}): # to avoid retry delays
+        openml._api_calls._send_request(
+            request_method="get",
+            url="/dummy",
+            data={},
+            md5_checksum="5d41402abc4b2a76b9719d911017c592",
+        )
+
+
+@patch("requests.Session")
+def test_checksum_mismatch(Session_class_mock):
+    Session_class_mock.return_value.__enter__.return_value.get.return_value = _mock_response()
+
+    with openml.config.overwrite_config_context({"connection_n_retries": 1}): # to avoid retry delays
+        with pytest.raises(OpenMLHashException):
+            openml._api_calls._send_request(
+                request_method="get",
+                url="/dummy",
+                data={},
+                md5_checksum="00000000000000000000000000000000",
+            )
+
+
+@patch("requests.Session")
+def test_checksum_skipped_when_flag_off(Session_class_mock):
+    Session_class_mock.return_value.__enter__.return_value.get.return_value = _mock_response()
+
+    with openml.config.overwrite_config_context({
+        "check_digest": False,
+        "connection_n_retries": 1,  # to avoid retry delays
+    }):
+        # should NOT raise even though checksum mismatches
+        openml._api_calls._send_request(
+            request_method="get",
+            url="/dummy",
+            data={},
+            md5_checksum="not-a-real-sum",
+        )