From 99326c493b040b76db7aae9e3a08286c262761bb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 18 Mar 2026 15:47:54 +0000 Subject: [PATCH 1/8] activate anon bucket test --- tests/test_real_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_real_s3.py b/tests/test_real_s3.py index 1af391d9..36215ed2 100644 --- a/tests/test_real_s3.py +++ b/tests/test_real_s3.py @@ -31,9 +31,9 @@ def test_anon_s3(): }, active_storage_url=active_storage_url) active._version = 2 - with pytest.raises(ReductionistError): - result = active.min()[:] - assert result == 197.69595 + + result = active.min()[:] + assert result == 197.69595 def test_s3_small_file(): From b0fb765fd0af27b6404c782ceed33c75d9fc02ff Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 18 Mar 2026 15:48:14 +0000 Subject: [PATCH 2/8] anon auth in reductionist --- activestorage/reductionist.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 17b26d21..aa56531d 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -24,11 +24,6 @@ def get_session(username: str, password: str, :returns: a client session object. """ session = requests.Session() - # TODO Stack-HPC - # we need to allow Anon buckets. though this - # will break connection to data server - # if username is None and password is None: - # return session session.auth = (username, password) session.verify = cacert or False return session From 367db92be62292d130ef03603f4a3d5b054fe86e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 18 Mar 2026 16:17:13 +0000 Subject: [PATCH 3/8] correct handling of secrets and keys --- activestorage/active.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 438c52c0..513dc1e6 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -515,20 +515,19 @@ def _from_storage(self, ds, indexer, chunks, out_shape, out_dtype, if self.interface_type == "s3" and self._version == 2: if self.storage_options is not None: key, secret = None, None - if self.storage_options.get("anon", None) is True: - print("Reductionist session for Anon S3 bucket.") - session = reductionist.get_session( - None, None, S3_ACTIVE_STORAGE_CACERT) if "key" in self.storage_options: key = self.storage_options["key"] if "secret" in self.storage_options: secret = self.storage_options["secret"] - if key and secret: - session = reductionist.get_session( - key, secret, S3_ACTIVE_STORAGE_CACERT) + elif self.storage_options.get("anon", None) is True: + print("Reductionist session for Anon S3 bucket.") + key = None + secret = None else: - session = reductionist.get_session( - S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT) + key = S3_ACCESS_KEY + secret = S3_SECRET_KEY + session = reductionist.get_session( + key, secret, S3_ACTIVE_STORAGE_CACERT) else: session = reductionist.get_session(S3_ACCESS_KEY, S3_SECRET_KEY, From a21ed6e23ca37e93c162f484f79b860e53ca24cc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 18 Mar 2026 16:17:34 +0000 Subject: [PATCH 4/8] add a print statement --- activestorage/reductionist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index aa56531d..002c454b 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -86,6 +86,7 @@ def reduce_chunk(session, print(f"Reductionist request data dictionary: {request_data}") api_operation = "sum" if operation == "mean" else operation or "select" url = f'{server}/v2/{api_operation}/' + print("Reductionist Session auth:", session.auth) response = request(session, url, request_data) if response.ok: From 263804cc7f0557787c89e710fe98ef342b494bdc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 18 Mar 2026 16:21:58 +0000 Subject: [PATCH 5/8] flake8 --- activestorage/active.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/active.py b/activestorage/active.py index 513dc1e6..efa29b47 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -527,7 +527,7 @@ def _from_storage(self, ds, indexer, chunks, out_shape, out_dtype, key = S3_ACCESS_KEY secret = S3_SECRET_KEY session = reductionist.get_session( - key, secret, S3_ACTIVE_STORAGE_CACERT) + key, secret, S3_ACTIVE_STORAGE_CACERT) else: session = reductionist.get_session(S3_ACCESS_KEY, S3_SECRET_KEY, From 4aab9a424b9c7af549f3fe804744f14e7ad46af7 Mon Sep 17 00:00:00 2001 From: Max Norton Date: Tue, 31 Mar 2026 16:30:28 +0100 Subject: [PATCH 6/8] Fix anon S3 bucket access --- activestorage/active.py | 26 ++++++++++++++------------ activestorage/reductionist.py | 4 +++- tests/test_real_s3.py | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index efa29b47..db7c11e2 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -514,20 +514,22 @@ def _from_storage(self, ds, indexer, chunks, out_shape, out_dtype, # Create a shared session object. if self.interface_type == "s3" and self._version == 2: if self.storage_options is not None: - key, secret = None, None - if "key" in self.storage_options: - key = self.storage_options["key"] - if "secret" in self.storage_options: - secret = self.storage_options["secret"] - elif self.storage_options.get("anon", None) is True: + if self.storage_options.get("anon", None) is True: print("Reductionist session for Anon S3 bucket.") - key = None - secret = None + session = reductionist.get_session( + None, None, S3_ACTIVE_STORAGE_CACERT) else: - key = S3_ACCESS_KEY - secret = S3_SECRET_KEY - session = reductionist.get_session( - key, secret, S3_ACTIVE_STORAGE_CACERT) + key, secret = None, None + if "key" in self.storage_options: + key = self.storage_options["key"] + if "secret" in self.storage_options: + secret = self.storage_options["secret"] + if key and secret: + session = reductionist.get_session( + key, secret, S3_ACTIVE_STORAGE_CACERT) + else: + session = reductionist.get_session( + S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT) else: session = reductionist.get_session(S3_ACCESS_KEY, S3_SECRET_KEY, diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 002c454b..46bf77c2 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -24,8 +24,10 @@ def get_session(username: str, password: str, :returns: a client session object. """ session = requests.Session() - session.auth = (username, password) session.verify = cacert or False + if username is None and password is None: + return session + session.auth = (username, password) return session diff --git a/tests/test_real_s3.py b/tests/test_real_s3.py index 36215ed2..908727f8 100644 --- a/tests/test_real_s3.py +++ b/tests/test_real_s3.py @@ -31,9 +31,9 @@ def test_anon_s3(): }, active_storage_url=active_storage_url) active._version = 2 - result = active.min()[:] - assert result == 197.69595 + print("Result is", result) + assert result[0][0][0] == 197.69595 def test_s3_small_file(): From 9805a0348fcd86e6e6cce096406d377a879b8483 Mon Sep 17 00:00:00 2001 From: Max Norton Date: Tue, 31 Mar 2026 16:35:20 +0100 Subject: [PATCH 7/8] Make flake8 happy --- activestorage/reductionist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 46bf77c2..4f10cd9c 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -26,7 +26,7 @@ def get_session(username: str, password: str, session = requests.Session() session.verify = cacert or False if username is None and password is None: - return session + return session session.auth = (username, password) return session From 2337e7585dd58c5f68e947aa0465605b885ac91b Mon Sep 17 00:00:00 2001 From: Max Norton Date: Wed, 1 Apr 2026 16:13:54 +0100 Subject: [PATCH 8/8] Improve error checking on Reductionist response: - Reductionist will only report 500 internal server errors - here we check JSON body - Any other error response we check text of body - this way we catch the likes of HAProxy connection limits --- activestorage/reductionist.py | 14 +++++++----- tests/unit/test_reductionist.py | 40 ++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py index 4f10cd9c..e98fa0e6 100644 --- a/activestorage/reductionist.py +++ b/activestorage/reductionist.py @@ -88,7 +88,6 @@ def reduce_chunk(session, print(f"Reductionist request data dictionary: {request_data}") api_operation = "sum" if operation == "mean" else operation or "select" url = f'{server}/v2/{api_operation}/' - print("Reductionist Session auth:", session.auth) response = request(session, url, request_data) if response.ok: @@ -253,8 +252,13 @@ def __init__(self, status_code, error): def decode_and_raise_error(response): """Decode an error response and raise ReductionistError.""" - try: - error = json.dumps(response.json()) + if response.status_code == http.client.INTERNAL_SERVER_ERROR: + try: + error = json.dumps(response.json()) + except requests.exceptions.JSONDecodeError as exc: + error = http.client.responses.get(response.status_code, "-") + raise ReductionistError(response.status_code, error) from exc raise ReductionistError(response.status_code, error) - except requests.exceptions.JSONDecodeError as exc: - raise ReductionistError(response.status_code, "-") from exc + + error = http.client.responses.get(response.status_code, "-") + raise ReductionistError(response.status_code, error) diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py index 5d850f7e..8b6f3046 100644 --- a/tests/unit/test_reductionist.py +++ b/tests/unit/test_reductionist.py @@ -299,4 +299,42 @@ def test_reduce_chunk_not_found(mock_request): operation) print("Not found exc from reductionist", str(exc.value)) - assert str(exc.value) == 'Reductionist error: HTTP 404: -' + assert str(exc.value) == 'Reductionist error: HTTP 404: Not Found' + + +@mock.patch.object(reductionist, 'request') +def test_reductionist_internal_server_error_json(mock_request): + """Unit test for Reductionist 500 JSON response.""" + response = requests.Response() + response.status_code = 500 + response._content = b'{"detail": "backend exploded"}' + response.headers["Content-Type"] = "application/json" + mock_request.return_value = response + + active_url = "https://s3.example.com" + access_key = "fake-access" + secret_key = "fake-secret" + cacert = None + s3_url = "https://active.example.com" + offset = 2 + size = 128 + compression = None + filters = None + missing = [] + dtype = np.dtype("int32") + shape = (32, ) + axis = (0, ) + order = "C" + chunk_selection = [slice(0, 2, 1)] + operation = "min" + + session = reductionist.get_session(access_key, secret_key, cacert) + with pytest.raises(reductionist.ReductionistError) as exc: + reductionist.reduce_chunk(session, active_url, s3_url, + offset, size, compression, filters, missing, + dtype, shape, order, chunk_selection, axis, + operation) + + assert ( + str(exc.value) == 'Reductionist error: HTTP 500: {"detail": "backend exploded"}' + )