Merge pull request #1221 from jakob-keller/bump-botocore

Bump `botocore` dependency specification

Author: jakob-keller

CHANGES.rst

@@ -1,6 +1,10 @@
 Changes
 -------
 
+2.16.0 (2024-12-16)
+^^^^^^^^^^^^^^^^^^^
+* bump botocore dependency specification
+
 2.15.2 (2024-10-09)
 ^^^^^^^^^^^^^^^^^^^
 * relax botocore dependency specification

aiobotocore/__init__.py

@@ -1 +1 @@
-__version__ = '2.15.2'
+__version__ = '2.16.0'

aiobotocore/handlers.py

@@ -1,13 +1,14 @@
 from botocore.handlers import (
     ETree,
-    XMLParseError,
     _get_cross_region_presigned_url,
     _get_presigned_url_source_and_destination_regions,
+    _looks_like_special_case_error,
     logger,
 )
 
 
 async def check_for_200_error(response, **kwargs):
+    """This function has been deprecated, but is kept for backwards compatibility."""
     # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
     # There are two opportunities for a copy request to return an error. One
     # can occur when Amazon S3 receives the copy request and the other can
@@ -28,7 +29,9 @@ async def check_for_200_error(response, **kwargs):
         # trying to retrieve the response.  See Endpoint._get_response().
         return
     http_response, parsed = response
-    if await _looks_like_special_case_error(http_response):
+    if _looks_like_special_case_error(
+        http_response.status_code, await http_response.content
+    ):
         logger.debug(
             "Error found for response with 200 status code, "
             "errors: %s, changing status code to "
@@ -38,24 +41,6 @@ async def check_for_200_error(response, **kwargs):
         http_response.status_code = 500
 
 
-async def _looks_like_special_case_error(http_response):
-    if http_response.status_code == 200:
-        try:
-            parser = ETree.XMLParser(
-                target=ETree.TreeBuilder(), encoding='utf-8'
-            )
-            parser.feed(await http_response.content)
-            root = parser.close()
-        except XMLParseError:
-            # In cases of network disruptions, we may end up with a partial
-            # streamed response from S3. We need to treat these cases as
-            # 500 Service Errors and try again.
-            return True
-        if root.tag == 'Error':
-            return True
-    return False
-
-
 async def inject_presigned_url_ec2(params, request_signer, model, **kwargs):
     # The customer can still provide this, so we should pass if they do.
     if 'PresignedUrl' in params['body']:

aiobotocore/hooks.py

@@ -1,4 +1,3 @@
-from botocore.handlers import check_for_200_error as boto_check_for_200_error
 from botocore.handlers import (
     inject_presigned_url_ec2 as boto_inject_presigned_url_ec2,
 )
@@ -9,6 +8,9 @@
     parse_get_bucket_location as boto_parse_get_bucket_location,
 )
 from botocore.hooks import HierarchicalEmitter, logger
+from botocore.signers import (
+    add_dsql_generate_db_auth_token_methods as boto_add_dsql_generate_db_auth_token_methods,
+)
 from botocore.signers import (
     add_generate_db_auth_token as boto_add_generate_db_auth_token,
 )
@@ -21,12 +23,12 @@
 
 from ._helpers import resolve_awaitable
 from .handlers import (
-    check_for_200_error,
     inject_presigned_url_ec2,
     inject_presigned_url_rds,
     parse_get_bucket_location,
 )
 from .signers import (
+    add_dsql_generate_db_auth_token_methods,
     add_generate_db_auth_token,
     add_generate_presigned_post,
     add_generate_presigned_url,
@@ -39,7 +41,7 @@
     boto_add_generate_presigned_post: add_generate_presigned_post,
     boto_add_generate_db_auth_token: add_generate_db_auth_token,
     boto_parse_get_bucket_location: parse_get_bucket_location,
-    boto_check_for_200_error: check_for_200_error,
+    boto_add_dsql_generate_db_auth_token_methods: add_dsql_generate_db_auth_token_methods,
 }
 
 

aiobotocore/signers.py

@@ -2,7 +2,7 @@
 
 import botocore
 import botocore.auth
-from botocore.exceptions import UnknownClientMethodError
+from botocore.exceptions import ParamValidationError, UnknownClientMethodError
 from botocore.signers import (
     RequestSigner,
     S3PostPresigner,
@@ -200,6 +200,15 @@ def add_generate_db_auth_token(class_attributes, **kwargs):
     class_attributes['generate_db_auth_token'] = generate_db_auth_token
 
 
+def add_dsql_generate_db_auth_token_methods(class_attributes, **kwargs):
+    class_attributes['generate_db_connect_auth_token'] = (
+        dsql_generate_db_connect_auth_token
+    )
+    class_attributes['generate_db_connect_admin_auth_token'] = (
+        dsql_generate_db_connect_admin_auth_token
+    )
+
+
 async def generate_db_auth_token(
     self, DBHostname, Port, DBUsername, Region=None
 ):
@@ -256,6 +265,86 @@ async def generate_db_auth_token(
     return presigned_url[len(scheme) :]
 
 
+async def _dsql_generate_db_auth_token(
+    self, Hostname, Action, Region=None, ExpiresIn=900
+):
+    """Generate a DSQL database token for an arbitrary action.
+    :type Hostname: str
+    :param Hostname: The DSQL endpoint host name.
+    :type Action: str
+    :param Action: Action to perform on the cluster (DbConnectAdmin or DbConnect).
+    :type Region: str
+    :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used.
+    :type ExpiresIn: int
+    :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds).
+    :return: A presigned url which can be used as an auth token.
+    """
+    possible_actions = ("DbConnect", "DbConnectAdmin")
+
+    if Action not in possible_actions:
+        raise ParamValidationError(
+            report=f"Received {Action} for action but expected one of: {', '.join(possible_actions)}"
+        )
+
+    if Region is None:
+        Region = self.meta.region_name
+
+    request_dict = {
+        'url_path': '/',
+        'query_string': '',
+        'headers': {},
+        'body': {
+            'Action': Action,
+        },
+        'method': 'GET',
+    }
+    scheme = 'https://'
+    endpoint_url = f'{scheme}{Hostname}'
+    prepare_request_dict(request_dict, endpoint_url)
+    presigned_url = await self._request_signer.generate_presigned_url(
+        operation_name=Action,
+        request_dict=request_dict,
+        region_name=Region,
+        expires_in=ExpiresIn,
+        signing_name='dsql',
+    )
+    return presigned_url[len(scheme) :]
+
+
+async def dsql_generate_db_connect_auth_token(
+    self, Hostname, Region=None, ExpiresIn=900
+):
+    """Generate a DSQL database token for the "DbConnect" action.
+    :type Hostname: str
+    :param Hostname: The DSQL endpoint host name.
+    :type Region: str
+    :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used.
+    :type ExpiresIn: int
+    :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds).
+    :return: A presigned url which can be used as an auth token.
+    """
+    return await _dsql_generate_db_auth_token(
+        self, Hostname, "DbConnect", Region, ExpiresIn
+    )
+
+
+async def dsql_generate_db_connect_admin_auth_token(
+    self, Hostname, Region=None, ExpiresIn=900
+):
+    """Generate a DSQL database token for the "DbConnectAdmin" action.
+    :type Hostname: str
+    :param Hostname: The DSQL endpoint host name.
+    :type Region: str
+    :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used.
+    :type ExpiresIn: int
+    :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds).
+    :return: A presigned url which can be used as an auth token.
+    """
+    return await _dsql_generate_db_auth_token(
+        self, Hostname, "DbConnectAdmin", Region, ExpiresIn
+    )
+
+
 class AioS3PostPresigner(S3PostPresigner):
     async def generate_presigned_post(
         self,

aiobotocore/utils.py

@@ -461,13 +461,18 @@ async def redirect_from_error(
             0
         ].status_code in (301, 302, 307)
         is_permanent_redirect = error_code == 'PermanentRedirect'
+        is_opt_in_region_redirect = (
+            error_code == 'IllegalLocationConstraintException'
+            and operation.name != 'CreateBucket'
+        )
         if not any(
             [
                 is_special_head_object,
                 is_wrong_signing_region,
                 is_permanent_redirect,
                 is_special_head_bucket,
                 is_redirect_status,
+                is_opt_in_region_redirect,
             ]
         ):
             return

pyproject.toml

@@ -30,18 +30,18 @@ classifiers = [
 dynamic = ["version", "readme"]
 
 dependencies = [
-    "botocore >=1.35.16, <1.35.37", # NOTE: When updating, always keep `project.optional-dependencies` aligned
+    "botocore >=1.35.74, <1.35.82", # NOTE: When updating, always keep `project.optional-dependencies` aligned
     "aiohttp >=3.9.2, <4.0.0",
     "wrapt >=1.10.10, <2.0.0",
     "aioitertools >=0.5.1, <1.0.0",
 ]
 
 [project.optional-dependencies]
 awscli = [
-    "awscli >=1.34.16, <1.35.3",
+    "awscli >=1.36.15, <1.36.23",
 ]
 boto3 = [
-    "boto3 >=1.35.16, <1.35.37",
+    "boto3 >=1.35.74, <1.35.82",
 ]
 
 [project.urls]

requirements-dev.in

@@ -22,4 +22,5 @@ docker~=7.1
 moto[server,s3,sqs,awslambda,dynamodb,cloudformation,sns,batch,ec2,rds]~=4.2.9
 pre-commit~=3.5.0
 pytest-asyncio~=0.23.8
+time-machine~=2.15.0
 tomli; python_version < "3.11"  # Requirement for tests/test_version.py

tests/boto_tests/__init__.py

@@ -0,0 +1,39 @@
+# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from botocore.compat import parse_qs, urlparse
+
+
+def _urlparse(url):
+    if isinstance(url, bytes):
+        # Not really necessary, but it helps to reduce noise on Python 2.x
+        url = url.decode('utf8')
+    return urlparse(url)
+
+
+def assert_url_equal(url1, url2):
+    parts1 = _urlparse(url1)
+    parts2 = _urlparse(url2)
+
+    # Because the query string ordering isn't relevant, we have to parse
+    # every single part manually and then handle the query string.
+    assert parts1.scheme == parts2.scheme
+    assert parts1.netloc == parts2.netloc
+    assert parts1.path == parts2.path
+    assert parts1.params == parts2.params
+    assert parts1.fragment == parts2.fragment
+    assert parts1.username == parts2.username
+    assert parts1.password == parts2.password
+    assert parts1.hostname == parts2.hostname
+    assert parts1.port == parts2.port
+    assert parse_qs(parts1.query) == parse_qs(parts2.query)

tests/boto_tests/unit/test_handlers.py

@@ -0,0 +1,53 @@
+# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from unittest import mock
+
+from aiobotocore import handlers
+
+
+class TestHandlers:
+    async def test_500_status_code_set_for_200_response(self):
+        http_response = mock.Mock()
+        http_response.status_code = 200
+
+        async def content():
+            return """
+            <Error>
+              <Code>AccessDenied</Code>
+              <Message>Access Denied</Message>
+              <RequestId>id</RequestId>
+              <HostId>hostid</HostId>
+            </Error>
+        """
+
+        http_response.content = content()
+        await handlers.check_for_200_error((http_response, {}))
+        assert http_response.status_code == 500
+
+    async def test_200_response_with_no_error_left_untouched(self):
+        http_response = mock.Mock()
+        http_response.status_code = 200
+
+        async def content():
+            return "<NotAnError></NotAnError>"
+
+        http_response.content = content()
+        await handlers.check_for_200_error((http_response, {}))
+        # We don't touch the status code since there are no errors present.
+        assert http_response.status_code == 200
+
+    async def test_500_response_can_be_none(self):
+        # A 500 response can raise an exception, which means the response
+        # object is None.  We need to handle this case.
+        await handlers.check_for_200_error(None)