From b5d677d7566f9f1bba5ce49c551e453d1db4c7f9 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 08:49:12 -0500 Subject: [PATCH 01/31] Separate unit from integration tests --- test/runtests.jl | 12 +-- test/unit/AWS.jl | 135 ++++++++++++++++++++++++ test/{ => unit}/AWSConfig.jl | 2 +- test/{ => unit}/AWSExceptions.jl | 0 test/{ => unit}/AWSMetadataUtilities.jl | 8 +- test/{ => unit}/test_pkg.jl | 2 +- test/{ => unit}/utilities.jl | 0 7 files changed, 147 insertions(+), 12 deletions(-) rename test/{ => unit}/AWSConfig.jl (95%) rename test/{ => unit}/AWSExceptions.jl (100%) rename test/{ => unit}/AWSMetadataUtilities.jl (98%) rename test/{ => unit}/test_pkg.jl (85%) rename test/{ => unit}/utilities.jl (100%) diff --git a/test/runtests.jl b/test/runtests.jl index d59632c00..a7e3b343f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -61,11 +61,17 @@ if !RUN_UNIT_TESTS && !RUN_INTEGRATION_TESTS end const AWS_CONFIG = Ref{AbstractAWSConfig}() +const RESOURCE_DIR = joinpath(@__DIR__, "resources") @testset "AWS.jl" begin @testset "Unit Tests" begin if RUN_UNIT_TESTS include("unit/AWS.jl") + include("unit/AWSExceptions.jl") + include("unit/AWSMetadataUtilities.jl") + include("unit/test_pkg.jl") + include("unit/utilities.jl") + include("unit/AWSConfig.jl") else @warn "Skipping unit tests" end @@ -76,12 +82,6 @@ const AWS_CONFIG = Ref{AbstractAWSConfig}() if RUN_INTEGRATION_TESTS AWS_CONFIG[] = AWSConfig() - include("AWSExceptions.jl") - include("AWSMetadataUtilities.jl") - include("test_pkg.jl") - include("utilities.jl") - include("AWSConfig.jl") - backends = [AWS.HTTPBackend, AWS.DownloadsBackend] @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 46d5c32e0..fdf5524a3 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -111,3 +111,138 @@ @test !(Symbol("STS.X") in names(@__MODULE__; all=true)) end end + +@testset "global config, kwargs" begin + try + region = "us-east-2" + AWS.global_aws_config(; region=region) + + @test AWS.global_aws_config().region == region + finally + AWS.aws_config[] = AWSConfig() + end +end + +@testset "set global aws config" begin + test_region = "test region" + expected = AWSConfig(; region=test_region) + + try + AWS.global_aws_config(expected) + result = AWS.global_aws_config() + + @test result.region == test_region + finally + AWS.global_aws_config(AWSConfig()) + end +end + +@testset "set user agent" begin + old_user_agent = AWS.user_agent[] + new_user_agent = "new user agent" + + try + @test AWS.user_agent[] == "AWS.jl/$(pkgversion(AWS))" + set_user_agent(new_user_agent) + @test AWS.user_agent[] == new_user_agent + finally + set_user_agent(old_user_agent) + end +end + +@testset "sign" begin + aws = AWS.AWSConfig(; region="us-east-1") + + access_key = "access-key" + secret_key = "ssh... it is a secret" + + aws.credentials.access_key_id = access_key + aws.credentials.secret_key = secret_key + aws.credentials.token = "" + + time = DateTime(2020) + date = Dates.format(time, dateformat"yyyymmdd") + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + headers=LittleDict( + "Host" => "s3.us-east-1.amazonaws.com", "User-Agent" => "AWS.jl/1.0.0" + ), + resource="/test-resource", + url="https://s3.us-east-1.amazonaws.com/test-resource", + ) + + @testset "sign v2" begin + result = AWS.sign_aws2!(aws, request, time) + @test result === request + content = result.content + content_type = result.headers["Content-Type"] + + expected_access_key = "AWSAccessKeyId=$access_key" + expected_expires = "Expires=2020-01-01T00%3A02%3A00Z" + expected_signature_method = "SignatureMethod=HmacSHA256" + expected_signature_version = "SignatureVersion=2" + expected_signature = "Signature=O0MLzMKpEcfVZeHy0tyxVAuZF%2BvrvbgIGgqbWtJLTQ0%3D" + + expected_content = join( + [ + expected_access_key, + expected_expires, + expected_signature_method, + expected_signature_version, + expected_signature, + ], + '&', + ) + + @test content == expected_content + end + + @testset "sign v4" begin + @testset "basic" begin + expected_x_amz_content_sha256 = bytes2hex(digest(MD_SHA256, request.content)) + expected_content_md5 = base64encode(digest(MD_MD5, request.content)) + expected_x_amz_date = Dates.format(time, dateformat"yyyymmdd\THHMMSS\Z") + + result = AWS.sign_aws4!(aws, request, time) + @test result === request + headers = result.headers + + @test headers["x-amz-content-sha256"] == expected_x_amz_content_sha256 + @test headers["Content-MD5"] == expected_content_md5 + @test headers["x-amz-date"] == expected_x_amz_date + + authorization_header = split(headers["Authorization"], ' ') + @test length(authorization_header) == 4 + @test authorization_header[1] == "AWS4-HMAC-SHA256" + @test authorization_header[2] == + "Credential=$access_key/$date/us-east-1/$(request.service)/aws4_request," + @test authorization_header[3] == + "SignedHeaders=content-md5;content-type;host;user-agent;x-amz-content-sha256;x-amz-date," + @test authorization_header[4] == + "Signature=0f292eaf0b66cf353bafcb1b9b6d90ee27064236a60f17f6fc5bd7d40173a0be" + end + + @testset "duplicate query params" begin + request_query = deepcopy(request) + request_query.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions" + + request_dup = deepcopy(request) + request_dup.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions&versions=" + + AWS.sign_aws4!(aws, request_query, time) + AWS.sign_aws4!(aws, request_dup, time) + + authorization_header_query = split(request_query.headers["Authorization"], ' ') + authorization_header_dup = split(request_dup.headers["Authorization"], ' ') + + @test length(authorization_header_query) == 4 + @test length(authorization_header_dup) == 4 + + # Signatures should differ + @test authorization_header_query[4] != authorization_header_dup[4] + end + end +end diff --git a/test/AWSConfig.jl b/test/unit/AWSConfig.jl similarity index 95% rename from test/AWSConfig.jl rename to test/unit/AWSConfig.jl index e7db09245..c94551e86 100644 --- a/test/AWSConfig.jl +++ b/test/unit/AWSConfig.jl @@ -1,7 +1,7 @@ @testset "AWSConfig" begin @testset "default profile assumes role" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "default-role") + config_dir = joinpath(@__DIR__, "..", "configs", "default-role") # Avoid calling out to STS with invalid credentials patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) diff --git a/test/AWSExceptions.jl b/test/unit/AWSExceptions.jl similarity index 100% rename from test/AWSExceptions.jl rename to test/unit/AWSExceptions.jl diff --git a/test/AWSMetadataUtilities.jl b/test/unit/AWSMetadataUtilities.jl similarity index 98% rename from test/AWSMetadataUtilities.jl rename to test/unit/AWSMetadataUtilities.jl index d07af2eb3..9d2d5a455 100644 --- a/test/AWSMetadataUtilities.jl +++ b/test/unit/AWSMetadataUtilities.jl @@ -71,7 +71,7 @@ end end @testset "_generate_low_level_definitions" begin - services = JSON.parsefile(joinpath(@__DIR__, "resources/services.json")) + services = JSON.parsefile(joinpath(RESOURCE_DIR, "services.json")) @testset "rest-xml" begin expected = "const s3 = AWS.RestXMLService(\"s3\", \"s3\", \"2006-03-01\")" @@ -251,7 +251,7 @@ end end @testset "_get_function_parameters" begin - shapes = JSON.parsefile(joinpath(@__DIR__, "resources/shapes.json")) + shapes = JSON.parsefile(joinpath(RESOURCE_DIR, "shapes.json")) @testset "required params" begin input = "RequiredParams" @@ -310,8 +310,8 @@ end @testset "_generate_high_level_definitions" begin service_name = "sample_service" protocol = "rest-xml" - operations = JSON.parsefile(joinpath(@__DIR__, "resources/operations.json")) - shapes = JSON.parsefile(joinpath(@__DIR__, "resources/shapes.json")) + operations = JSON.parsefile(joinpath(RESOURCE_DIR, "operations.json")) + shapes = JSON.parsefile(joinpath(RESOURCE_DIR, "shapes.json")) expected_result = """ \"\"\" diff --git a/test/test_pkg.jl b/test/unit/test_pkg.jl similarity index 85% rename from test/test_pkg.jl rename to test/unit/test_pkg.jl index 608ef3a7f..56af2a237 100644 --- a/test/test_pkg.jl +++ b/test/unit/test_pkg.jl @@ -1,4 +1,4 @@ -path = joinpath(@__DIR__, "resources", "TestPkg") +path = joinpath(RESOURCE_DIR, "TestPkg") Pkg.develop(; path=path) VERSION >= v"1.8" && Pkg.precompile("TestPkg") diff --git a/test/utilities.jl b/test/unit/utilities.jl similarity index 100% rename from test/utilities.jl rename to test/unit/utilities.jl From fcea2e547bea10d1980db283afea60affd569890 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 09:00:13 -0500 Subject: [PATCH 02/31] fixup! Separate unit from integration tests --- test/unit/AWS.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index fdf5524a3..77694d84a 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -151,14 +151,11 @@ end end @testset "sign" begin - aws = AWS.AWSConfig(; region="us-east-1") - access_key = "access-key" secret_key = "ssh... it is a secret" - aws.credentials.access_key_id = access_key - aws.credentials.secret_key = secret_key - aws.credentials.token = "" + creds = AWS.AWSCredentials(access_key secret_key) + aws = AWS.AWSConfig(; creds, region="us-east-1") time = DateTime(2020) date = Dates.format(time, dateformat"yyyymmdd") From cbd1b31becf712c6716e61a80ff64308f7276fde Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 09:03:22 -0500 Subject: [PATCH 03/31] fixup! Separate unit from integration tests --- test/AWS.jl | 135 ----------------------------------------------- test/unit/AWS.jl | 2 +- 2 files changed, 1 insertion(+), 136 deletions(-) diff --git a/test/AWS.jl b/test/AWS.jl index 1567e37e5..7d3b67947 100644 --- a/test/AWS.jl +++ b/test/AWS.jl @@ -1,138 +1,3 @@ -@testset "global config, kwargs" begin - try - region = "us-east-2" - AWS.global_aws_config(; region=region) - - @test AWS.global_aws_config().region == region - finally - AWS.aws_config[] = AWSConfig() - end -end - -@testset "set global aws config" begin - test_region = "test region" - expected = AWSConfig(; region=test_region) - - try - AWS.global_aws_config(expected) - result = AWS.global_aws_config() - - @test result.region == test_region - finally - AWS.global_aws_config(AWSConfig()) - end -end - -@testset "set user agent" begin - old_user_agent = AWS.user_agent[] - new_user_agent = "new user agent" - - try - @test AWS.user_agent[] == "AWS.jl/$(pkgversion(AWS))" - set_user_agent(new_user_agent) - @test AWS.user_agent[] == new_user_agent - finally - set_user_agent(old_user_agent) - end -end - -@testset "sign" begin - aws = AWS.AWSConfig(; region="us-east-1") - - access_key = "access-key" - secret_key = "ssh... it is a secret" - - aws.credentials.access_key_id = access_key - aws.credentials.secret_key = secret_key - aws.credentials.token = "" - - time = DateTime(2020) - date = Dates.format(time, dateformat"yyyymmdd") - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - headers=LittleDict( - "Host" => "s3.us-east-1.amazonaws.com", "User-Agent" => "AWS.jl/1.0.0" - ), - resource="/test-resource", - url="https://s3.us-east-1.amazonaws.com/test-resource", - ) - - @testset "sign v2" begin - result = AWS.sign_aws2!(aws, request, time) - @test result === request - content = result.content - content_type = result.headers["Content-Type"] - - expected_access_key = "AWSAccessKeyId=$access_key" - expected_expires = "Expires=2020-01-01T00%3A02%3A00Z" - expected_signature_method = "SignatureMethod=HmacSHA256" - expected_signature_version = "SignatureVersion=2" - expected_signature = "Signature=O0MLzMKpEcfVZeHy0tyxVAuZF%2BvrvbgIGgqbWtJLTQ0%3D" - - expected_content = join( - [ - expected_access_key, - expected_expires, - expected_signature_method, - expected_signature_version, - expected_signature, - ], - '&', - ) - - @test content == expected_content - end - - @testset "sign v4" begin - @testset "basic" begin - expected_x_amz_content_sha256 = bytes2hex(digest(MD_SHA256, request.content)) - expected_content_md5 = base64encode(digest(MD_MD5, request.content)) - expected_x_amz_date = Dates.format(time, dateformat"yyyymmdd\THHMMSS\Z") - - result = AWS.sign_aws4!(aws, request, time) - @test result === request - headers = result.headers - - @test headers["x-amz-content-sha256"] == expected_x_amz_content_sha256 - @test headers["Content-MD5"] == expected_content_md5 - @test headers["x-amz-date"] == expected_x_amz_date - - authorization_header = split(headers["Authorization"], ' ') - @test length(authorization_header) == 4 - @test authorization_header[1] == "AWS4-HMAC-SHA256" - @test authorization_header[2] == - "Credential=$access_key/$date/us-east-1/$(request.service)/aws4_request," - @test authorization_header[3] == - "SignedHeaders=content-md5;content-type;host;user-agent;x-amz-content-sha256;x-amz-date," - @test authorization_header[4] == - "Signature=0f292eaf0b66cf353bafcb1b9b6d90ee27064236a60f17f6fc5bd7d40173a0be" - end - - @testset "duplicate query params" begin - request_query = deepcopy(request) - request_query.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions" - - request_dup = deepcopy(request) - request_dup.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions&versions=" - - AWS.sign_aws4!(aws, request_query, time) - AWS.sign_aws4!(aws, request_dup, time) - - authorization_header_query = split(request_query.headers["Authorization"], ' ') - authorization_header_dup = split(request_dup.headers["Authorization"], ' ') - - @test length(authorization_header_query) == 4 - @test length(authorization_header_dup) == 4 - - # Signatures should differ - @test authorization_header_query[4] != authorization_header_dup[4] - end - end -end - @testset "submit_request" begin aws = AWS.AWSConfig() diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 77694d84a..1d80de95c 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -154,7 +154,7 @@ end access_key = "access-key" secret_key = "ssh... it is a secret" - creds = AWS.AWSCredentials(access_key secret_key) + creds = AWS.AWSCredentials(access_key, secret_key) aws = AWS.AWSConfig(; creds, region="us-east-1") time = DateTime(2020) From 6b3ecec3aa9333f31a8343e8518032559e785384 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 09:23:35 -0500 Subject: [PATCH 04/31] Iterating --- test/runtests.jl | 3 ++- test/unit/AWS.jl | 32 +++++++++++++++++++++---------- test/{ => unit}/AWSCredentials.jl | 25 ++---------------------- test/{ => unit}/IMDS.jl | 0 4 files changed, 26 insertions(+), 34 deletions(-) rename test/{ => unit}/AWSCredentials.jl (98%) rename test/{ => unit}/IMDS.jl (100%) diff --git a/test/runtests.jl b/test/runtests.jl index a7e3b343f..f8e664024 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -72,6 +72,8 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") include("unit/test_pkg.jl") include("unit/utilities.jl") include("unit/AWSConfig.jl") + include("unit/IMDS.jl") + include("unit/AWSCredentials.jl") else @warn "Skipping unit tests" end @@ -86,7 +88,6 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() include("AWS.jl") - include("IMDS.jl") include("AWSCredentials.jl") include("role.jl") include("issues.jl") diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 1d80de95c..6f2ed9d15 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -113,27 +113,39 @@ end @testset "global config, kwargs" begin + creds = AWS.AWSCredentials("access-key", "secret") + region = "us-east-2" + + old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing try - region = "us-east-2" - AWS.global_aws_config(; region=region) + AWS.global_aws_config(; creds, region) - @test AWS.global_aws_config().region == region + result = AWS.global_aws_config() + @test result.region == region + @test result.credentials == creds finally - AWS.aws_config[] = AWSConfig() + if !isnothing(old_config) + AWS.aws_config[] = old_config + end end end @testset "set global aws config" begin - test_region = "test region" - expected = AWSConfig(; region=test_region) + creds = AWS.AWSCredentials("access-key", "secret") + region = "test region" + config = AWSConfig(; creds, region) + old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing try - AWS.global_aws_config(expected) - result = AWS.global_aws_config() + AWS.global_aws_config(config) - @test result.region == test_region + result = AWS.global_aws_config() + @test result.region == region + @test result.credentials == creds finally - AWS.global_aws_config(AWSConfig()) + if !isnothing(old_config) + AWS.aws_config[] = old_config + end end end diff --git a/test/AWSCredentials.jl b/test/unit/AWSCredentials.jl similarity index 98% rename from test/AWSCredentials.jl rename to test/unit/AWSCredentials.jl index 13f346883..dbe82a444 100644 --- a/test/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -18,27 +18,6 @@ const EXPIRATION_FMT = dateformat"yyyy-mm-dd\THH:MM:SS\Z" http_header(h::Vector, k, d="") = get(Dict(h), k, d) http_header(args...) = HTTP.header(args...) -@testset "Load Credentials" begin - user = aws_user_arn(AWS_CONFIG[]) - @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) - AWS_CONFIG[].region = "us-east-1" - - @test_ecode("InvalidAction", AWSServices.iam("GetFoo")) - - @test_ecode( - ["AccessDenied", "NoSuchEntity"], - AWSServices.iam("GetUser", Dict("UserName" => "notauser")) - ) - - @test_ecode("ValidationError", AWSServices.iam("GetUser", Dict("UserName" => "@#!%%!"))) - - # Please note: If testing in a managed Corporate AWS environment, this can set off alarms... - @test_ecode( - ["AccessDenied", "EntityAlreadyExists"], - AWSServices.iam("CreateUser", Dict("UserName" => "root")) - ) -end - @testset "_role_session_name" begin @test AWS._role_session_name("prefix-", "name", "-suffix") == "prefix-name-suffix" @test AWS._role_session_name("a"^22, "b"^22, "c"^22) == "a"^22 * "b"^20 * "c"^22 @@ -68,7 +47,7 @@ end @testset "default profile" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "default-role") + config_dir = joinpath(@__DIR__, "..", "configs", "default-role") patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) @@ -89,7 +68,7 @@ end @testset "profile with role and MFA" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "role-with-mfa") + config_dir = joinpath(@__DIR__, "..", "configs", "role-with-mfa") mfa_token = "123456" sent_token = Ref("") diff --git a/test/IMDS.jl b/test/unit/IMDS.jl similarity index 100% rename from test/IMDS.jl rename to test/unit/IMDS.jl From d4d8374221891a8edaa350faa5cc5da06cf5e3d5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:28:21 -0500 Subject: [PATCH 05/31] Update _aws_profile_config --- src/utilities/credentials.jl | 13 ++++++-- test/unit/AWSCredentials.jl | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 6e2bc5240..812f80b83 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -36,11 +36,20 @@ function _get_ini_value( end function _aws_profile_config(ini::Inifile, profile::AbstractString) - if profile != "default" || !haskey(sections(ini), "default") + # Prefer using "profile default" over "default" + if profile != "default" || haskey(sections(ini), "profile default") profile = "profile $profile" end - return get(sections(ini), profile, Dict()) + content = copy(get(sections(ini), profile, IniFile.HTSS())) + source_profile = pop!(content, "source_profile", nothing) + + # Fallback on settings specified in the source profile + if !isnothing(source_profile) + content = merge(_aws_profile_config(ini, source_profile), content) + end + + return content end function _aws_profile_config(ini::Inifile, profile::Nothing) diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index dbe82a444..886a4df0f 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -23,6 +23,67 @@ http_header(args...) = HTTP.header(args...) @test AWS._role_session_name("a"^22, "b"^22, "c"^22) == "a"^22 * "b"^20 * "c"^22 end +@testset "_aws_profile_config" begin + using AWS: _aws_profile_config + + @testset "source profile" begin + buffer = IOBuffer( + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + """ + ) + ini = Inifile() + read(ini, buffer) + + # Basic profile + config = _aws_profile_config(ini, "test") + @test keys(config) ⊆ Set(["output", "region"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + + + config = _aws_profile_config(ini, "test:dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" + + # Conflicting keys should use the first defined entry + config = _aws_profile_config(ini, "test:sub-dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" + end + + @testset "default profile" begin + buffer = IOBuffer( + """ + [default] + region = default-1 + + [profile default] + region = default-2 + """ + ) + ini = Inifile() + read(ini, buffer) + + config = _aws_profile_config(ini, "default") + @test config["region"] == "default-2" + end +end + @testset "aws_get_profile_settings" begin @testset "no profile" begin @test aws_get_profile_settings("foo", Inifile()) === nothing From 622f513f14faf5c8dba76d0c92bbef3f6d32cc06 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:28:54 -0500 Subject: [PATCH 06/31] fixup --- test/AWSCredentials.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 test/AWSCredentials.jl diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl new file mode 100644 index 000000000..9070533e2 --- /dev/null +++ b/test/AWSCredentials.jl @@ -0,0 +1,20 @@ +@testset "Load Credentials" begin + user = aws_user_arn(AWS_CONFIG[]) + @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) + AWS_CONFIG[].region = "us-east-1" + + @test_ecode("InvalidAction", AWSServices.iam("GetFoo")) + + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSServices.iam("GetUser", Dict("UserName" => "notauser")) + ) + + @test_ecode("ValidationError", AWSServices.iam("GetUser", Dict("UserName" => "@#!%%!"))) + + # Please note: If testing in a managed Corporate AWS environment, this can set off alarms... + @test_ecode( + ["AccessDenied", "EntityAlreadyExists"], + AWSServices.iam("CreateUser", Dict("UserName" => "root")) + ) +end From 9ecfe5a04e2292a0d3d2a16c48f3c447e72b297c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:35:28 -0500 Subject: [PATCH 07/31] Improve AWS config support for `source_profile` --- src/utilities/credentials.jl | 17 ++++++++-- test/AWSCredentials.jl | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 6e2bc5240..2d8885e47 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -36,11 +36,22 @@ function _get_ini_value( end function _aws_profile_config(ini::Inifile, profile::AbstractString) - if profile != "default" || !haskey(sections(ini), "default") - profile = "profile $profile" + # Prefer using "profile default" over "default" + section_name = if profile != "default" || haskey(sections(ini), "profile default") + "profile $profile" + else + "default" + end + + content = copy(get(sections(ini), section_name, IniFile.HTSS())) + source_profile = pop!(content, "source_profile", nothing) + + # Fallback on settings specified in the source profile + if !isnothing(source_profile) + content = merge(_aws_profile_config(ini, source_profile), content) end - return get(sections(ini), profile, Dict()) + return content end function _aws_profile_config(ini::Inifile, profile::Nothing) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 13f346883..ca719cfff 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -50,6 +50,70 @@ end end end +@testset "_aws_profile_config" begin + using AWS: _aws_profile_config + + @testset "source profile" begin + buffer = IOBuffer( + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + """ + ) + ini = Inifile() + read(ini, buffer) + + # Basic profile + config = _aws_profile_config(ini, "test") + @test keys(config) ⊆ Set(["output", "region"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + + + config = _aws_profile_config(ini, "test:dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" + + # Conflicting keys should use the first defined entry + config = _aws_profile_config(ini, "test:sub-dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" + end + + # AWS CLI (version v2.27.15) will use "profile default" over "default" when both are + # defined within the configuration. This is true when `AWS_PROFILE` is unset or + # `AWS_PROFILE="default". + @testset "default profile" begin + buffer = IOBuffer( + """ + [default] + region = default-1 + + [profile default] + region = default-2 + """ + ) + ini = Inifile() + read(ini, buffer) + + config = _aws_profile_config(ini, "default") + @test config["region"] == "default-2" + end +end + @testset "_aws_get_role" begin profile = "foobar" ini = Inifile() From a30b1b6ab845990caf270d0b08437ddabd565016 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:47:39 -0500 Subject: [PATCH 08/31] fixup --- test/AWSCredentials.jl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index ca719cfff..db0f8405f 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -54,6 +54,7 @@ end using AWS: _aws_profile_config @testset "source profile" begin + #! format: off buffer = IOBuffer( """ [profile test] @@ -69,34 +70,46 @@ end role_arn = arn:aws:iam::123456789000:role/SubDev """ ) + #! format: on ini = Inifile() read(ini, buffer) - # Basic profile + # Only the fields from profile "test" config = _aws_profile_config(ini, "test") @test keys(config) ⊆ Set(["output", "region"]) @test config["output"] == "json" @test config["region"] == "us-east-1" - + # Combine the profile "test:dev" section with fields from profile "test" config = _aws_profile_config(ini, "test:dev") @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" + # Ensure we haven't mutated the contents of the `ini` + section = sections(ini)["test:dev"] + @test !haskey(section, "region") + @test !haskey(section, "output") + # Conflicting keys should use the first defined entry config = _aws_profile_config(ini, "test:sub-dev") @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" + + # Ensure we haven't mutated the contents of the `ini` + section = sections(ini)["test:sub-dev"] + @test !haskey(section, "region") + @test !haskey(section, "output") end # AWS CLI (version v2.27.15) will use "profile default" over "default" when both are # defined within the configuration. This is true when `AWS_PROFILE` is unset or # `AWS_PROFILE="default". @testset "default profile" begin + #! format: off buffer = IOBuffer( """ [default] @@ -106,6 +119,7 @@ end region = default-2 """ ) + #! format: on ini = Inifile() read(ini, buffer) From b21f5688a2cabc2a039e1c35dd0ae96317f65c3a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:48:42 -0500 Subject: [PATCH 09/31] Make tests specificially unit tests --- test/AWSCredentials.jl | 78 ------------------------------------- test/runtests.jl | 1 + test/unit/AWSCredentials.jl | 77 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 78 deletions(-) create mode 100644 test/unit/AWSCredentials.jl diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index db0f8405f..13f346883 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -50,84 +50,6 @@ end end end -@testset "_aws_profile_config" begin - using AWS: _aws_profile_config - - @testset "source profile" begin - #! format: off - buffer = IOBuffer( - """ - [profile test] - output = json - region = us-east-1 - - [profile test:dev] - source_profile = test - role_arn = arn:aws:iam::123456789000:role/Dev - - [profile test:sub-dev] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/SubDev - """ - ) - #! format: on - ini = Inifile() - read(ini, buffer) - - # Only the fields from profile "test" - config = _aws_profile_config(ini, "test") - @test keys(config) ⊆ Set(["output", "region"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - - # Combine the profile "test:dev" section with fields from profile "test" - config = _aws_profile_config(ini, "test:dev") - @test keys(config) ⊆ Set(["output", "region", "role_arn"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" - - # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["test:dev"] - @test !haskey(section, "region") - @test !haskey(section, "output") - - # Conflicting keys should use the first defined entry - config = _aws_profile_config(ini, "test:sub-dev") - @test keys(config) ⊆ Set(["output", "region", "role_arn"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" - - # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["test:sub-dev"] - @test !haskey(section, "region") - @test !haskey(section, "output") - end - - # AWS CLI (version v2.27.15) will use "profile default" over "default" when both are - # defined within the configuration. This is true when `AWS_PROFILE` is unset or - # `AWS_PROFILE="default". - @testset "default profile" begin - #! format: off - buffer = IOBuffer( - """ - [default] - region = default-1 - - [profile default] - region = default-2 - """ - ) - #! format: on - ini = Inifile() - read(ini, buffer) - - config = _aws_profile_config(ini, "default") - @test config["region"] == "default-2" - end -end - @testset "_aws_get_role" begin profile = "foobar" ini = Inifile() diff --git a/test/runtests.jl b/test/runtests.jl index d59632c00..2b0d13ca8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,6 +66,7 @@ const AWS_CONFIG = Ref{AbstractAWSConfig}() @testset "Unit Tests" begin if RUN_UNIT_TESTS include("unit/AWS.jl") + include("unit/AWSCredentials.jl") else @warn "Skipping unit tests" end diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl new file mode 100644 index 000000000..f212c12bb --- /dev/null +++ b/test/unit/AWSCredentials.jl @@ -0,0 +1,77 @@ +@testset "_aws_profile_config" begin + using AWS: _aws_profile_config + + @testset "source profile" begin + #! format: off + buffer = IOBuffer( + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + """ + ) + #! format: on + ini = Inifile() + read(ini, buffer) + + # Only the fields from profile "test" + config = _aws_profile_config(ini, "test") + @test keys(config) ⊆ Set(["output", "region"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + + # Combine the profile "test:dev" section with fields from profile "test" + config = _aws_profile_config(ini, "test:dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" + + # Ensure we haven't mutated the contents of the `ini` + section = sections(ini)["test:dev"] + @test !haskey(section, "region") + @test !haskey(section, "output") + + # Conflicting keys should use the first defined entry + config = _aws_profile_config(ini, "test:sub-dev") + @test keys(config) ⊆ Set(["output", "region", "role_arn"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" + + # Ensure we haven't mutated the contents of the `ini` + section = sections(ini)["test:sub-dev"] + @test !haskey(section, "region") + @test !haskey(section, "output") + end + + # AWS CLI (version v2.27.15) will use "profile default" over "default" when both are + # defined within the configuration. This is true when `AWS_PROFILE` is unset or + # `AWS_PROFILE="default". + @testset "default profile" begin + #! format: off + buffer = IOBuffer( + """ + [default] + region = default-1 + + [profile default] + region = default-2 + """ + ) + #! format: on + ini = Inifile() + read(ini, buffer) + + config = _aws_profile_config(ini, "default") + @test config["region"] == "default-2" + end +end From 06f947c8c0f910cb07652aa877928d09b00485fc Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:53:47 -0500 Subject: [PATCH 10/31] fixup --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 2b0d13ca8..bd2bee277 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,7 +23,7 @@ using Dates using Downloads using GitHub using HTTP -using IniFile: Inifile +using IniFile: Inifile, sections using JSON using OrderedCollections: LittleDict, OrderedDict using MbedTLS: digest, MD_SHA256, MD_MD5 From b8c523129682c609b936f7d7aa3ff2770c778e59 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 10:57:31 -0500 Subject: [PATCH 11/31] fixup --- test/unit/AWSCredentials.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index f212c12bb..73add8103 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -36,7 +36,7 @@ @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["test:dev"] + section = sections(ini)["profile test:dev"] @test !haskey(section, "region") @test !haskey(section, "output") @@ -48,7 +48,7 @@ @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["test:sub-dev"] + section = sections(ini)["profile test:sub-dev"] @test !haskey(section, "region") @test !haskey(section, "output") end From ebea4327b9f76f57874fbdbad4a51d5f72939c0c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 12:23:14 -0500 Subject: [PATCH 12/31] fixup! Merge branch 'cv/aws-profile-config' into cv/test-reorg --- test/unit/AWSCredentials.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index 65143dd18..9cbaa1d88 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -1423,5 +1423,3 @@ end end end end -======= ->>>>>>> cv/aws-profile-config From 13300906d1026790684b4929ad1ca7c76fc706ac Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 12:23:43 -0500 Subject: [PATCH 13/31] Make resursive profile merging optional --- src/AWSCredentials.jl | 2 +- src/utilities/credentials.jl | 25 ++++++++++++++----------- test/unit/AWSCredentials.jl | 6 +++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 9f83aa3ce..5c91634c7 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -657,7 +657,7 @@ function aws_get_region(; profile=nothing, config=nothing, default=DEFAULT_REGIO @something( get(ENV, "AWS_REGION", nothing), get(ENV, "AWS_DEFAULT_REGION", nothing), - get(_aws_profile_config(config, profile), "region", nothing), + get(_aws_profile_config(config, profile; recursive=true), "region", nothing), @mock(IMDS.region()), Some(default), ) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 2d8885e47..553694762 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -35,7 +35,7 @@ function _get_ini_value( return value end -function _aws_profile_config(ini::Inifile, profile::AbstractString) +function _aws_profile_config(ini::Inifile, profile::AbstractString; recursive::Bool=false) # Prefer using "profile default" over "default" section_name = if profile != "default" || haskey(sections(ini), "profile default") "profile $profile" @@ -44,27 +44,30 @@ function _aws_profile_config(ini::Inifile, profile::AbstractString) end content = copy(get(sections(ini), section_name, IniFile.HTSS())) - source_profile = pop!(content, "source_profile", nothing) - # Fallback on settings specified in the source profile - if !isnothing(source_profile) - content = merge(_aws_profile_config(ini, source_profile), content) + if recursive + source_profile = pop!(content, "source_profile", nothing) + + # Fallback on settings specified in the source profile + if !isnothing(source_profile) + content = merge(_aws_profile_config(ini, source_profile; recursive), content) + end end return content end -function _aws_profile_config(ini::Inifile, profile::Nothing) - return _aws_profile_config(ini, _aws_get_profile()) +function _aws_profile_config(ini::Inifile, profile::Nothing; kwargs...) + return _aws_profile_config(ini, _aws_get_profile(); kwargs...) end -function _aws_profile_config(config_file::AbstractString, profile) +function _aws_profile_config(config_file::AbstractString, profile; kwargs...) isfile(config_file) || return Dict() - return _aws_profile_config(read(Inifile(), config_file), profile) + return _aws_profile_config(read(Inifile(), config_file), profile; kwargs...) end -function _aws_profile_config(config_file::Nothing, profile) - return _aws_profile_config(dot_aws_config_file(), profile) +function _aws_profile_config(config_file::Nothing, profile; kwargs...) + return _aws_profile_config(dot_aws_config_file(), profile; kwargs...) end """ diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index 9cbaa1d88..3fbf744ea 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -48,13 +48,13 @@ end read(ini, buffer) # Only the fields from profile "test" - config = _aws_profile_config(ini, "test") + config = _aws_profile_config(ini, "test"; recursive=true) @test keys(config) ⊆ Set(["output", "region"]) @test config["output"] == "json" @test config["region"] == "us-east-1" # Combine the profile "test:dev" section with fields from profile "test" - config = _aws_profile_config(ini, "test:dev") + config = _aws_profile_config(ini, "test:dev"; recursive=true) @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" @@ -66,7 +66,7 @@ end @test !haskey(section, "output") # Conflicting keys should use the first defined entry - config = _aws_profile_config(ini, "test:sub-dev") + config = _aws_profile_config(ini, "test:sub-dev"; recursive=true) @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" From 272b7b6334aecd749e296f879da0bab97f822de3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:06:05 -0500 Subject: [PATCH 14/31] More unit tests --- test/AWS.jl | 508 ---------------------------------------- test/runtests.jl | 33 ++- test/unit/AWS.jl | 515 ++++++++++++++++++++++++++++++++++++++++- test/unit/AWSConfig.jl | 19 +- 4 files changed, 543 insertions(+), 532 deletions(-) diff --git a/test/AWS.jl b/test/AWS.jl index 7d3b67947..11f02aac5 100644 --- a/test/AWS.jl +++ b/test/AWS.jl @@ -1,511 +1,3 @@ -@testset "submit_request" begin - aws = AWS.AWSConfig() - - function _expected_xml(body::AbstractString, dict_type::Type) - parsed = parse_xml(body) - return xml_dict(XMLDict.root(parsed.x), dict_type) - end - - @testset "301 redirect" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="HEAD", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - apply(Patches._aws_http_request_patch(Patches._response(; status=301))) do - @test_throws AWSException AWS.submit_request(aws, request) - end - end - - @testset "HEAD response" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="HEAD", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - response = apply(Patches._aws_http_request_patch()) do - AWS.submit_request(aws, request) - end - - # Access to response headers - @test response.response.headers == Patches.headers - @test response.response.headers isa Vector - - # Access to streaming content - @test response.io isa IO - - # Content as a string - @test String(take!(response.io)) == Patches.body - - # Backwards compatibility with those expecting an `HTTP.Response` - @test response.headers == Patches.headers - @test response.headers isa Vector - @test String(response.body) == Patches.body - end - - @testset "GET response" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - response = apply(Patches._aws_http_request_patch()) do - AWS.submit_request(aws, request) - end - - # Access to response headers - @test response.response.headers == Patches.headers - @test response.response.headers isa Vector - - # Access to streaming content - @test response.io isa IO - - # Content as a string - @test String(take!(response.io)) == Patches.body - - # Backwards compatibility with those expecting an `HTTP.Response` - @test response.headers == Patches.headers - @test response.headers isa Vector - @test String(response.body) == Patches.body - end - - @testset "Default throttling" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - retries = Ref{Int}(0) - exception = apply(Patches._throttling_patch(retries)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - @test exception.code == "SlowDown" - @test retries[] == AWS.max_attempts(aws) - end - - @testset "Custom throttling" begin - aws = AWS.AWSConfig(; max_attempts=1) - @test AWS.max_attempts(aws) == 1 - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - retries = Ref{Int}(0) - exception = apply(Patches._throttling_patch(retries)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - @test exception.code == "SlowDown" - @test retries[] == AWS.max_attempts(aws) - end - - @testset "Not authorized" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - message = "User is not authorized to perform: action on resource with an explicit deny" - - # Simulate the HTTP.request behaviour with a HTTP 400 response - exception = apply(Patches.gen_http_options_400_patches(message)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - - # If handled incorrectly using a `response_stream` may result in the body data being - # lost. Mainly, this is a problem when using a temporary I/O stream instead of - # writing directly to the `response_stream`. - @test exception.message == message - @test exception.streamed_body !== nothing - end - - @testset "Not authorized with BufferStream response_stream" begin - buf = Base.BufferStream() - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - response_stream=buf, - use_response_type=true, - ) - message = "User is not authorized to perform: action on resource with an explicit deny" - # Simulate the HTTP.request behaviour with a HTTP 400 response - exception = apply(Patches.gen_http_options_400_patches(message)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - @test exception isa AWSException - # If handled incorrectly using a `response_stream` may result in the body data being - # lost. Mainly, this is a problem when using a temporary I/O stream instead of - # writing directly to the `response_stream`. - @test exception.message == message - @test exception.streamed_body !== nothing - end - - @testset "read MIME-type" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - @testset "invalid content type" begin - headers = Pair["Content-Type" => ""] - body = "" - expected_body_type = Vector{UInt8} - expected_body = b"" - - r = Patches._response(; headers=headers, body=body) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "text/xml" begin - headers = Pair["Content-Type" => "text/xml"] - expected_body_type = LittleDict{Union{String,Symbol},Any} - expected_body = _expected_xml(Patches.body, expected_body_type) - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "application/xml" begin - headers = Pair["Content-Type" => "application/xml"] - expected_body_type = LittleDict{Union{String,Symbol},Any} - expected_body = _expected_xml(Patches.body, expected_body_type) - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "application/json" begin - headers = ["Content-Type" => "application/json"] - body = JSON.json( - Dict{String,Any}( - "Marker" => nothing, - "VaultList" => Any[Dict{String,Any}( - "VaultName" => "test", - "SizeInBytes" => 0, - "NumberOfArchives" => 0, - "CreationDate" => "2020-06-22T03:14:41.754Z", - "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", - "LastInventoryDate" => nothing, - )], - ), - ) - - expected_body_type = LittleDict{String,Any} - expected_body = JSON.parse(body; dicttype=expected_body_type) - - r = Patches._response(; body=body, headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "text/html" begin - headers = ["Content-Type" => "text/html"] - expected_body = Patches.body - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa String - @test content == expected_body - end - - # Note: `S3.create_multipart_upload` is an example of a response type that doesn't - # specify a Content-Type. - @testset "missing content type" begin - headers = Pair[] - body = """ - - text - """ - expected_body_type = AbstractDict - expected_body = Dict{String,Any}("body" => "text") - - r = Patches._response(; headers=headers, body=body) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - - content = parse(response, MIME"text/plain"()) - @test content isa String - @test content == body - end - end -end - -struct TestBackend <: AWS.AbstractBackend - param::Int -end - -function AWS._http_request(backend::TestBackend, ::AWS.Request, ::IO) - return backend.param -end - -@testset "HTTPBackend" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - backend=AWS.HTTPBackend(), - ) - io = IOBuffer() - - apply(Patches._http_options_patches) do - # No default options - @test isempty(AWS._http_request(request.backend, request, io)) - - # We can pass HTTP options via the backend - custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5)) - @test custom_backend isa AWS.AbstractBackend - @test AWS._http_request(custom_backend, request, io) == Dict(:connection_limit => 5) - - # We can pass options per-request - request.http_options = Dict(:pipeline_limit => 20) - @test AWS._http_request(request.backend, request, io) == Dict(:pipeline_limit => 20) - @test AWS._http_request(custom_backend, request, io) == - Dict(:pipeline_limit => 20, :connection_limit => 5) - - # per-request options override backend options: - custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5)) - @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20) - end - - request.backend = TestBackend(2) - @test AWS._http_request(request.backend, request, io) == 2 - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - backend=TestBackend(4), - ) - @test AWS._http_request(request.backend, request, io) == 4 - - # Let's test setting the default backend - prev_backend = AWS.DEFAULT_BACKEND[] - try - AWS.DEFAULT_BACKEND[] = TestBackend(3) - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - ) - @test AWS._http_request(request.backend, request, io) == 3 - finally - AWS.DEFAULT_BACKEND[] = prev_backend - end -end - -@testset "_generate_rest_resource" begin - request_uri = "/{Bucket}/{Key+}" - args = Dict{String,Any}("Bucket" => "aws.jl-test", "Key" => "Test-Key") - - expected = "/$(args["Bucket"])/$(args["Key"])" - result = AWS._generate_rest_resource(request_uri, args) - @test result == expected -end - -@testset "generate_service_url" begin - region = "us-east-2" - resource = "/aws.jl-test---timestamp" - config = AWSConfig() - config.region = region - - request = Request(; - service="service", - api_version="api_version", - request_method="GET", - resource=resource, - ) - - @testset "regionless endpoints" for regionless_endpoint in ("iam", "route53") - endpoint = "sdb" - request.service = regionless_endpoint - expected_result = "https://$regionless_endpoint.amazonaws.com$resource" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end - - @testset "region service" begin - endpoint = "sdb" - request.service = endpoint - expected_result = "https://$endpoint.$region.amazonaws.com$resource" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end - - @testset "sdb -- us-east-1 region exception" begin - endpoint = "sdb" - request.service = endpoint - expected_result = "https://$endpoint.amazonaws.com$resource" - config.region = "us-east-1" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end -end - -@testset "_flatten_query" begin - high_level_value = "high_level_value" - entry_1 = LittleDict( - "low_level_key_1" => "low_level_value_1", "low_level_key_2" => "low_level_value_2" - ) - entry_2 = LittleDict( - "low_level_key_3" => "low_level_value_3", "low_level_key_4" => "low_level_value_4" - ) - - args = LittleDict( - "high_level_key" => high_level_value, "high_level_array" => [entry_1, entry_2] - ) - - @testset "non-special case suffix" begin - service = "sts" - result = AWS._flatten_query(service, args) - - expected = Pair{String,String}[ - "high_level_key" => "high_level_value", - "high_level_array.member.1.low_level_key_1" => "low_level_value_1", - "high_level_array.member.1.low_level_key_2" => "low_level_value_2", - "high_level_array.member.2.low_level_key_3" => "low_level_value_3", - "high_level_array.member.2.low_level_key_4" => "low_level_value_4", - ] - - @test result == expected - end - - @testset "sqs - special casing suffix" begin - service = "sqs" - result = AWS._flatten_query(service, args) - - expected = Pair{String,String}[ - "high_level_key" => "high_level_value", - "high_level_array.1.low_level_key_1" => "low_level_value_1", - "high_level_array.1.low_level_key_2" => "low_level_value_2", - "high_level_array.2.low_level_key_3" => "low_level_value_3", - "high_level_array.2.low_level_key_4" => "low_level_value_4", - ] - - @test result == expected - end -end - -@testset "_clean_s3_uri" begin - uri = "/test-bucket/*)=('! +@,:.txt?list-objects=v2" - expected_uri = "/test-bucket/%2A%29%3D%28%27%21%20%2B%40%2C%3A.txt?list-objects=v2" - @test AWS._clean_s3_uri(uri) == expected_uri - - # make sure that other parts of the uri aren't changed by `_clean_s3_uri` - for uri in ( - "https://julialang.org", - "http://julialang.org", - "http://julialang.org:8080", - "/onlypath", - "/path?query= +99", - "/anchor?query=yes#anchor1", - ) - @test AWS._clean_s3_uri(uri) == uri - end -end - @testset "STS" begin @testset "high-level" begin @service STS diff --git a/test/runtests.jl b/test/runtests.jl index b16fc6d31..cb4ddf937 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -49,6 +49,16 @@ function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) end +function _fake_aws_config(; kwargs...) + # Fake AWS credentials as shown in the AWS documentation: + # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html + access_key_id = "AKIAIOSFODNN7EXAMPLE" + secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + creds = AWS.AWSCredentials(access_key_id, secret_access_key) + region = "us-example-1" + return AWSConfig(; creds, region, kwargs...) +end + testset_role(role_name) = "AWS.jl-$role_name" const RUN_UNIT_TESTS = get(ENV, "RUN_UNIT_TESTS", "true") == "true" @@ -66,14 +76,21 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") @testset "AWS.jl" begin @testset "Unit Tests" begin if RUN_UNIT_TESTS - include("unit/AWS.jl") - include("unit/AWSExceptions.jl") - include("unit/AWSMetadataUtilities.jl") - include("unit/test_pkg.jl") - include("unit/utilities.jl") - include("unit/AWSConfig.jl") - include("unit/IMDS.jl") - include("unit/AWSCredentials.jl") + # Force these tests to run without a valid AWS configuration being present + withenv( + [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., + "AWS_CONFIG_FILE" => "/dev/null", + "AWS_SHARED_CREDENTIALS_FILE" => "/dev/null", + ) do + include("unit/AWS.jl") + include("unit/AWSExceptions.jl") + include("unit/AWSMetadataUtilities.jl") + include("unit/test_pkg.jl") + include("unit/utilities.jl") + include("unit/AWSConfig.jl") + include("unit/IMDS.jl") + include("unit/AWSCredentials.jl") + end else @warn "Skipping unit tests" end diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 6f2ed9d15..679e98ff6 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -131,17 +131,15 @@ end end @testset "set global aws config" begin - creds = AWS.AWSCredentials("access-key", "secret") - region = "test region" - config = AWSConfig(; creds, region) + config = _fake_aws_config() old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing try AWS.global_aws_config(config) result = AWS.global_aws_config() - @test result.region == region - @test result.credentials == creds + @test result.region == config.region + @test result.credentials == config.credentials finally if !isnothing(old_config) AWS.aws_config[] = old_config @@ -255,3 +253,510 @@ end end end end + +@testset "submit_request" begin + aws = _fake_aws_config() + + function _expected_xml(body::AbstractString, dict_type::Type) + parsed = parse_xml(body) + return xml_dict(XMLDict.root(parsed.x), dict_type) + end + + @testset "301 redirect" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="HEAD", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + apply(Patches._aws_http_request_patch(Patches._response(; status=301))) do + @test_throws AWSException AWS.submit_request(aws, request) + end + end + + @testset "HEAD response" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="HEAD", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + response = apply(Patches._aws_http_request_patch()) do + AWS.submit_request(aws, request) + end + + # Access to response headers + @test response.response.headers == Patches.headers + @test response.response.headers isa Vector + + # Access to streaming content + @test response.io isa IO + + # Content as a string + @test String(take!(response.io)) == Patches.body + + # Backwards compatibility with those expecting an `HTTP.Response` + @test response.headers == Patches.headers + @test response.headers isa Vector + @test String(response.body) == Patches.body + end + + @testset "GET response" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + response = apply(Patches._aws_http_request_patch()) do + AWS.submit_request(aws, request) + end + + # Access to response headers + @test response.response.headers == Patches.headers + @test response.response.headers isa Vector + + # Access to streaming content + @test response.io isa IO + + # Content as a string + @test String(take!(response.io)) == Patches.body + + # Backwards compatibility with those expecting an `HTTP.Response` + @test response.headers == Patches.headers + @test response.headers isa Vector + @test String(response.body) == Patches.body + end + + @testset "Default throttling" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + + @testset "Custom throttling" begin + aws = _fake_aws_config(; max_attempts=1) + @test AWS.max_attempts(aws) == 1 + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + + @testset "Not authorized" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + message = "User is not authorized to perform: action on resource with an explicit deny" + + # Simulate the HTTP.request behaviour with a HTTP 400 response + exception = apply(Patches.gen_http_options_400_patches(message)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + + # If handled incorrectly using a `response_stream` may result in the body data being + # lost. Mainly, this is a problem when using a temporary I/O stream instead of + # writing directly to the `response_stream`. + @test exception.message == message + @test exception.streamed_body !== nothing + end + + @testset "Not authorized with BufferStream response_stream" begin + buf = Base.BufferStream() + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + response_stream=buf, + use_response_type=true, + ) + message = "User is not authorized to perform: action on resource with an explicit deny" + # Simulate the HTTP.request behaviour with a HTTP 400 response + exception = apply(Patches.gen_http_options_400_patches(message)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + @test exception isa AWSException + # If handled incorrectly using a `response_stream` may result in the body data being + # lost. Mainly, this is a problem when using a temporary I/O stream instead of + # writing directly to the `response_stream`. + @test exception.message == message + @test exception.streamed_body !== nothing + end + + @testset "read MIME-type" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + @testset "invalid content type" begin + headers = Pair["Content-Type" => ""] + body = "" + expected_body_type = Vector{UInt8} + expected_body = b"" + + r = Patches._response(; headers=headers, body=body) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "text/xml" begin + headers = Pair["Content-Type" => "text/xml"] + expected_body_type = LittleDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "application/xml" begin + headers = Pair["Content-Type" => "application/xml"] + expected_body_type = LittleDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "application/json" begin + headers = ["Content-Type" => "application/json"] + body = JSON.json( + Dict{String,Any}( + "Marker" => nothing, + "VaultList" => Any[Dict{String,Any}( + "VaultName" => "test", + "SizeInBytes" => 0, + "NumberOfArchives" => 0, + "CreationDate" => "2020-06-22T03:14:41.754Z", + "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", + "LastInventoryDate" => nothing, + )], + ), + ) + + expected_body_type = LittleDict{String,Any} + expected_body = JSON.parse(body; dicttype=expected_body_type) + + r = Patches._response(; body=body, headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "text/html" begin + headers = ["Content-Type" => "text/html"] + expected_body = Patches.body + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa String + @test content == expected_body + end + + # Note: `S3.create_multipart_upload` is an example of a response type that doesn't + # specify a Content-Type. + @testset "missing content type" begin + headers = Pair[] + body = """ + + text + """ + expected_body_type = AbstractDict + expected_body = Dict{String,Any}("body" => "text") + + r = Patches._response(; headers=headers, body=body) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + + content = parse(response, MIME"text/plain"()) + @test content isa String + @test content == body + end + end +end + +struct TestBackend <: AWS.AbstractBackend + param::Int +end + +function AWS._http_request(backend::TestBackend, ::AWS.Request, ::IO) + return backend.param +end + +@testset "HTTPBackend" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + backend=AWS.HTTPBackend(), + ) + io = IOBuffer() + + apply(Patches._http_options_patches) do + # No default options + @test isempty(AWS._http_request(request.backend, request, io)) + + # We can pass HTTP options via the backend + custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5)) + @test custom_backend isa AWS.AbstractBackend + @test AWS._http_request(custom_backend, request, io) == Dict(:connection_limit => 5) + + # We can pass options per-request + request.http_options = Dict(:pipeline_limit => 20) + @test AWS._http_request(request.backend, request, io) == Dict(:pipeline_limit => 20) + @test AWS._http_request(custom_backend, request, io) == + Dict(:pipeline_limit => 20, :connection_limit => 5) + + # per-request options override backend options: + custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5)) + @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20) + end + + request.backend = TestBackend(2) + @test AWS._http_request(request.backend, request, io) == 2 + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + backend=TestBackend(4), + ) + @test AWS._http_request(request.backend, request, io) == 4 + + # Let's test setting the default backend + prev_backend = AWS.DEFAULT_BACKEND[] + try + AWS.DEFAULT_BACKEND[] = TestBackend(3) + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + ) + @test AWS._http_request(request.backend, request, io) == 3 + finally + AWS.DEFAULT_BACKEND[] = prev_backend + end +end + +@testset "_generate_rest_resource" begin + request_uri = "/{Bucket}/{Key+}" + args = Dict{String,Any}("Bucket" => "aws.jl-test", "Key" => "Test-Key") + + expected = "/$(args["Bucket"])/$(args["Key"])" + result = AWS._generate_rest_resource(request_uri, args) + @test result == expected +end + +@testset "generate_service_url" begin + region = "us-east-2" + resource = "/aws.jl-test---timestamp" + config = _fake_aws_config(; region) + + request = Request(; + service="service", + api_version="api_version", + request_method="GET", + resource=resource, + ) + + @testset "regionless endpoints" for regionless_endpoint in ("iam", "route53") + endpoint = "sdb" + request.service = regionless_endpoint + expected_result = "https://$regionless_endpoint.amazonaws.com$resource" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end + + @testset "region service" begin + endpoint = "sdb" + request.service = endpoint + expected_result = "https://$endpoint.$region.amazonaws.com$resource" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end + + @testset "sdb -- us-east-1 region exception" begin + endpoint = "sdb" + request.service = endpoint + expected_result = "https://$endpoint.amazonaws.com$resource" + config.region = "us-east-1" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end +end + +@testset "_flatten_query" begin + high_level_value = "high_level_value" + entry_1 = LittleDict( + "low_level_key_1" => "low_level_value_1", "low_level_key_2" => "low_level_value_2" + ) + entry_2 = LittleDict( + "low_level_key_3" => "low_level_value_3", "low_level_key_4" => "low_level_value_4" + ) + + args = LittleDict( + "high_level_key" => high_level_value, "high_level_array" => [entry_1, entry_2] + ) + + @testset "non-special case suffix" begin + service = "sts" + result = AWS._flatten_query(service, args) + + expected = Pair{String,String}[ + "high_level_key" => "high_level_value", + "high_level_array.member.1.low_level_key_1" => "low_level_value_1", + "high_level_array.member.1.low_level_key_2" => "low_level_value_2", + "high_level_array.member.2.low_level_key_3" => "low_level_value_3", + "high_level_array.member.2.low_level_key_4" => "low_level_value_4", + ] + + @test result == expected + end + + @testset "sqs - special casing suffix" begin + service = "sqs" + result = AWS._flatten_query(service, args) + + expected = Pair{String,String}[ + "high_level_key" => "high_level_value", + "high_level_array.1.low_level_key_1" => "low_level_value_1", + "high_level_array.1.low_level_key_2" => "low_level_value_2", + "high_level_array.2.low_level_key_3" => "low_level_value_3", + "high_level_array.2.low_level_key_4" => "low_level_value_4", + ] + + @test result == expected + end +end + +@testset "_clean_s3_uri" begin + uri = "/test-bucket/*)=('! +@,:.txt?list-objects=v2" + expected_uri = "/test-bucket/%2A%29%3D%28%27%21%20%2B%40%2C%3A.txt?list-objects=v2" + @test AWS._clean_s3_uri(uri) == expected_uri + + # make sure that other parts of the uri aren't changed by `_clean_s3_uri` + for uri in ( + "https://julialang.org", + "http://julialang.org", + "http://julialang.org:8080", + "/onlypath", + "/path?query= +99", + "/anchor?query=yes#anchor1", + ) + @test AWS._clean_s3_uri(uri) == uri + end +end diff --git a/test/unit/AWSConfig.jl b/test/unit/AWSConfig.jl index c94551e86..44d18686a 100644 --- a/test/unit/AWSConfig.jl +++ b/test/unit/AWSConfig.jl @@ -7,7 +7,6 @@ patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) config = withenv( - [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), ) do @@ -22,16 +21,14 @@ @testset "default profile section names" begin allowed_default_sections = ["default", "profile default"] mktemp() do config_path, _ - withenv([k => nothing for k in filter(startswith("AWS_"), keys(ENV))]...) do - for default_section_str in allowed_default_sections - config = """ - [$default_section_str] - region = xx-yy-1 - """ - write(config_path, config) - region = aws_get_region(; profile="default", config=config_path) - @test region == "xx-yy-1" - end + for default_section_str in allowed_default_sections + config = """ + [$default_section_str] + region = xx-yy-1 + """ + write(config_path, config) + region = aws_get_region(; profile="default", config=config_path) + @test region == "xx-yy-1" end end end From 287e82bd656ecc96e9dc97143810a29796d0449b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:18:18 -0500 Subject: [PATCH 15/31] Refactor integration tests --- test/AWS.jl | 880 +++++++++++++++++++++++++++------------------------- 1 file changed, 452 insertions(+), 428 deletions(-) diff --git a/test/AWS.jl b/test/AWS.jl index 11f02aac5..f0004c3a7 100644 --- a/test/AWS.jl +++ b/test/AWS.jl @@ -1,541 +1,565 @@ -@testset "STS" begin - @testset "high-level" begin - @service STS +@testset "query service" begin + @testset "STS" begin + @test AWSServices.sts isa AWS.QueryService - response = STS.get_caller_identity() - d = response["GetCallerIdentityResult"] + @testset "high-level" begin + @service STS - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end - - @testset "low-level" begin - response = AWSServices.sts("GetCallerIdentity") - d = response["GetCallerIdentityResult"] + response = STS.get_caller_identity() + d = response["GetCallerIdentityResult"] - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end -end + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) + end -@testset "json" begin - @testset "high-level secrets manager" begin - @service Secrets_Manager + @testset "low-level" begin + response = AWSServices.sts("GetCallerIdentity") + d = response["GetCallerIdentityResult"] - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) + end + end - function _get_secret_string(secret_name) - response = Secrets_Manager.get_secret_value(secret_name) + @testset "IAM" begin + @test AWSServices.iam isa AWS.QueryService - return response["SecretString"] - end + @testset "high-level" begin + @service IAM - Secrets_Manager.create_secret( - secret_name, - LittleDict( - "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) - ), - ) - - try - @test _get_secret_string(secret_name) == secret_string - finally - Secrets_Manager.delete_secret( - secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], ) - end - - @test_throws AWSException _get_secret_string(secret_name) - end + expected_policy_document = JSON.json(expected_policy_document) - @testset "low-level secrets manager" begin - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" + response = IAM.create_policy(expected_policy_document, expected_policy_name) + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - function _get_secret_string(secret_name) - response = AWSServices.secrets_manager( - "GetSecretValue", LittleDict("SecretId" => secret_name) - ) + try + response_policy_version = IAM.get_policy_version(policy_arn, "v1") + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + IAM.delete_policy(policy_arn) + end - return response["SecretString"] + @test_throws AWSException IAM.get_policy(policy_arn) end - resp = AWSServices.secrets_manager( - "CreateSecret", - LittleDict( - "Name" => secret_name, - "SecretString" => secret_string, - "ClientRequestToken" => string(uuid4()), - ), - ) - - try - @test _get_secret_string(secret_name) == secret_string - finally - AWSServices.secrets_manager( - "DeleteSecret", + @testset "low-level" begin + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], + ) + expected_policy_document = JSON.json(expected_policy_document) + + response = AWSServices.iam( + "CreatePolicy", LittleDict( - "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" + "PolicyName" => expected_policy_name, + "PolicyDocument" => expected_policy_document, ), ) - end + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - @test_throws AWSException _get_secret_string(secret_name) + try + response_policy_version = AWSServices.iam( + "GetPolicyVersion", + LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), + ) + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) + end + + @test_throws AWSException AWSServices.iam( + "GetPolicy", LittleDict("PolicyArn" => policy_arn) + ) + end end -end -@testset "query" begin - @testset "high-level iam" begin - @service IAM + @testset "SQS" begin + @test AWSServices.sqs isa AWS.QueryService - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) - - response = IAM.create_policy(expected_policy_document, expected_policy_name) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = IAM.get_policy_version(policy_arn, "v1") - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - IAM.delete_policy(policy_arn) - end + @testset "high-level" begin + @service SQS - @test_throws AWSException IAM.get_policy(policy_arn) - end + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" - @testset "low-level iam" begin - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) - - response = AWSServices.iam( - "CreatePolicy", - LittleDict( - "PolicyName" => expected_policy_name, - "PolicyDocument" => expected_policy_document, - ), - ) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = AWSServices.iam( - "GetPolicyVersion", - LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), - ) - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) - end + function _get_queue_url(queue_name) + result = SQS.get_queue_url(queue_name) - @test_throws AWSException AWSServices.iam( - "GetPolicy", LittleDict("PolicyArn" => policy_arn) - ) - end + return result["QueueUrl"] + end - @testset "high-level sqs" begin - @service SQS + # Create Queue + SQS.create_queue(queue_name) + queue_url = _get_queue_url(queue_name) - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" + try + # Get Queues + @test !isempty(queue_url) - function _get_queue_url(queue_name) - result = SQS.get_queue_url(queue_name) + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" - return result["QueueUrl"] + SQS.send_message(expected_message, queue_url) + + response = SQS.receive_message(queue_url) + receipt_handle = only(response["Messages"])["ReceiptHandle"] + + response = SQS.delete_message_batch( + [ + LittleDict( + "Id" => expected_message_id, "ReceiptHandle" => receipt_handle + ), + ], + queue_url, + ) + + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id + + SQS.send_message(expected_message, queue_url) + + result = SQS.receive_message(queue_url) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + SQS.delete_queue(queue_url) + end + + @test_throws AWSException _get_queue_url(queue_name) end - # Create Queue - SQS.create_queue(queue_name) - queue_url = _get_queue_url(queue_name) + @testset "low-level" begin + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" + + function _get_queue_url(queue_name) + result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) + + return result["QueueUrl"] + end + + # Create Queue + AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) - try - # Get Queues + queue_url = _get_queue_url(queue_name) @test !isempty(queue_url) - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" + try + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" - SQS.send_message(expected_message, queue_url) + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) - response = SQS.receive_message(queue_url) - receipt_handle = only(response["Messages"])["ReceiptHandle"] + response = AWSServices.sqs( + "ReceiveMessage", LittleDict("QueueUrl" => queue_url) + ) + receipt_handle = only(response["Messages"])["ReceiptHandle"] - response = SQS.delete_message_batch( - [ + response = AWSServices.sqs( + "DeleteMessageBatch", LittleDict( - "Id" => expected_message_id, "ReceiptHandle" => receipt_handle + "QueueUrl" => queue_url, + "Entries" => [ + LittleDict( + "Id" => expected_message_id, + "ReceiptHandle" => receipt_handle, + ), + ], ), - ], - queue_url, - ) + ) + + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id + + # Send message + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) + + # Receive Message + result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) + end - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id + @test_throws AWSException _get_queue_url(queue_name) + end + end +end - SQS.send_message(expected_message, queue_url) +@testset "JSON service" begin + @testset "Secrets Manager" + @test AWSServices.secrets_manager isa AWS.JSONService - result = SQS.receive_message(queue_url) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - SQS.delete_queue(queue_url) - end + @testset "high-level" begin + @service Secrets_Manager - @test_throws AWSException _get_queue_url(queue_name) - end + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" - @testset "low-level sqs" begin - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" + function _get_secret_string(secret_name) + response = Secrets_Manager.get_secret_value(secret_name) - function _get_queue_url(queue_name) - result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) + return response["SecretString"] + end - return result["QueueUrl"] - end + Secrets_Manager.create_secret( + secret_name, + LittleDict( + "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) + ), + ) - # Create Queue - AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) + try + @test _get_secret_string(secret_name) == secret_string + finally + Secrets_Manager.delete_secret( + secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") + ) + end - queue_url = _get_queue_url(queue_name) - @test !isempty(queue_url) + @test_throws AWSException _get_secret_string(secret_name) + end - try - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" + @testset "low-level" begin + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) + function _get_secret_string(secret_name) + response = AWSServices.secrets_manager( + "GetSecretValue", LittleDict("SecretId" => secret_name) + ) - response = AWSServices.sqs( - "ReceiveMessage", LittleDict("QueueUrl" => queue_url) - ) - receipt_handle = only(response["Messages"])["ReceiptHandle"] + return response["SecretString"] + end - response = AWSServices.sqs( - "DeleteMessageBatch", + resp = AWSServices.secrets_manager( + "CreateSecret", LittleDict( - "QueueUrl" => queue_url, - "Entries" => [ - LittleDict( - "Id" => expected_message_id, - "ReceiptHandle" => receipt_handle, - ), - ], + "Name" => secret_name, + "SecretString" => secret_string, + "ClientRequestToken" => string(uuid4()), ), ) - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id - - # Send message - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) + try + @test _get_secret_string(secret_name) == secret_string + finally + AWSServices.secrets_manager( + "DeleteSecret", + LittleDict( + "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" + ), + ) + end - # Receive Message - result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) + @test_throws AWSException _get_secret_string(secret_name) end - - @test_throws AWSException _get_queue_url(queue_name) end end -@testset "rest-xml" begin - @testset "high-level s3" begin - @service S3 - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = string(uuid4()) - function _bucket_exists(bucket_name) - try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) +@testset "Rest XML" begin + @testset "S3" begin + @test AWSServices.s3 isa RestXMLService + + @testset "high-level" begin + @service S3 + + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = string(uuid4()) + + function _bucket_exists(bucket_name) + try + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end end end - end - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) - @test length([result["Contents"]]) == max_keys - - # GET with an IO target - mktemp() do f, io - S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) - flush(io) - @test read(f, String) == body - end - finally - # DELETE with parameters operation - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) + # HEAD operation + @test _bucket_exists(bucket_name) == false - # DELETE operation - S3.delete_bucket(bucket_name) + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) - sleep(2) + try + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # GET with an IO target + mktemp() do f, io + S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) + flush(io) + @test read(f, String) == body + end + finally + # DELETE with parameters operation + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) + + # DELETE operation + S3.delete_bucket(bucket_name) + + sleep(2) + end + + @test _bucket_exists(bucket_name) == false end - @test _bucket_exists(bucket_name) == false - end + @testset "low-level" begin + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = "*)=('! +@,:.txt" # Special characters which S3 allows + + function _bucket_exists(bucket_name) + try + AWSServices.s3("HEAD", "/$bucket_name") + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end + end + end - @testset "low-level s3" begin - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = "*)=('! +@,:.txt" # Special characters which S3 allows + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + AWSServices.s3("PUT", "/$bucket_name") + @test _bucket_exists(bucket_name) - function _bucket_exists(bucket_name) try - AWSServices.s3("HEAD", "/$bucket_name") - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end + # PUT with parameters operation + body = Array{UInt8}("sample-file-body") + AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) + @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body + + # GET operation + result = AWSServices.s3("GET", "/$bucket_name") + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # POST with parameters operation + body = """ + + + $file_name + + + """ + + AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) + @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") + finally + # DELETE operation + AWSServices.s3("DELETE", "/$bucket_name") + + sleep(2) end - end - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - AWSServices.s3("PUT", "/$bucket_name") - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = Array{UInt8}("sample-file-body") - AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) - @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body - - # GET operation - result = AWSServices.s3("GET", "/$bucket_name") - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) - @test length([result["Contents"]]) == max_keys - - # POST with parameters operation - body = """ - - - $file_name - - - """ - - AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) - @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") - finally - # DELETE operation - AWSServices.s3("DELETE", "/$bucket_name") - - sleep(2) + @test _bucket_exists(bucket_name) == false end - @test _bucket_exists(bucket_name) == false - end + @testset "additional operations" begin + @service S3 - @testset "additional S3 operations" begin - @service S3 + bucket_name = "aws-jl-test---" * _now_formatted() - bucket_name = "aws-jl-test---" * _now_formatted() + # Testing a file name with various special & Unicode characters + file_name = "$(uuid4())/📁!!/@ +*" - # Testing a file name with various special & Unicode characters - file_name = "$(uuid4())/📁!!/@ +*" + function _bucket_exists(bucket_name) + try + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end + end + end + + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) - function _bucket_exists(bucket_name) try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + finally + # DELETE the file, check that it's gone, and then DELETE the bucket + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) + S3.delete_bucket(bucket_name) + sleep(2) end - end - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - finally - # DELETE the file, check that it's gone, and then DELETE the bucket - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) - S3.delete_bucket(bucket_name) - sleep(2) + @test _bucket_exists(bucket_name) == false end - - @test _bucket_exists(bucket_name) == false end end -@testset "rest-json" begin - @testset "high-level glacier" begin - @service Glacier +@testset "Rest JSON" begin + @testset "Glacier" begin + @test AWSServices.glacier isa RestJSONService - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] - - # PUT - for vault in vault_names - Glacier.create_vault("-", vault) - end + @testset "high-level" begin + @service Glacier - try - # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + # PUT for vault in vault_names - Glacier.add_tags_to_vault("-", vault, tags) + Glacier.create_vault("-", vault) end - for vault in vault_names - result_tags = Glacier.list_tags_for_vault("-", vault) - @test result_tags == tags - end + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - args = LittleDict("limit" => limit) - result = Glacier.list_vaults("-", args) - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - Glacier.delete_vault("-", vault) - end - end + for vault in vault_names + Glacier.add_tags_to_vault("-", vault, tags) + end - result = Glacier.list_vaults("-") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] + for vault in vault_names + result_tags = Glacier.list_tags_for_vault("-", vault) + @test result_tags == tags + end - for vault in vault_names - @test !(vault in res_vault_names) - end - end + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + args = LittleDict("limit" => limit) + result = Glacier.list_vaults("-", args) + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE + for vault in vault_names + Glacier.delete_vault("-", vault) + end + end - @testset "low-level glacier" begin - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + result = Glacier.list_vaults("-") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] - # PUT - for vault in vault_names - AWSServices.glacier("PUT", "/-/vaults/$vault") + for vault in vault_names + @test !(vault in res_vault_names) + end end - try - # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + @testset "low-level" begin + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + # PUT for vault in vault_names - AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) + AWSServices.glacier("PUT", "/-/vaults/$vault") end - for vault in vault_names - result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - @test result_tags == tags - end + for vault in vault_names + AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) + end - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - params = LittleDict("limit" => limit) - result = AWSServices.glacier("GET", "/-/vaults/", params) + for vault in vault_names + result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - AWSServices.glacier("DELETE", "/-/vaults/$vault") + @test result_tags == tags + end + + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + params = LittleDict("limit" => limit) + result = AWSServices.glacier("GET", "/-/vaults/", params) + + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE + for vault in vault_names + AWSServices.glacier("DELETE", "/-/vaults/$vault") + end end - end - result = AWSServices.glacier("GET", "/-/vaults") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] + result = AWSServices.glacier("GET", "/-/vaults") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] - for vault in vault_names - @test !(vault in res_vault_names) + for vault in vault_names + @test !(vault in res_vault_names) + end end end end From dbe72d885774ac9682508fae65f0ab54351d58cb Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:23:18 -0500 Subject: [PATCH 16/31] Move integration tests --- test/{ => integration}/AWS.jl | 0 test/{ => integration}/AWSCredentials.jl | 0 test/{ => integration}/minio.jl | 0 test/{ => integration}/role.jl | 0 test/runtests.jl | 19 ++++++++++--------- 5 files changed, 10 insertions(+), 9 deletions(-) rename test/{ => integration}/AWS.jl (100%) rename test/{ => integration}/AWSCredentials.jl (100%) rename test/{ => integration}/minio.jl (100%) rename test/{ => integration}/role.jl (100%) diff --git a/test/AWS.jl b/test/integration/AWS.jl similarity index 100% rename from test/AWS.jl rename to test/integration/AWS.jl diff --git a/test/AWSCredentials.jl b/test/integration/AWSCredentials.jl similarity index 100% rename from test/AWSCredentials.jl rename to test/integration/AWSCredentials.jl diff --git a/test/minio.jl b/test/integration/minio.jl similarity index 100% rename from test/minio.jl rename to test/integration/minio.jl diff --git a/test/role.jl b/test/integration/role.jl similarity index 100% rename from test/role.jl rename to test/integration/role.jl diff --git a/test/runtests.jl b/test/runtests.jl index cb4ddf937..c87d846ee 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -41,10 +41,6 @@ Mocking.activate() include("patch.jl") include("resources/totp.jl") -const TEST_MINIO = begin - all(k -> haskey(ENV, k), ("MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MINIO_REGION_NAME")) -end - function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) end @@ -63,6 +59,9 @@ testset_role(role_name) = "AWS.jl-$role_name" const RUN_UNIT_TESTS = get(ENV, "RUN_UNIT_TESTS", "true") == "true" const RUN_INTEGRATION_TESTS = get(ENV, "RUN_INTEGRATION_TESTS", "true") == "true" +const RUN_MINIO_INTEGRATION_TESTS = begin + all(k -> haskey(ENV, k), ("MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MINIO_REGION_NAME")) +end # Avoid the situation where we all tests are silently skipped. Most likely due to the wrong # environmental variables being used. @@ -104,13 +103,15 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") backends = [AWS.HTTPBackend, AWS.DownloadsBackend] @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() - include("AWS.jl") - include("AWSCredentials.jl") - include("role.jl") + include("integration/AWS.jl") + include("integration/AWSCredentials.jl") + include("integration/role.jl") include("issues.jl") - if TEST_MINIO - include("minio.jl") + if RUN_MINIO_INTEGRATION_TESTS + include("integration/minio.jl") + else + @warn "Skipping MinIO integration tests" end end else From 07824e15fc5706360471cee099b7c382f4a83d01 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:34:56 -0500 Subject: [PATCH 17/31] Wrapping up --- test/{ => integration}/issues.jl | 76 -------------------------------- test/runtests.jl | 13 +----- test/unit/AWS.jl | 16 ++++--- 3 files changed, 12 insertions(+), 93 deletions(-) rename test/{ => integration}/issues.jl (61%) diff --git a/test/issues.jl b/test/integration/issues.jl similarity index 61% rename from test/issues.jl rename to test/integration/issues.jl index f0821da76..61542b725 100644 --- a/test/issues.jl +++ b/test/integration/issues.jl @@ -151,82 +151,6 @@ try S3.delete_object(BUCKET_NAME, file_name) end end - - # https://github.com/JuliaCloud/AWS.jl/issues/515 - @testset "issue 515" begin - function _incomplete_patch(; data, num_attempts_to_fail=4) - attempt_num = 0 - n = length(data) - - function _downloads_response(content_length) - headers = ["content-length" => string(content_length)] - return Downloads.Response("http", "", 200, "HTTP/1.1 200 OK", headers) - end - - patch = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend - @patch function HTTP.request(args...; response_stream, kwargs...) - attempt_num += 1 - if attempt_num <= num_attempts_to_fail - write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained - throw(HTTP.RequestError(HTTP.Request(), EOFError())) - else - write(response_stream, data) - return HTTP.Response(200, "{\"Location\": \"us-east-1\"}") - end - end - elseif AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend - @patch function Downloads.request(args...; output, kwargs...) - attempt_num += 1 - if attempt_num <= num_attempts_to_fail - write(output, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained - message = "transfer closed with 1 bytes remaining to read" - e = Downloads.RequestError("", 18, message, _downloads_response(n)) - throw(e) - else - write(output, data) - return _downloads_response(n) - end - end - end - - return patch - end - - n = 100 - data = rand(UInt8, n) - bucket = "julialang2" # use public bucket as dummy - key = "bin/versions.json" - config = AWSConfig(; creds=nothing) - - @testset "Fail 2 attempts then succeed" begin - apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do - retrieved = S3.get_object(bucket, key; aws_config=config) - - @test length(retrieved) == n - @test retrieved == data - end - end - - @testset "Fail all 4 attempts then throw" begin - err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend - HTTP.RequestError - else - Downloads.RequestError - end - io = IOBuffer() - - apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do - params = Dict("response_stream" => io) - @test_throws err_t S3.get_object(bucket, key, params; aws_config=config) - - seekstart(io) - retrieved = read(io) - @test length(retrieved) == n - 1 - @test retrieved == data[1:(n - 1)] - end - end - end - finally S3.delete_bucket(BUCKET_NAME) end diff --git a/test/runtests.jl b/test/runtests.jl index c87d846ee..5f862376d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -45,16 +45,6 @@ function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) end -function _fake_aws_config(; kwargs...) - # Fake AWS credentials as shown in the AWS documentation: - # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html - access_key_id = "AKIAIOSFODNN7EXAMPLE" - secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - creds = AWS.AWSCredentials(access_key_id, secret_access_key) - region = "us-example-1" - return AWSConfig(; creds, region, kwargs...) -end - testset_role(role_name) = "AWS.jl-$role_name" const RUN_UNIT_TESTS = get(ENV, "RUN_UNIT_TESTS", "true") == "true" @@ -89,6 +79,7 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") include("unit/AWSConfig.jl") include("unit/IMDS.jl") include("unit/AWSCredentials.jl") + include("unit/issues.jl") end else @warn "Skipping unit tests" @@ -106,7 +97,7 @@ const RESOURCE_DIR = joinpath(@__DIR__, "resources") include("integration/AWS.jl") include("integration/AWSCredentials.jl") include("integration/role.jl") - include("issues.jl") + include("integration/issues.jl") if RUN_MINIO_INTEGRATION_TESTS include("integration/minio.jl") diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 679e98ff6..fb37e28f6 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -113,8 +113,12 @@ end @testset "global config, kwargs" begin - creds = AWS.AWSCredentials("access-key", "secret") - region = "us-east-2" + # Fake AWS credentials as shown in the AWS documentation: + # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html + access_key_id = "AKIAIOSFODNN7EXAMPLE" + secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + creds = AWS.AWSCredentials(access_key_id, secret_access_key) + region = "us-example-1" old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing try @@ -131,7 +135,7 @@ end end @testset "set global aws config" begin - config = _fake_aws_config() + config = AWSConfig(; creds=nothing, region="us-example-2") old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing try @@ -255,7 +259,7 @@ end end @testset "submit_request" begin - aws = _fake_aws_config() + aws = AWSConfig(; creds=nothing) function _expected_xml(body::AbstractString, dict_type::Type) parsed = parse_xml(body) @@ -362,7 +366,7 @@ end end @testset "Custom throttling" begin - aws = _fake_aws_config(; max_attempts=1) + aws = AWSConfig(; creds=nothing, max_attempts=1) @test AWS.max_attempts(aws) == 1 request = Request(; @@ -661,7 +665,7 @@ end @testset "generate_service_url" begin region = "us-east-2" resource = "/aws.jl-test---timestamp" - config = _fake_aws_config(; region) + config = AWSConfig(; region, creds=nothing) request = Request(; service="service", From 1ad838e09e17121cee8594d679828f46c03c4d64 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:37:09 -0500 Subject: [PATCH 18/31] fixup --- test/integration/AWS.jl | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/integration/AWS.jl b/test/integration/AWS.jl index f0004c3a7..19ecc9346 100644 --- a/test/integration/AWS.jl +++ b/test/integration/AWS.jl @@ -158,7 +158,9 @@ expected_message = "Hello for AWS.jl" function _get_queue_url(queue_name) - result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) + result = AWSServices.sqs( + "GetQueueUrl", LittleDict("QueueName" => queue_name) + ) return result["QueueUrl"] end @@ -206,7 +208,9 @@ ) # Receive Message - result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) + result = AWSServices.sqs( + "ReceiveMessage", LittleDict("QueueUrl" => queue_url) + ) message = only(result["Messages"])["Body"] @test message == expected_message finally @@ -219,7 +223,7 @@ end @testset "JSON service" begin - @testset "Secrets Manager" + @testset "Secrets Manager" begin @test AWSServices.secrets_manager isa AWS.JSONService @testset "high-level" begin @@ -289,8 +293,6 @@ end end end - - @testset "Rest XML" begin @testset "S3" begin @test AWSServices.s3 isa RestXMLService @@ -392,7 +394,9 @@ end # GET with parameters operation max_keys = 1 - result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) + result = AWSServices.s3( + "GET", "/$bucket_name", Dict("max_keys" => max_keys) + ) @test length([result["Contents"]]) == max_keys # POST with parameters operation @@ -483,7 +487,9 @@ end try # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + tags = Dict( + "Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02") + ) for vault in vault_names Glacier.add_tags_to_vault("-", vault, tags) @@ -527,7 +533,9 @@ end try # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + tags = Dict( + "Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02") + ) for vault in vault_names AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) From 88bae50c2ec5a060207f1759b4e568e22a6218a9 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:42:12 -0500 Subject: [PATCH 19/31] fixup --- test/unit/AWS.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index fb37e28f6..49c6e9c17 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -168,8 +168,8 @@ end access_key = "access-key" secret_key = "ssh... it is a secret" - creds = AWS.AWSCredentials(access_key, secret_key) - aws = AWS.AWSConfig(; creds, region="us-east-1") + creds = AWSCredentials(access_key, secret_key) + aws = AWSConfig(; creds, region="us-east-1") time = DateTime(2020) date = Dates.format(time, dateformat"yyyymmdd") @@ -259,7 +259,7 @@ end end @testset "submit_request" begin - aws = AWSConfig(; creds=nothing) + aws = AWSConfig(; creds=nothing, region="us-east-1") function _expected_xml(body::AbstractString, dict_type::Type) parsed = parse_xml(body) @@ -366,7 +366,7 @@ end end @testset "Custom throttling" begin - aws = AWSConfig(; creds=nothing, max_attempts=1) + aws = AWSConfig(; creds=nothing, region="us-east-1", max_attempts=1) @test AWS.max_attempts(aws) == 1 request = Request(; @@ -665,7 +665,7 @@ end @testset "generate_service_url" begin region = "us-east-2" resource = "/aws.jl-test---timestamp" - config = AWSConfig(; region, creds=nothing) + config = AWSConfig(; creds=nothing, region) request = Request(; service="service", From 04a4495f5a3f1faf82cc1c3e12168732d0d53f05 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:48:34 -0500 Subject: [PATCH 20/31] Rename other test dirs --- test/{configs => config}/default-role/config | 0 test/{configs => config}/default-role/credentials | 0 test/{configs => config}/role-with-mfa/config | 0 test/{configs => config}/role-with-mfa/credentials | 0 test/{resources => resource}/Manifest.toml | 0 test/{resources => resource}/Project.toml | 0 test/{resources => resource}/TestPkg/Project.toml | 0 test/{resources => resource}/TestPkg/src/TestPkg.jl | 0 test/{resources => resource}/aws_jl_test.yaml | 0 test/{resources => resource}/operations.json | 0 test/{resources => resource}/services.json | 0 test/{resources => resource}/setup.jl | 0 test/{resources => resource}/shapes.json | 0 test/{resources => resource}/totp.jl | 0 test/runtests.jl | 1 - test/unit/AWSCredentials.jl | 4 ++-- test/unit/AWSMetadataUtilities.jl | 8 ++++---- test/unit/test_pkg.jl | 2 +- 18 files changed, 7 insertions(+), 8 deletions(-) rename test/{configs => config}/default-role/config (100%) rename test/{configs => config}/default-role/credentials (100%) rename test/{configs => config}/role-with-mfa/config (100%) rename test/{configs => config}/role-with-mfa/credentials (100%) rename test/{resources => resource}/Manifest.toml (100%) rename test/{resources => resource}/Project.toml (100%) rename test/{resources => resource}/TestPkg/Project.toml (100%) rename test/{resources => resource}/TestPkg/src/TestPkg.jl (100%) rename test/{resources => resource}/aws_jl_test.yaml (100%) rename test/{resources => resource}/operations.json (100%) rename test/{resources => resource}/services.json (100%) rename test/{resources => resource}/setup.jl (100%) rename test/{resources => resource}/shapes.json (100%) rename test/{resources => resource}/totp.jl (100%) diff --git a/test/configs/default-role/config b/test/config/default-role/config similarity index 100% rename from test/configs/default-role/config rename to test/config/default-role/config diff --git a/test/configs/default-role/credentials b/test/config/default-role/credentials similarity index 100% rename from test/configs/default-role/credentials rename to test/config/default-role/credentials diff --git a/test/configs/role-with-mfa/config b/test/config/role-with-mfa/config similarity index 100% rename from test/configs/role-with-mfa/config rename to test/config/role-with-mfa/config diff --git a/test/configs/role-with-mfa/credentials b/test/config/role-with-mfa/credentials similarity index 100% rename from test/configs/role-with-mfa/credentials rename to test/config/role-with-mfa/credentials diff --git a/test/resources/Manifest.toml b/test/resource/Manifest.toml similarity index 100% rename from test/resources/Manifest.toml rename to test/resource/Manifest.toml diff --git a/test/resources/Project.toml b/test/resource/Project.toml similarity index 100% rename from test/resources/Project.toml rename to test/resource/Project.toml diff --git a/test/resources/TestPkg/Project.toml b/test/resource/TestPkg/Project.toml similarity index 100% rename from test/resources/TestPkg/Project.toml rename to test/resource/TestPkg/Project.toml diff --git a/test/resources/TestPkg/src/TestPkg.jl b/test/resource/TestPkg/src/TestPkg.jl similarity index 100% rename from test/resources/TestPkg/src/TestPkg.jl rename to test/resource/TestPkg/src/TestPkg.jl diff --git a/test/resources/aws_jl_test.yaml b/test/resource/aws_jl_test.yaml similarity index 100% rename from test/resources/aws_jl_test.yaml rename to test/resource/aws_jl_test.yaml diff --git a/test/resources/operations.json b/test/resource/operations.json similarity index 100% rename from test/resources/operations.json rename to test/resource/operations.json diff --git a/test/resources/services.json b/test/resource/services.json similarity index 100% rename from test/resources/services.json rename to test/resource/services.json diff --git a/test/resources/setup.jl b/test/resource/setup.jl similarity index 100% rename from test/resources/setup.jl rename to test/resource/setup.jl diff --git a/test/resources/shapes.json b/test/resource/shapes.json similarity index 100% rename from test/resources/shapes.json rename to test/resource/shapes.json diff --git a/test/resources/totp.jl b/test/resource/totp.jl similarity index 100% rename from test/resources/totp.jl rename to test/resource/totp.jl diff --git a/test/runtests.jl b/test/runtests.jl index 5f862376d..0d6df33fd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -60,7 +60,6 @@ if !RUN_UNIT_TESTS && !RUN_INTEGRATION_TESTS end const AWS_CONFIG = Ref{AbstractAWSConfig}() -const RESOURCE_DIR = joinpath(@__DIR__, "resources") @testset "AWS.jl" begin @testset "Unit Tests" begin diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index 3fbf744ea..94a37660b 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -125,7 +125,7 @@ end @testset "default profile" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "..", "configs", "default-role") + config_dir = joinpath(@__DIR__, "..", "config", "default-role") patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) @@ -146,7 +146,7 @@ end @testset "profile with role and MFA" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "..", "configs", "role-with-mfa") + config_dir = joinpath(@__DIR__, "..", "config", "role-with-mfa") mfa_token = "123456" sent_token = Ref("") diff --git a/test/unit/AWSMetadataUtilities.jl b/test/unit/AWSMetadataUtilities.jl index 9d2d5a455..a075eddfd 100644 --- a/test/unit/AWSMetadataUtilities.jl +++ b/test/unit/AWSMetadataUtilities.jl @@ -71,7 +71,7 @@ end end @testset "_generate_low_level_definitions" begin - services = JSON.parsefile(joinpath(RESOURCE_DIR, "services.json")) + services = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "services.json")) @testset "rest-xml" begin expected = "const s3 = AWS.RestXMLService(\"s3\", \"s3\", \"2006-03-01\")" @@ -251,7 +251,7 @@ end end @testset "_get_function_parameters" begin - shapes = JSON.parsefile(joinpath(RESOURCE_DIR, "shapes.json")) + shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "shapes.json")) @testset "required params" begin input = "RequiredParams" @@ -310,8 +310,8 @@ end @testset "_generate_high_level_definitions" begin service_name = "sample_service" protocol = "rest-xml" - operations = JSON.parsefile(joinpath(RESOURCE_DIR, "operations.json")) - shapes = JSON.parsefile(joinpath(RESOURCE_DIR, "shapes.json")) + operations = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "operations.json")) + shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resources", "shapes.json")) expected_result = """ \"\"\" diff --git a/test/unit/test_pkg.jl b/test/unit/test_pkg.jl index 56af2a237..ef3584080 100644 --- a/test/unit/test_pkg.jl +++ b/test/unit/test_pkg.jl @@ -1,4 +1,4 @@ -path = joinpath(RESOURCE_DIR, "TestPkg") +path = joinpath(@__DIR__, "..", "resource", "TestPkg") Pkg.develop(; path=path) VERSION >= v"1.8" && Pkg.precompile("TestPkg") From fc95c3c81ddd0d63eaaf6802b60d0989fd9d9071 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:49:33 -0500 Subject: [PATCH 21/31] Add forgotten file --- test/unit/issues.jl | 76 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/unit/issues.jl diff --git a/test/unit/issues.jl b/test/unit/issues.jl new file mode 100644 index 000000000..60fbc8bfb --- /dev/null +++ b/test/unit/issues.jl @@ -0,0 +1,76 @@ +@service S3 + +# https://github.com/JuliaCloud/AWS.jl/issues/515 +@testset "issue 515" begin + function _incomplete_patch(; data, num_attempts_to_fail=4) + attempt_num = 0 + n = length(data) + + function _downloads_response(content_length) + headers = ["content-length" => string(content_length)] + return Downloads.Response("http", "", 200, "HTTP/1.1 200 OK", headers) + end + + patch = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend + @patch function HTTP.request(args...; response_stream, kwargs...) + attempt_num += 1 + if attempt_num <= num_attempts_to_fail + write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained + throw(HTTP.RequestError(HTTP.Request(), EOFError())) + else + write(response_stream, data) + return HTTP.Response(200, "{\"Location\": \"us-east-1\"}") + end + end + elseif AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend + @patch function Downloads.request(args...; output, kwargs...) + attempt_num += 1 + if attempt_num <= num_attempts_to_fail + write(output, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained + message = "transfer closed with 1 bytes remaining to read" + e = Downloads.RequestError("", 18, message, _downloads_response(n)) + throw(e) + else + write(output, data) + return _downloads_response(n) + end + end + end + + return patch + end + + n = 100 + data = rand(UInt8, n) + bucket = "julialang2" # use public bucket as dummy + key = "bin/versions.json" + config = AWSConfig(; creds=nothing, region="us-east-1") + + @testset "Fail 2 attempts then succeed" begin + apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do + retrieved = S3.get_object(bucket, key; aws_config=config) + + @test length(retrieved) == n + @test retrieved == data + end + end + + @testset "Fail all 4 attempts then throw" begin + err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend + HTTP.RequestError + else + Downloads.RequestError + end + io = IOBuffer() + + apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do + params = Dict("response_stream" => io) + @test_throws err_t S3.get_object(bucket, key, params; aws_config=config) + + seekstart(io) + retrieved = read(io) + @test length(retrieved) == n - 1 + @test retrieved == data[1:(n - 1)] + end + end +end From 6c0271334d2d41462f2fdf5a54387afc0751512b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:51:07 -0500 Subject: [PATCH 22/31] fixup --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 0d6df33fd..7328bac48 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,7 +39,7 @@ using StableRNGs Mocking.activate() include("patch.jl") -include("resources/totp.jl") +include("resource/totp.jl") function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) From 695ecd4be990cf8a8ecf8944a6103a6b2e58261e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:56:24 -0500 Subject: [PATCH 23/31] fixup! fixup --- test/unit/AWSConfig.jl | 2 +- test/unit/AWSMetadataUtilities.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/AWSConfig.jl b/test/unit/AWSConfig.jl index 44d18686a..7bcd6d9a8 100644 --- a/test/unit/AWSConfig.jl +++ b/test/unit/AWSConfig.jl @@ -1,7 +1,7 @@ @testset "AWSConfig" begin @testset "default profile assumes role" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "..", "configs", "default-role") + config_dir = joinpath(@__DIR__, "..", "config", "default-role") # Avoid calling out to STS with invalid credentials patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) diff --git a/test/unit/AWSMetadataUtilities.jl b/test/unit/AWSMetadataUtilities.jl index a075eddfd..e138ec996 100644 --- a/test/unit/AWSMetadataUtilities.jl +++ b/test/unit/AWSMetadataUtilities.jl @@ -311,7 +311,7 @@ end service_name = "sample_service" protocol = "rest-xml" operations = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "operations.json")) - shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resources", "shapes.json")) + shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "shapes.json")) expected_result = """ \"\"\" From fb1579f868f6833fbd78d6774eeab6ca06021eaa Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 13:59:36 -0500 Subject: [PATCH 24/31] Drop AWS_CONFIG ref --- test/integration/AWSCredentials.jl | 5 +++-- test/integration/role.jl | 2 +- test/runtests.jl | 4 ---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/test/integration/AWSCredentials.jl b/test/integration/AWSCredentials.jl index 9070533e2..b1e790305 100644 --- a/test/integration/AWSCredentials.jl +++ b/test/integration/AWSCredentials.jl @@ -1,7 +1,8 @@ @testset "Load Credentials" begin - user = aws_user_arn(AWS_CONFIG[]) + config = global_aws_config() + user = aws_user_arn(config) @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) - AWS_CONFIG[].region = "us-east-1" + config.region = "us-east-1" @test_ecode("InvalidAction", AWSServices.iam("GetFoo")) diff --git a/test/integration/role.jl b/test/integration/role.jl index 310923fb9..d4fb83c3e 100644 --- a/test/integration/role.jl +++ b/test/integration/role.jl @@ -48,7 +48,7 @@ end @testset "assume_role / assume_role_creds" begin # In order to mitigate the effects of using `assume_role` in order to test itself we'll # use the lowest-level call with as many defaults as possible. - base_config = AWS_CONFIG[] + base_config = global_aws_config() creds = assume_role_creds(base_config, testset_role("AssumeRoleTestset")) config = AWSConfig(; creds) @test get_assumed_role(config) == testset_role("AssumeRoleTestset") diff --git a/test/runtests.jl b/test/runtests.jl index 7328bac48..9850a58f4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -59,8 +59,6 @@ if !RUN_UNIT_TESTS && !RUN_INTEGRATION_TESTS error("All tests have been disabled") end -const AWS_CONFIG = Ref{AbstractAWSConfig}() - @testset "AWS.jl" begin @testset "Unit Tests" begin if RUN_UNIT_TESTS @@ -88,8 +86,6 @@ const AWS_CONFIG = Ref{AbstractAWSConfig}() # TODO: Some of these tests are actually unit tests and need to be refactored @testset "Integration Tests" begin if RUN_INTEGRATION_TESTS - AWS_CONFIG[] = AWSConfig() - backends = [AWS.HTTPBackend, AWS.DownloadsBackend] @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() From a1b772e5e91038c0a4cb1276e7d017209094fd97 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 14:00:58 -0500 Subject: [PATCH 25/31] Revert AWS integration tests --- test/integration/AWS.jl | 888 +++++++++++++++++++--------------------- 1 file changed, 428 insertions(+), 460 deletions(-) diff --git a/test/integration/AWS.jl b/test/integration/AWS.jl index 19ecc9346..11f02aac5 100644 --- a/test/integration/AWS.jl +++ b/test/integration/AWS.jl @@ -1,573 +1,541 @@ -@testset "query service" begin - @testset "STS" begin - @test AWSServices.sts isa AWS.QueryService +@testset "STS" begin + @testset "high-level" begin + @service STS - @testset "high-level" begin - @service STS + response = STS.get_caller_identity() + d = response["GetCallerIdentityResult"] - response = STS.get_caller_identity() - d = response["GetCallerIdentityResult"] - - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) + end - @testset "low-level" begin - response = AWSServices.sts("GetCallerIdentity") - d = response["GetCallerIdentityResult"] + @testset "low-level" begin + response = AWSServices.sts("GetCallerIdentity") + d = response["GetCallerIdentityResult"] - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) end +end - @testset "IAM" begin - @test AWSServices.iam isa AWS.QueryService +@testset "json" begin + @testset "high-level secrets manager" begin + @service Secrets_Manager - @testset "high-level" begin - @service IAM + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) + function _get_secret_string(secret_name) + response = Secrets_Manager.get_secret_value(secret_name) - response = IAM.create_policy(expected_policy_document, expected_policy_name) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = IAM.get_policy_version(policy_arn, "v1") - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - IAM.delete_policy(policy_arn) - end - - @test_throws AWSException IAM.get_policy(policy_arn) + return response["SecretString"] end - @testset "low-level" begin - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) - - response = AWSServices.iam( - "CreatePolicy", - LittleDict( - "PolicyName" => expected_policy_name, - "PolicyDocument" => expected_policy_document, - ), - ) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = AWSServices.iam( - "GetPolicyVersion", - LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), - ) - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) - end - - @test_throws AWSException AWSServices.iam( - "GetPolicy", LittleDict("PolicyArn" => policy_arn) + Secrets_Manager.create_secret( + secret_name, + LittleDict( + "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) + ), + ) + + try + @test _get_secret_string(secret_name) == secret_string + finally + Secrets_Manager.delete_secret( + secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") ) end - end - @testset "SQS" begin - @test AWSServices.sqs isa AWS.QueryService + @test_throws AWSException _get_secret_string(secret_name) + end - @testset "high-level" begin - @service SQS + @testset "low-level secrets manager" begin + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" + function _get_secret_string(secret_name) + response = AWSServices.secrets_manager( + "GetSecretValue", LittleDict("SecretId" => secret_name) + ) - function _get_queue_url(queue_name) - result = SQS.get_queue_url(queue_name) + return response["SecretString"] + end - return result["QueueUrl"] - end + resp = AWSServices.secrets_manager( + "CreateSecret", + LittleDict( + "Name" => secret_name, + "SecretString" => secret_string, + "ClientRequestToken" => string(uuid4()), + ), + ) + + try + @test _get_secret_string(secret_name) == secret_string + finally + AWSServices.secrets_manager( + "DeleteSecret", + LittleDict( + "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" + ), + ) + end - # Create Queue - SQS.create_queue(queue_name) - queue_url = _get_queue_url(queue_name) + @test_throws AWSException _get_secret_string(secret_name) + end +end - try - # Get Queues - @test !isempty(queue_url) +@testset "query" begin + @testset "high-level iam" begin + @service IAM - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], + ) + expected_policy_document = JSON.json(expected_policy_document) + + response = IAM.create_policy(expected_policy_document, expected_policy_name) + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] + + try + response_policy_version = IAM.get_policy_version(policy_arn, "v1") + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + IAM.delete_policy(policy_arn) + end - SQS.send_message(expected_message, queue_url) + @test_throws AWSException IAM.get_policy(policy_arn) + end - response = SQS.receive_message(queue_url) - receipt_handle = only(response["Messages"])["ReceiptHandle"] + @testset "low-level iam" begin + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], + ) + expected_policy_document = JSON.json(expected_policy_document) + + response = AWSServices.iam( + "CreatePolicy", + LittleDict( + "PolicyName" => expected_policy_name, + "PolicyDocument" => expected_policy_document, + ), + ) + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] + + try + response_policy_version = AWSServices.iam( + "GetPolicyVersion", + LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), + ) + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) + end - response = SQS.delete_message_batch( - [ - LittleDict( - "Id" => expected_message_id, "ReceiptHandle" => receipt_handle - ), - ], - queue_url, - ) + @test_throws AWSException AWSServices.iam( + "GetPolicy", LittleDict("PolicyArn" => policy_arn) + ) + end - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id + @testset "high-level sqs" begin + @service SQS - SQS.send_message(expected_message, queue_url) + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" - result = SQS.receive_message(queue_url) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - SQS.delete_queue(queue_url) - end + function _get_queue_url(queue_name) + result = SQS.get_queue_url(queue_name) - @test_throws AWSException _get_queue_url(queue_name) + return result["QueueUrl"] end - @testset "low-level" begin - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" + # Create Queue + SQS.create_queue(queue_name) + queue_url = _get_queue_url(queue_name) - function _get_queue_url(queue_name) - result = AWSServices.sqs( - "GetQueueUrl", LittleDict("QueueName" => queue_name) - ) - - return result["QueueUrl"] - end - - # Create Queue - AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) - - queue_url = _get_queue_url(queue_name) + try + # Get Queues @test !isempty(queue_url) - try - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) + SQS.send_message(expected_message, queue_url) - response = AWSServices.sqs( - "ReceiveMessage", LittleDict("QueueUrl" => queue_url) - ) - receipt_handle = only(response["Messages"])["ReceiptHandle"] + response = SQS.receive_message(queue_url) + receipt_handle = only(response["Messages"])["ReceiptHandle"] - response = AWSServices.sqs( - "DeleteMessageBatch", + response = SQS.delete_message_batch( + [ LittleDict( - "QueueUrl" => queue_url, - "Entries" => [ - LittleDict( - "Id" => expected_message_id, - "ReceiptHandle" => receipt_handle, - ), - ], + "Id" => expected_message_id, "ReceiptHandle" => receipt_handle ), - ) - - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id - - # Send message - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) - - # Receive Message - result = AWSServices.sqs( - "ReceiveMessage", LittleDict("QueueUrl" => queue_url) - ) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) - end + ], + queue_url, + ) - @test_throws AWSException _get_queue_url(queue_name) - end - end -end + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id -@testset "JSON service" begin - @testset "Secrets Manager" begin - @test AWSServices.secrets_manager isa AWS.JSONService + SQS.send_message(expected_message, queue_url) - @testset "high-level" begin - @service Secrets_Manager + result = SQS.receive_message(queue_url) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + SQS.delete_queue(queue_url) + end - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" + @test_throws AWSException _get_queue_url(queue_name) + end - function _get_secret_string(secret_name) - response = Secrets_Manager.get_secret_value(secret_name) + @testset "low-level sqs" begin + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" - return response["SecretString"] - end + function _get_queue_url(queue_name) + result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) - Secrets_Manager.create_secret( - secret_name, - LittleDict( - "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) - ), - ) + return result["QueueUrl"] + end - try - @test _get_secret_string(secret_name) == secret_string - finally - Secrets_Manager.delete_secret( - secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") - ) - end + # Create Queue + AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) - @test_throws AWSException _get_secret_string(secret_name) - end + queue_url = _get_queue_url(queue_name) + @test !isempty(queue_url) - @testset "low-level" begin - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" + try + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" - function _get_secret_string(secret_name) - response = AWSServices.secrets_manager( - "GetSecretValue", LittleDict("SecretId" => secret_name) - ) + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) - return response["SecretString"] - end + response = AWSServices.sqs( + "ReceiveMessage", LittleDict("QueueUrl" => queue_url) + ) + receipt_handle = only(response["Messages"])["ReceiptHandle"] - resp = AWSServices.secrets_manager( - "CreateSecret", + response = AWSServices.sqs( + "DeleteMessageBatch", LittleDict( - "Name" => secret_name, - "SecretString" => secret_string, - "ClientRequestToken" => string(uuid4()), + "QueueUrl" => queue_url, + "Entries" => [ + LittleDict( + "Id" => expected_message_id, + "ReceiptHandle" => receipt_handle, + ), + ], ), ) - try - @test _get_secret_string(secret_name) == secret_string - finally - AWSServices.secrets_manager( - "DeleteSecret", - LittleDict( - "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" - ), - ) - end + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id + + # Send message + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) - @test_throws AWSException _get_secret_string(secret_name) + # Receive Message + result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) end + + @test_throws AWSException _get_queue_url(queue_name) end end -@testset "Rest XML" begin - @testset "S3" begin - @test AWSServices.s3 isa RestXMLService - - @testset "high-level" begin - @service S3 - - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = string(uuid4()) - - function _bucket_exists(bucket_name) - try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end - end - - # HEAD operation - @test _bucket_exists(bucket_name) == false +@testset "rest-xml" begin + @testset "high-level s3" begin + @service S3 - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = string(uuid4()) + function _bucket_exists(bucket_name) try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) - @test length([result["Contents"]]) == max_keys - - # GET with an IO target - mktemp() do f, io - S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) - flush(io) - @test read(f, String) == body + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) end - finally - # DELETE with parameters operation - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) - - # DELETE operation - S3.delete_bucket(bucket_name) - - sleep(2) end - - @test _bucket_exists(bucket_name) == false end - @testset "low-level" begin - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = "*)=('! +@,:.txt" # Special characters which S3 allows - - function _bucket_exists(bucket_name) - try - AWSServices.s3("HEAD", "/$bucket_name") - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # GET with an IO target + mktemp() do f, io + S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) + flush(io) + @test read(f, String) == body end + finally + # DELETE with parameters operation + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) - # HEAD operation - @test _bucket_exists(bucket_name) == false + # DELETE operation + S3.delete_bucket(bucket_name) - # PUT operation - AWSServices.s3("PUT", "/$bucket_name") - @test _bucket_exists(bucket_name) + sleep(2) + end + + @test _bucket_exists(bucket_name) == false + end + + @testset "low-level s3" begin + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = "*)=('! +@,:.txt" # Special characters which S3 allows + function _bucket_exists(bucket_name) try - # PUT with parameters operation - body = Array{UInt8}("sample-file-body") - AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) - @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body - - # GET operation - result = AWSServices.s3("GET", "/$bucket_name") - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = AWSServices.s3( - "GET", "/$bucket_name", Dict("max_keys" => max_keys) - ) - @test length([result["Contents"]]) == max_keys - - # POST with parameters operation - body = """ - - - $file_name - - - """ - - AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) - @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") - finally - # DELETE operation - AWSServices.s3("DELETE", "/$bucket_name") - - sleep(2) + AWSServices.s3("HEAD", "/$bucket_name") + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end end - - @test _bucket_exists(bucket_name) == false end - @testset "additional operations" begin - @service S3 - - bucket_name = "aws-jl-test---" * _now_formatted() + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + AWSServices.s3("PUT", "/$bucket_name") + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = Array{UInt8}("sample-file-body") + AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) + @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body + + # GET operation + result = AWSServices.s3("GET", "/$bucket_name") + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # POST with parameters operation + body = """ + + + $file_name + + + """ + + AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) + @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") + finally + # DELETE operation + AWSServices.s3("DELETE", "/$bucket_name") + + sleep(2) + end - # Testing a file name with various special & Unicode characters - file_name = "$(uuid4())/📁!!/@ +*" + @test _bucket_exists(bucket_name) == false + end - function _bucket_exists(bucket_name) - try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end - end + @testset "additional S3 operations" begin + @service S3 - # HEAD operation - @test _bucket_exists(bucket_name) == false + bucket_name = "aws-jl-test---" * _now_formatted() - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) + # Testing a file name with various special & Unicode characters + file_name = "$(uuid4())/📁!!/@ +*" + function _bucket_exists(bucket_name) try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - finally - # DELETE the file, check that it's gone, and then DELETE the bucket - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) - S3.delete_bucket(bucket_name) - sleep(2) + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end end + end - @test _bucket_exists(bucket_name) == false + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + finally + # DELETE the file, check that it's gone, and then DELETE the bucket + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) + S3.delete_bucket(bucket_name) + sleep(2) end + + @test _bucket_exists(bucket_name) == false end end -@testset "Rest JSON" begin - @testset "Glacier" begin - @test AWSServices.glacier isa RestJSONService +@testset "rest-json" begin + @testset "high-level glacier" begin + @service Glacier + + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] - @testset "high-level" begin - @service Glacier + # PUT + for vault in vault_names + Glacier.create_vault("-", vault) + end - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - # PUT for vault in vault_names - Glacier.create_vault("-", vault) + Glacier.add_tags_to_vault("-", vault, tags) end - try - # POST - tags = Dict( - "Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02") - ) - - for vault in vault_names - Glacier.add_tags_to_vault("-", vault, tags) - end - - for vault in vault_names - result_tags = Glacier.list_tags_for_vault("-", vault) - @test result_tags == tags - end - - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - args = LittleDict("limit" => limit) - result = Glacier.list_vaults("-", args) - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - Glacier.delete_vault("-", vault) - end + for vault in vault_names + result_tags = Glacier.list_tags_for_vault("-", vault) + @test result_tags == tags end - result = Glacier.list_vaults("-") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] - + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + args = LittleDict("limit" => limit) + result = Glacier.list_vaults("-", args) + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE for vault in vault_names - @test !(vault in res_vault_names) + Glacier.delete_vault("-", vault) end end - @testset "low-level" begin - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + result = Glacier.list_vaults("-") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] - # PUT - for vault in vault_names - AWSServices.glacier("PUT", "/-/vaults/$vault") - end + for vault in vault_names + @test !(vault in res_vault_names) + end + end - try - # POST - tags = Dict( - "Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02") - ) + @testset "low-level glacier" begin + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] - for vault in vault_names - AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) - end + # PUT + for vault in vault_names + AWSServices.glacier("PUT", "/-/vaults/$vault") + end - for vault in vault_names - result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - @test result_tags == tags - end + for vault in vault_names + AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) + end - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - params = LittleDict("limit" => limit) - result = AWSServices.glacier("GET", "/-/vaults/", params) - - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - AWSServices.glacier("DELETE", "/-/vaults/$vault") - end + for vault in vault_names + result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") + + @test result_tags == tags end - result = AWSServices.glacier("GET", "/-/vaults") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + params = LittleDict("limit" => limit) + result = AWSServices.glacier("GET", "/-/vaults/", params) + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE for vault in vault_names - @test !(vault in res_vault_names) + AWSServices.glacier("DELETE", "/-/vaults/$vault") end end + + result = AWSServices.glacier("GET", "/-/vaults") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] + + for vault in vault_names + @test !(vault in res_vault_names) + end end end From b2a626806f51284324897bab0d8e7e10d4b36f90 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 12:23:43 -0500 Subject: [PATCH 26/31] Make resursive profile merging optional --- src/AWSCredentials.jl | 2 +- src/utilities/credentials.jl | 25 +++++++++++++----------- test/unit/AWSCredentials.jl | 38 ++++++++++++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 9f83aa3ce..5c91634c7 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -657,7 +657,7 @@ function aws_get_region(; profile=nothing, config=nothing, default=DEFAULT_REGIO @something( get(ENV, "AWS_REGION", nothing), get(ENV, "AWS_DEFAULT_REGION", nothing), - get(_aws_profile_config(config, profile), "region", nothing), + get(_aws_profile_config(config, profile; recursive=true), "region", nothing), @mock(IMDS.region()), Some(default), ) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 2d8885e47..553694762 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -35,7 +35,7 @@ function _get_ini_value( return value end -function _aws_profile_config(ini::Inifile, profile::AbstractString) +function _aws_profile_config(ini::Inifile, profile::AbstractString; recursive::Bool=false) # Prefer using "profile default" over "default" section_name = if profile != "default" || haskey(sections(ini), "profile default") "profile $profile" @@ -44,27 +44,30 @@ function _aws_profile_config(ini::Inifile, profile::AbstractString) end content = copy(get(sections(ini), section_name, IniFile.HTSS())) - source_profile = pop!(content, "source_profile", nothing) - # Fallback on settings specified in the source profile - if !isnothing(source_profile) - content = merge(_aws_profile_config(ini, source_profile), content) + if recursive + source_profile = pop!(content, "source_profile", nothing) + + # Fallback on settings specified in the source profile + if !isnothing(source_profile) + content = merge(_aws_profile_config(ini, source_profile; recursive), content) + end end return content end -function _aws_profile_config(ini::Inifile, profile::Nothing) - return _aws_profile_config(ini, _aws_get_profile()) +function _aws_profile_config(ini::Inifile, profile::Nothing; kwargs...) + return _aws_profile_config(ini, _aws_get_profile(); kwargs...) end -function _aws_profile_config(config_file::AbstractString, profile) +function _aws_profile_config(config_file::AbstractString, profile; kwargs...) isfile(config_file) || return Dict() - return _aws_profile_config(read(Inifile(), config_file), profile) + return _aws_profile_config(read(Inifile(), config_file), profile; kwargs...) end -function _aws_profile_config(config_file::Nothing, profile) - return _aws_profile_config(dot_aws_config_file(), profile) +function _aws_profile_config(config_file::Nothing, profile; kwargs...) + return _aws_profile_config(dot_aws_config_file(), profile; kwargs...) end """ diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index 73add8103..a02623022 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -1,7 +1,37 @@ @testset "_aws_profile_config" begin using AWS: _aws_profile_config - @testset "source profile" begin + @testset "non-recursive" begin + #! format: off + buffer = IOBuffer( + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + """ + ) + #! format: on + ini = Inifile() + read(ini, buffer) + + # Only the fields from profile "test" + config = _aws_profile_config(ini, "test") + @test keys(config) ⊆ Set(["output", "region"]) + @test config["output"] == "json" + @test config["region"] == "us-east-1" + + # Only the fields from profile "test:dev" + config = _aws_profile_config(ini, "test:dev") + @test keys(config) ⊆ Set(["source_profile", "role_arn"]) + @test config["source_profile"] == "test" + @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" + end + + @testset "recursive" begin #! format: off buffer = IOBuffer( """ @@ -23,13 +53,13 @@ read(ini, buffer) # Only the fields from profile "test" - config = _aws_profile_config(ini, "test") + config = _aws_profile_config(ini, "test"; recursive=true) @test keys(config) ⊆ Set(["output", "region"]) @test config["output"] == "json" @test config["region"] == "us-east-1" # Combine the profile "test:dev" section with fields from profile "test" - config = _aws_profile_config(ini, "test:dev") + config = _aws_profile_config(ini, "test:dev"; recursive=true) @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" @@ -41,7 +71,7 @@ @test !haskey(section, "output") # Conflicting keys should use the first defined entry - config = _aws_profile_config(ini, "test:sub-dev") + config = _aws_profile_config(ini, "test:sub-dev"; recursive=true) @test keys(config) ⊆ Set(["output", "region", "role_arn"]) @test config["output"] == "json" @test config["region"] == "us-east-1" From 464909b2b66bbf76f1da5409b2c11da4008d8dba Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 Jul 2025 15:54:58 -0500 Subject: [PATCH 27/31] Inheriting is not supported --- src/AWSCredentials.jl | 2 +- src/utilities/credentials.jl | 27 ++++-------- test/AWSCredentials.jl | 4 +- test/unit/AWSCredentials.jl | 84 +++++++++--------------------------- 4 files changed, 30 insertions(+), 87 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 5c91634c7..9f83aa3ce 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -657,7 +657,7 @@ function aws_get_region(; profile=nothing, config=nothing, default=DEFAULT_REGIO @something( get(ENV, "AWS_REGION", nothing), get(ENV, "AWS_DEFAULT_REGION", nothing), - get(_aws_profile_config(config, profile; recursive=true), "region", nothing), + get(_aws_profile_config(config, profile), "region", nothing), @mock(IMDS.region()), Some(default), ) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 553694762..20e87d0cb 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -35,7 +35,7 @@ function _get_ini_value( return value end -function _aws_profile_config(ini::Inifile, profile::AbstractString; recursive::Bool=false) +function _aws_profile_config(ini::Inifile, profile::AbstractString) # Prefer using "profile default" over "default" section_name = if profile != "default" || haskey(sections(ini), "profile default") "profile $profile" @@ -43,31 +43,20 @@ function _aws_profile_config(ini::Inifile, profile::AbstractString; recursive::B "default" end - content = copy(get(sections(ini), section_name, IniFile.HTSS())) - - if recursive - source_profile = pop!(content, "source_profile", nothing) - - # Fallback on settings specified in the source profile - if !isnothing(source_profile) - content = merge(_aws_profile_config(ini, source_profile; recursive), content) - end - end - - return content + return copy(get(sections(ini), section_name, IniFile.HTSS())) end -function _aws_profile_config(ini::Inifile, profile::Nothing; kwargs...) - return _aws_profile_config(ini, _aws_get_profile(); kwargs...) +function _aws_profile_config(ini::Inifile, profile::Nothing) + return _aws_profile_config(ini, _aws_get_profile()) end -function _aws_profile_config(config_file::AbstractString, profile; kwargs...) +function _aws_profile_config(config_file::AbstractString, profile) isfile(config_file) || return Dict() - return _aws_profile_config(read(Inifile(), config_file), profile; kwargs...) + return _aws_profile_config(read(Inifile(), config_file), profile) end -function _aws_profile_config(config_file::Nothing, profile; kwargs...) - return _aws_profile_config(dot_aws_config_file(), profile; kwargs...) +function _aws_profile_config(config_file::Nothing, profile) + return _aws_profile_config(dot_aws_config_file(), profile) end """ diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 13f346883..af2f13c0d 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -213,7 +213,6 @@ end """ [profile test] output = json - region = us-east-1 [profile test:dev] source_profile = test @@ -226,8 +225,6 @@ end [profile test2] aws_access_key_id = WRONG_ACCESS_ID aws_secret_access_key = WRONG_ACCESS_KEY - output = json - region = us-east-1 [profile test3] source_profile = test:dev @@ -264,6 +261,7 @@ end "AWS_DEFAULT_PROFILE" => "test", "AWS_PROFILE" => nothing, "AWS_ACCESS_KEY_ID" => nothing, + "AWS_REGION" => "us-east-1", ) do @testset "Loading" begin # Check credentials load diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index a02623022..2baa12bf7 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -1,86 +1,42 @@ @testset "_aws_profile_config" begin using AWS: _aws_profile_config - @testset "non-recursive" begin + @testset "access" begin #! format: off buffer = IOBuffer( """ - [profile test] + [profile A] output = json region = us-east-1 - [profile test:dev] - source_profile = test - role_arn = arn:aws:iam::123456789000:role/Dev + [profile B] + role_arn = arn:aws:iam::123456789000:role/A + source_profile = A """ ) #! format: on ini = Inifile() read(ini, buffer) - # Only the fields from profile "test" - config = _aws_profile_config(ini, "test") + # Retrieve fields from profile "A" + config = _aws_profile_config(ini, "A") @test keys(config) ⊆ Set(["output", "region"]) @test config["output"] == "json" @test config["region"] == "us-east-1" - # Only the fields from profile "test:dev" - config = _aws_profile_config(ini, "test:dev") - @test keys(config) ⊆ Set(["source_profile", "role_arn"]) - @test config["source_profile"] == "test" - @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" - end - - @testset "recursive" begin - #! format: off - buffer = IOBuffer( - """ - [profile test] - output = json - region = us-east-1 - - [profile test:dev] - source_profile = test - role_arn = arn:aws:iam::123456789000:role/Dev - - [profile test:sub-dev] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/SubDev - """ - ) - #! format: on - ini = Inifile() - read(ini, buffer) - - # Only the fields from profile "test" - config = _aws_profile_config(ini, "test"; recursive=true) - @test keys(config) ⊆ Set(["output", "region"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - - # Combine the profile "test:dev" section with fields from profile "test" - config = _aws_profile_config(ini, "test:dev"; recursive=true) - @test keys(config) ⊆ Set(["output", "region", "role_arn"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - @test config["role_arn"] == "arn:aws:iam::123456789000:role/Dev" - - # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["profile test:dev"] - @test !haskey(section, "region") - @test !haskey(section, "output") - - # Conflicting keys should use the first defined entry - config = _aws_profile_config(ini, "test:sub-dev"; recursive=true) - @test keys(config) ⊆ Set(["output", "region", "role_arn"]) - @test config["output"] == "json" - @test config["region"] == "us-east-1" - @test config["role_arn"] == "arn:aws:iam::123456789000:role/SubDev" - - # Ensure we haven't mutated the contents of the `ini` - section = sections(ini)["profile test:sub-dev"] - @test !haskey(section, "region") - @test !haskey(section, "output") + # Ensure that mutating the returned dictionary does not mutated the contents of the + # `ini` + config["output"] = "yaml-stream" + section = sections(ini)["profile A"] + @test section["output"] == "json" + + # Use of `source_profile` does not inherit properties from the source. This mirrors + # the AWS CLI (version v2.27.15) behavior for `region` which can be seen using + # `aws configure list --profile X` + config = _aws_profile_config(ini, "B") + @test keys(config) ⊆ Set(["role_arn", "source_profile"]) + @test config["role_arn"] == "arn:aws:iam::123456789000:role/A" + @test config["source_profile"] == "A" end # AWS CLI (version v2.27.15) will use "profile default" over "default" when both are From 41f678c404d6a180bc4c13f0cf6c07235d9a88f7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 3 Jul 2025 08:20:12 -0500 Subject: [PATCH 28/31] Comment on requirements for tests --- test/runtests.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 9850a58f4..fc4482ab4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -60,9 +60,10 @@ if !RUN_UNIT_TESTS && !RUN_INTEGRATION_TESTS end @testset "AWS.jl" begin + # Unit tests do not requires access to an AWS account @testset "Unit Tests" begin if RUN_UNIT_TESTS - # Force these tests to run without a valid AWS configuration being present + # Force unit tests to run without a valid AWS configuration being present withenv( [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., "AWS_CONFIG_FILE" => "/dev/null", @@ -83,7 +84,7 @@ end end end - # TODO: Some of these tests are actually unit tests and need to be refactored + # Integration tests require access to an AWS account @testset "Integration Tests" begin if RUN_INTEGRATION_TESTS backends = [AWS.HTTPBackend, AWS.DownloadsBackend] From 4bc22acbcf2d768ea327a44b6ed2a30561f61ccc Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 3 Jul 2025 08:20:27 -0500 Subject: [PATCH 29/31] Make integration tests opt-in --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index fc4482ab4..cbe8637a2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -48,7 +48,7 @@ end testset_role(role_name) = "AWS.jl-$role_name" const RUN_UNIT_TESTS = get(ENV, "RUN_UNIT_TESTS", "true") == "true" -const RUN_INTEGRATION_TESTS = get(ENV, "RUN_INTEGRATION_TESTS", "true") == "true" +const RUN_INTEGRATION_TESTS = get(ENV, "RUN_INTEGRATION_TESTS", "false") == "true" const RUN_MINIO_INTEGRATION_TESTS = begin all(k -> haskey(ENV, k), ("MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MINIO_REGION_NAME")) end From e93f07aa8ac9d391c882fefc7572c82c13f08734 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 3 Jul 2025 10:47:15 -0500 Subject: [PATCH 30/31] Reset default AWS configuration in integration tests --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index cbe8637a2..fef367061 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -87,6 +87,10 @@ end # Integration tests require access to an AWS account @testset "Integration Tests" begin if RUN_INTEGRATION_TESTS + # Reset the default AWS configuration as the unit tests may have messed with + # the global default. + global_aws_config(AWSConfig()) + backends = [AWS.HTTPBackend, AWS.DownloadsBackend] @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() From 329fb2dd435758cfb72b723f098d2e68e8b27b8f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 3 Jul 2025 10:59:01 -0500 Subject: [PATCH 31/31] fixup! Reset default AWS configuration in integration tests --- test/runtests.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index fef367061..a9f8e0617 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -87,13 +87,16 @@ end # Integration tests require access to an AWS account @testset "Integration Tests" begin if RUN_INTEGRATION_TESTS - # Reset the default AWS configuration as the unit tests may have messed with - # the global default. - global_aws_config(AWSConfig()) - + config = AWSConfig() backends = [AWS.HTTPBackend, AWS.DownloadsBackend] + @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() + + # Reset the default AWS configuration as the unit tests may have messed with + # the global default. + global_aws_config(config) + include("integration/AWS.jl") include("integration/AWSCredentials.jl") include("integration/role.jl")