diff --git a/test/AWS.jl b/test/AWS.jl deleted file mode 100644 index 1567e37e5..000000000 --- a/test/AWS.jl +++ /dev/null @@ -1,1184 +0,0 @@ -@testset "global config, kwargs" begin - try - region = "us-east-2" - AWS.global_aws_config(; region=region) - - @test AWS.global_aws_config().region == region - finally - AWS.aws_config[] = AWSConfig() - end -end - -@testset "set global aws config" begin - test_region = "test region" - expected = AWSConfig(; region=test_region) - - try - AWS.global_aws_config(expected) - result = AWS.global_aws_config() - - @test result.region == test_region - finally - AWS.global_aws_config(AWSConfig()) - end -end - -@testset "set user agent" begin - old_user_agent = AWS.user_agent[] - new_user_agent = "new user agent" - - try - @test AWS.user_agent[] == "AWS.jl/$(pkgversion(AWS))" - set_user_agent(new_user_agent) - @test AWS.user_agent[] == new_user_agent - finally - set_user_agent(old_user_agent) - end -end - -@testset "sign" begin - aws = AWS.AWSConfig(; region="us-east-1") - - access_key = "access-key" - secret_key = "ssh... it is a secret" - - aws.credentials.access_key_id = access_key - aws.credentials.secret_key = secret_key - aws.credentials.token = "" - - time = DateTime(2020) - date = Dates.format(time, dateformat"yyyymmdd") - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - headers=LittleDict( - "Host" => "s3.us-east-1.amazonaws.com", "User-Agent" => "AWS.jl/1.0.0" - ), - resource="/test-resource", - url="https://s3.us-east-1.amazonaws.com/test-resource", - ) - - @testset "sign v2" begin - result = AWS.sign_aws2!(aws, request, time) - @test result === request - content = result.content - content_type = result.headers["Content-Type"] - - expected_access_key = "AWSAccessKeyId=$access_key" - expected_expires = "Expires=2020-01-01T00%3A02%3A00Z" - expected_signature_method = "SignatureMethod=HmacSHA256" - expected_signature_version = "SignatureVersion=2" - expected_signature = "Signature=O0MLzMKpEcfVZeHy0tyxVAuZF%2BvrvbgIGgqbWtJLTQ0%3D" - - expected_content = join( - [ - expected_access_key, - expected_expires, - expected_signature_method, - expected_signature_version, - expected_signature, - ], - '&', - ) - - @test content == expected_content - end - - @testset "sign v4" begin - @testset "basic" begin - expected_x_amz_content_sha256 = bytes2hex(digest(MD_SHA256, request.content)) - expected_content_md5 = base64encode(digest(MD_MD5, request.content)) - expected_x_amz_date = Dates.format(time, dateformat"yyyymmdd\THHMMSS\Z") - - result = AWS.sign_aws4!(aws, request, time) - @test result === request - headers = result.headers - - @test headers["x-amz-content-sha256"] == expected_x_amz_content_sha256 - @test headers["Content-MD5"] == expected_content_md5 - @test headers["x-amz-date"] == expected_x_amz_date - - authorization_header = split(headers["Authorization"], ' ') - @test length(authorization_header) == 4 - @test authorization_header[1] == "AWS4-HMAC-SHA256" - @test authorization_header[2] == - "Credential=$access_key/$date/us-east-1/$(request.service)/aws4_request," - @test authorization_header[3] == - "SignedHeaders=content-md5;content-type;host;user-agent;x-amz-content-sha256;x-amz-date," - @test authorization_header[4] == - "Signature=0f292eaf0b66cf353bafcb1b9b6d90ee27064236a60f17f6fc5bd7d40173a0be" - end - - @testset "duplicate query params" begin - request_query = deepcopy(request) - request_query.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions" - - request_dup = deepcopy(request) - request_dup.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions&versions=" - - AWS.sign_aws4!(aws, request_query, time) - AWS.sign_aws4!(aws, request_dup, time) - - authorization_header_query = split(request_query.headers["Authorization"], ' ') - authorization_header_dup = split(request_dup.headers["Authorization"], ' ') - - @test length(authorization_header_query) == 4 - @test length(authorization_header_dup) == 4 - - # Signatures should differ - @test authorization_header_query[4] != authorization_header_dup[4] - end - end -end - -@testset "submit_request" begin - aws = AWS.AWSConfig() - - function _expected_xml(body::AbstractString, dict_type::Type) - parsed = parse_xml(body) - return xml_dict(XMLDict.root(parsed.x), dict_type) - end - - @testset "301 redirect" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="HEAD", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - apply(Patches._aws_http_request_patch(Patches._response(; status=301))) do - @test_throws AWSException AWS.submit_request(aws, request) - end - end - - @testset "HEAD response" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="HEAD", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - response = apply(Patches._aws_http_request_patch()) do - AWS.submit_request(aws, request) - end - - # Access to response headers - @test response.response.headers == Patches.headers - @test response.response.headers isa Vector - - # Access to streaming content - @test response.io isa IO - - # Content as a string - @test String(take!(response.io)) == Patches.body - - # Backwards compatibility with those expecting an `HTTP.Response` - @test response.headers == Patches.headers - @test response.headers isa Vector - @test String(response.body) == Patches.body - end - - @testset "GET response" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - response = apply(Patches._aws_http_request_patch()) do - AWS.submit_request(aws, request) - end - - # Access to response headers - @test response.response.headers == Patches.headers - @test response.response.headers isa Vector - - # Access to streaming content - @test response.io isa IO - - # Content as a string - @test String(take!(response.io)) == Patches.body - - # Backwards compatibility with those expecting an `HTTP.Response` - @test response.headers == Patches.headers - @test response.headers isa Vector - @test String(response.body) == Patches.body - end - - @testset "Default throttling" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - retries = Ref{Int}(0) - exception = apply(Patches._throttling_patch(retries)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - @test exception.code == "SlowDown" - @test retries[] == AWS.max_attempts(aws) - end - - @testset "Custom throttling" begin - aws = AWS.AWSConfig(; max_attempts=1) - @test AWS.max_attempts(aws) == 1 - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - retries = Ref{Int}(0) - exception = apply(Patches._throttling_patch(retries)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - @test exception.code == "SlowDown" - @test retries[] == AWS.max_attempts(aws) - end - - @testset "Not authorized" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - message = "User is not authorized to perform: action on resource with an explicit deny" - - # Simulate the HTTP.request behaviour with a HTTP 400 response - exception = apply(Patches.gen_http_options_400_patches(message)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - - @test exception isa AWSException - - # If handled incorrectly using a `response_stream` may result in the body data being - # lost. Mainly, this is a problem when using a temporary I/O stream instead of - # writing directly to the `response_stream`. - @test exception.message == message - @test exception.streamed_body !== nothing - end - - @testset "Not authorized with BufferStream response_stream" begin - buf = Base.BufferStream() - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - response_stream=buf, - use_response_type=true, - ) - message = "User is not authorized to perform: action on resource with an explicit deny" - # Simulate the HTTP.request behaviour with a HTTP 400 response - exception = apply(Patches.gen_http_options_400_patches(message)) do - try - AWS.submit_request(aws, request) - return nothing - catch e - if e isa AWSException - return e - else - rethrow() - end - end - end - @test exception isa AWSException - # If handled incorrectly using a `response_stream` may result in the body data being - # lost. Mainly, this is a problem when using a temporary I/O stream instead of - # writing directly to the `response_stream`. - @test exception.message == message - @test exception.streamed_body !== nothing - end - - @testset "read MIME-type" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - use_response_type=true, - ) - - @testset "invalid content type" begin - headers = Pair["Content-Type" => ""] - body = "" - expected_body_type = Vector{UInt8} - expected_body = b"" - - r = Patches._response(; headers=headers, body=body) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "text/xml" begin - headers = Pair["Content-Type" => "text/xml"] - expected_body_type = LittleDict{Union{String,Symbol},Any} - expected_body = _expected_xml(Patches.body, expected_body_type) - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "application/xml" begin - headers = Pair["Content-Type" => "application/xml"] - expected_body_type = LittleDict{Union{String,Symbol},Any} - expected_body = _expected_xml(Patches.body, expected_body_type) - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "application/json" begin - headers = ["Content-Type" => "application/json"] - body = JSON.json( - Dict{String,Any}( - "Marker" => nothing, - "VaultList" => Any[Dict{String,Any}( - "VaultName" => "test", - "SizeInBytes" => 0, - "NumberOfArchives" => 0, - "CreationDate" => "2020-06-22T03:14:41.754Z", - "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", - "LastInventoryDate" => nothing, - )], - ), - ) - - expected_body_type = LittleDict{String,Any} - expected_body = JSON.parse(body; dicttype=expected_body_type) - - r = Patches._response(; body=body, headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - end - - @testset "text/html" begin - headers = ["Content-Type" => "text/html"] - expected_body = Patches.body - - r = Patches._response(; headers=headers) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa String - @test content == expected_body - end - - # Note: `S3.create_multipart_upload` is an example of a response type that doesn't - # specify a Content-Type. - @testset "missing content type" begin - headers = Pair[] - body = """ - - text - """ - expected_body_type = AbstractDict - expected_body = Dict{String,Any}("body" => "text") - - r = Patches._response(; headers=headers, body=body) - response = apply(Patches._aws_http_request_patch(r)) do - AWS.submit_request(aws, request) - end - - content = parse(response) - @test content isa expected_body_type - @test content == expected_body - - content = parse(response, MIME"text/plain"()) - @test content isa String - @test content == body - end - end -end - -struct TestBackend <: AWS.AbstractBackend - param::Int -end - -function AWS._http_request(backend::TestBackend, ::AWS.Request, ::IO) - return backend.param -end - -@testset "HTTPBackend" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - backend=AWS.HTTPBackend(), - ) - io = IOBuffer() - - apply(Patches._http_options_patches) do - # No default options - @test isempty(AWS._http_request(request.backend, request, io)) - - # We can pass HTTP options via the backend - custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5)) - @test custom_backend isa AWS.AbstractBackend - @test AWS._http_request(custom_backend, request, io) == Dict(:connection_limit => 5) - - # We can pass options per-request - request.http_options = Dict(:pipeline_limit => 20) - @test AWS._http_request(request.backend, request, io) == Dict(:pipeline_limit => 20) - @test AWS._http_request(custom_backend, request, io) == - Dict(:pipeline_limit => 20, :connection_limit => 5) - - # per-request options override backend options: - custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5)) - @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20) - end - - request.backend = TestBackend(2) - @test AWS._http_request(request.backend, request, io) == 2 - - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - backend=TestBackend(4), - ) - @test AWS._http_request(request.backend, request, io) == 4 - - # Let's test setting the default backend - prev_backend = AWS.DEFAULT_BACKEND[] - try - AWS.DEFAULT_BACKEND[] = TestBackend(3) - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - ) - @test AWS._http_request(request.backend, request, io) == 3 - finally - AWS.DEFAULT_BACKEND[] = prev_backend - end -end - -@testset "_generate_rest_resource" begin - request_uri = "/{Bucket}/{Key+}" - args = Dict{String,Any}("Bucket" => "aws.jl-test", "Key" => "Test-Key") - - expected = "/$(args["Bucket"])/$(args["Key"])" - result = AWS._generate_rest_resource(request_uri, args) - @test result == expected -end - -@testset "generate_service_url" begin - region = "us-east-2" - resource = "/aws.jl-test---timestamp" - config = AWSConfig() - config.region = region - - request = Request(; - service="service", - api_version="api_version", - request_method="GET", - resource=resource, - ) - - @testset "regionless endpoints" for regionless_endpoint in ("iam", "route53") - endpoint = "sdb" - request.service = regionless_endpoint - expected_result = "https://$regionless_endpoint.amazonaws.com$resource" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end - - @testset "region service" begin - endpoint = "sdb" - request.service = endpoint - expected_result = "https://$endpoint.$region.amazonaws.com$resource" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end - - @testset "sdb -- us-east-1 region exception" begin - endpoint = "sdb" - request.service = endpoint - expected_result = "https://$endpoint.amazonaws.com$resource" - config.region = "us-east-1" - result = AWS.generate_service_url(config, request.service, request.resource) - - @test result == expected_result - end -end - -@testset "_flatten_query" begin - high_level_value = "high_level_value" - entry_1 = LittleDict( - "low_level_key_1" => "low_level_value_1", "low_level_key_2" => "low_level_value_2" - ) - entry_2 = LittleDict( - "low_level_key_3" => "low_level_value_3", "low_level_key_4" => "low_level_value_4" - ) - - args = LittleDict( - "high_level_key" => high_level_value, "high_level_array" => [entry_1, entry_2] - ) - - @testset "non-special case suffix" begin - service = "sts" - result = AWS._flatten_query(service, args) - - expected = Pair{String,String}[ - "high_level_key" => "high_level_value", - "high_level_array.member.1.low_level_key_1" => "low_level_value_1", - "high_level_array.member.1.low_level_key_2" => "low_level_value_2", - "high_level_array.member.2.low_level_key_3" => "low_level_value_3", - "high_level_array.member.2.low_level_key_4" => "low_level_value_4", - ] - - @test result == expected - end - - @testset "sqs - special casing suffix" begin - service = "sqs" - result = AWS._flatten_query(service, args) - - expected = Pair{String,String}[ - "high_level_key" => "high_level_value", - "high_level_array.1.low_level_key_1" => "low_level_value_1", - "high_level_array.1.low_level_key_2" => "low_level_value_2", - "high_level_array.2.low_level_key_3" => "low_level_value_3", - "high_level_array.2.low_level_key_4" => "low_level_value_4", - ] - - @test result == expected - end -end - -@testset "_clean_s3_uri" begin - uri = "/test-bucket/*)=('! +@,:.txt?list-objects=v2" - expected_uri = "/test-bucket/%2A%29%3D%28%27%21%20%2B%40%2C%3A.txt?list-objects=v2" - @test AWS._clean_s3_uri(uri) == expected_uri - - # make sure that other parts of the uri aren't changed by `_clean_s3_uri` - for uri in ( - "https://julialang.org", - "http://julialang.org", - "http://julialang.org:8080", - "/onlypath", - "/path?query= +99", - "/anchor?query=yes#anchor1", - ) - @test AWS._clean_s3_uri(uri) == uri - end -end - -@testset "STS" begin - @testset "high-level" begin - @service STS - - response = STS.get_caller_identity() - d = response["GetCallerIdentityResult"] - - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end - - @testset "low-level" begin - response = AWSServices.sts("GetCallerIdentity") - d = response["GetCallerIdentityResult"] - - @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) - @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) - @test all(isdigit, d["Account"]) - end -end - -@testset "json" begin - @testset "high-level secrets manager" begin - @service Secrets_Manager - - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" - - function _get_secret_string(secret_name) - response = Secrets_Manager.get_secret_value(secret_name) - - return response["SecretString"] - end - - Secrets_Manager.create_secret( - secret_name, - LittleDict( - "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) - ), - ) - - try - @test _get_secret_string(secret_name) == secret_string - finally - Secrets_Manager.delete_secret( - secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") - ) - end - - @test_throws AWSException _get_secret_string(secret_name) - end - - @testset "low-level secrets manager" begin - secret_name = "aws-jl-test---" * _now_formatted() - secret_string = "sshhh it is a secret!" - - function _get_secret_string(secret_name) - response = AWSServices.secrets_manager( - "GetSecretValue", LittleDict("SecretId" => secret_name) - ) - - return response["SecretString"] - end - - resp = AWSServices.secrets_manager( - "CreateSecret", - LittleDict( - "Name" => secret_name, - "SecretString" => secret_string, - "ClientRequestToken" => string(uuid4()), - ), - ) - - try - @test _get_secret_string(secret_name) == secret_string - finally - AWSServices.secrets_manager( - "DeleteSecret", - LittleDict( - "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" - ), - ) - end - - @test_throws AWSException _get_secret_string(secret_name) - end -end - -@testset "query" begin - @testset "high-level iam" begin - @service IAM - - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) - - response = IAM.create_policy(expected_policy_document, expected_policy_name) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = IAM.get_policy_version(policy_arn, "v1") - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - IAM.delete_policy(policy_arn) - end - - @test_throws AWSException IAM.get_policy(policy_arn) - end - - @testset "low-level iam" begin - policy_arn = "" - expected_policy_name = "aws-jl-test---" * _now_formatted() - expected_policy_document = LittleDict( - "Version" => "2012-10-17", - "Statement" => [ - LittleDict( - "Effect" => "Allow", - "Action" => ["s3:Get*", "s3:List*"], - "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], - ), - ], - ) - expected_policy_document = JSON.json(expected_policy_document) - - response = AWSServices.iam( - "CreatePolicy", - LittleDict( - "PolicyName" => expected_policy_name, - "PolicyDocument" => expected_policy_document, - ), - ) - policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] - - try - response_policy_version = AWSServices.iam( - "GetPolicyVersion", - LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), - ) - response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] - @test HTTP.unescapeuri(response_document) == expected_policy_document - finally - AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) - end - - @test_throws AWSException AWSServices.iam( - "GetPolicy", LittleDict("PolicyArn" => policy_arn) - ) - end - - @testset "high-level sqs" begin - @service SQS - - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" - - function _get_queue_url(queue_name) - result = SQS.get_queue_url(queue_name) - - return result["QueueUrl"] - end - - # Create Queue - SQS.create_queue(queue_name) - queue_url = _get_queue_url(queue_name) - - try - # Get Queues - @test !isempty(queue_url) - - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" - - SQS.send_message(expected_message, queue_url) - - response = SQS.receive_message(queue_url) - receipt_handle = only(response["Messages"])["ReceiptHandle"] - - response = SQS.delete_message_batch( - [ - LittleDict( - "Id" => expected_message_id, "ReceiptHandle" => receipt_handle - ), - ], - queue_url, - ) - - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id - - SQS.send_message(expected_message, queue_url) - - result = SQS.receive_message(queue_url) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - SQS.delete_queue(queue_url) - end - - @test_throws AWSException _get_queue_url(queue_name) - end - - @testset "low-level sqs" begin - queue_name = "aws-jl-test---" * _now_formatted() - expected_message = "Hello for AWS.jl" - - function _get_queue_url(queue_name) - result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) - - return result["QueueUrl"] - end - - # Create Queue - AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) - - queue_url = _get_queue_url(queue_name) - @test !isempty(queue_url) - - try - # Change Message Visibility Batch Request - expected_message_id = "aws-jl-test" - - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) - - response = AWSServices.sqs( - "ReceiveMessage", LittleDict("QueueUrl" => queue_url) - ) - receipt_handle = only(response["Messages"])["ReceiptHandle"] - - response = AWSServices.sqs( - "DeleteMessageBatch", - LittleDict( - "QueueUrl" => queue_url, - "Entries" => [ - LittleDict( - "Id" => expected_message_id, - "ReceiptHandle" => receipt_handle, - ), - ], - ), - ) - - message_id = only(response["Successful"])["Id"] - @test message_id == expected_message_id - - # Send message - AWSServices.sqs( - "SendMessage", - LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), - ) - - # Receive Message - result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) - message = only(result["Messages"])["Body"] - @test message == expected_message - finally - AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) - end - - @test_throws AWSException _get_queue_url(queue_name) - end -end - -@testset "rest-xml" begin - @testset "high-level s3" begin - @service S3 - - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = string(uuid4()) - - function _bucket_exists(bucket_name) - try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end - end - - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) - @test length([result["Contents"]]) == max_keys - - # GET with an IO target - mktemp() do f, io - S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) - flush(io) - @test read(f, String) == body - end - finally - # DELETE with parameters operation - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) - - # DELETE operation - S3.delete_bucket(bucket_name) - - sleep(2) - end - - @test _bucket_exists(bucket_name) == false - end - - @testset "low-level s3" begin - bucket_name = "aws-jl-test---" * _now_formatted() - file_name = "*)=('! +@,:.txt" # Special characters which S3 allows - - function _bucket_exists(bucket_name) - try - AWSServices.s3("HEAD", "/$bucket_name") - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end - end - - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - AWSServices.s3("PUT", "/$bucket_name") - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = Array{UInt8}("sample-file-body") - AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) - @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body - - # GET operation - result = AWSServices.s3("GET", "/$bucket_name") - @test result["Contents"]["Key"] == file_name - - # GET with parameters operation - max_keys = 1 - result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) - @test length([result["Contents"]]) == max_keys - - # POST with parameters operation - body = """ - - - $file_name - - - """ - - AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) - @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") - finally - # DELETE operation - AWSServices.s3("DELETE", "/$bucket_name") - - sleep(2) - end - - @test _bucket_exists(bucket_name) == false - end - - @testset "additional S3 operations" begin - @service S3 - - bucket_name = "aws-jl-test---" * _now_formatted() - - # Testing a file name with various special & Unicode characters - file_name = "$(uuid4())/📁!!/@ +*" - - function _bucket_exists(bucket_name) - try - S3.head_bucket(bucket_name) - return true - catch e - if e isa AWSException && e.cause.status == 404 - return false - else - rethrow(e) - end - end - end - - # HEAD operation - @test _bucket_exists(bucket_name) == false - - # PUT operation - S3.create_bucket(bucket_name) - @test _bucket_exists(bucket_name) - - try - # PUT with parameters operation - body = "sample-file-body" - S3.put_object(bucket_name, file_name, Dict("body" => body)) - @test !isempty(S3.get_object(bucket_name, file_name)) - - # GET operation - result = S3.list_objects(bucket_name) - @test result["Contents"]["Key"] == file_name - finally - # DELETE the file, check that it's gone, and then DELETE the bucket - S3.delete_object(bucket_name, file_name) - @test_throws AWSException S3.get_object(bucket_name, file_name) - S3.delete_bucket(bucket_name) - sleep(2) - end - - @test _bucket_exists(bucket_name) == false - end -end - -@testset "rest-json" begin - @testset "high-level glacier" begin - @service Glacier - - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] - - # PUT - for vault in vault_names - Glacier.create_vault("-", vault) - end - - try - # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - - for vault in vault_names - Glacier.add_tags_to_vault("-", vault, tags) - end - - for vault in vault_names - result_tags = Glacier.list_tags_for_vault("-", vault) - @test result_tags == tags - end - - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - args = LittleDict("limit" => limit) - result = Glacier.list_vaults("-", args) - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - Glacier.delete_vault("-", vault) - end - end - - result = Glacier.list_vaults("-") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] - - for vault in vault_names - @test !(vault in res_vault_names) - end - end - - @testset "low-level glacier" begin - timestamp = _now_formatted() - vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] - - # PUT - for vault in vault_names - AWSServices.glacier("PUT", "/-/vaults/$vault") - end - - try - # POST - tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) - - for vault in vault_names - AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) - end - - for vault in vault_names - result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") - - @test result_tags == tags - end - - # GET - # If this is an Integer AWS Coral cannot convert it to a String - # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" - limit = "1" - params = LittleDict("limit" => limit) - result = AWSServices.glacier("GET", "/-/vaults/", params) - - @test length(result["VaultList"]) == parse(Int, limit) - finally - # DELETE - for vault in vault_names - AWSServices.glacier("DELETE", "/-/vaults/$vault") - end - end - - result = AWSServices.glacier("GET", "/-/vaults") - res_vault_names = [v["VaultName"] for v in result["VaultList"]] - - for vault in vault_names - @test !(vault in res_vault_names) - end - end -end diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl deleted file mode 100644 index af2f13c0d..000000000 --- a/test/AWSCredentials.jl +++ /dev/null @@ -1,1366 +0,0 @@ -macro test_ecode(error_codes, expr) - quote - try - $expr - @test false - catch e - if e isa AWSException - @test e.code in [$error_codes;] - else - rethrow(e) - end - end - end -end - -const EXPIRATION_FMT = dateformat"yyyy-mm-dd\THH:MM:SS\Z" - -http_header(h::Vector, k, d="") = get(Dict(h), k, d) -http_header(args...) = HTTP.header(args...) - -@testset "Load Credentials" begin - user = aws_user_arn(AWS_CONFIG[]) - @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) - AWS_CONFIG[].region = "us-east-1" - - @test_ecode("InvalidAction", AWSServices.iam("GetFoo")) - - @test_ecode( - ["AccessDenied", "NoSuchEntity"], - AWSServices.iam("GetUser", Dict("UserName" => "notauser")) - ) - - @test_ecode("ValidationError", AWSServices.iam("GetUser", Dict("UserName" => "@#!%%!"))) - - # Please note: If testing in a managed Corporate AWS environment, this can set off alarms... - @test_ecode( - ["AccessDenied", "EntityAlreadyExists"], - AWSServices.iam("CreateUser", Dict("UserName" => "root")) - ) -end - -@testset "_role_session_name" begin - @test AWS._role_session_name("prefix-", "name", "-suffix") == "prefix-name-suffix" - @test AWS._role_session_name("a"^22, "b"^22, "c"^22) == "a"^22 * "b"^20 * "c"^22 -end - -@testset "aws_get_profile_settings" begin - @testset "no profile" begin - @test aws_get_profile_settings("foo", Inifile()) === nothing - end -end - -@testset "_aws_get_role" begin - profile = "foobar" - ini = Inifile() - - @testset "settings early exit" begin - apply(Patches.get_profile_settings_empty_patch) do - @test AWS._aws_get_role(profile, ini) === nothing - end - end - - @testset "source_profile early exit" begin - apply(Patches.get_profile_settings_empty_patch) do - @test AWS._aws_get_role(profile, ini) === nothing - end - end - - @testset "default profile" begin - access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "default-role") - - patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) - - cred = withenv( - "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), - "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), - "AWS_ACCESS_KEY_ID" => nothing, - "AWS_SECRET_ACCESS_KEY" => nothing, - ) do - ini = read(Inifile(), ENV["AWS_CONFIG_FILE"]) - apply(patch) do - AWS._aws_get_role("default", ini) - end - end - - @test cred.access_key_id == access_key_id - end - - @testset "profile with role and MFA" begin - access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "role-with-mfa") - - mfa_token = "123456" - sent_token = Ref("") - server_time = DateTime(0) - patches = [ - Patches._assume_role_patch( - "AssumeRole"; - access_key=access_key_id, - expiry=duration -> server_time + duration, - token_code_ref=sent_token, - ), - Patches._getpass_patch(; secret=mfa_token), - ] - - cred = withenv( - "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), - "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), - "AWS_ACCESS_KEY_ID" => nothing, - "AWS_SECRET_ACCESS_KEY" => nothing, - ) do - ini = read(Inifile(), ENV["AWS_CONFIG_FILE"]) - apply(patches) do - AWS._aws_get_role("role_and_mfa", ini) - end - end - - @test cred.access_key_id == access_key_id - @test cred.expiry == server_time + Second(1234) - @test sent_token[] == mfa_token - end -end - -@testset "AWSCredentials" begin - @testset "Defaults" begin - creds = AWSCredentials("access_key_id", "secret_key") - @test creds.token == "" - @test creds.user_arn == "" - @test creds.account_number == "" - @test creds.expiry == typemax(DateTime) - @test creds.renew === nothing - end - - @testset "Renewal" begin - # Credentials shouldn't throw an error if no renew function is supplied - creds = AWSCredentials("access_key_id", "secret_key"; renew=nothing) - newcreds = check_credentials(creds; force_refresh=true) - - # Creds should remain unchanged if no renew function exists - @test creds === newcreds - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - @test creds.renew === nothing - - # Creds should error if the renew function returns nothing - creds = AWSCredentials("access_key_id", "secret_key"; renew=() -> nothing) - @test_throws NoCredentials check_credentials(creds; force_refresh=true) - - # Creds should remain unchanged - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - - # Creds should take on value of a returned AWSCredentials except renew function - function gen_credentials() - i = 0 - return () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i")) - end - - creds = AWSCredentials( - "access_key_id", "secret_key"; renew=gen_credentials(), expiry=now(UTC) - ) - - @test creds.renew !== nothing - renewed = creds.renew() - - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - @test creds.expiry <= now(UTC) - @test AWS._will_expire(creds) - - @test renewed.access_key_id === "NEW_ID_1" - @test renewed.secret_key == "NEW_KEY_1" - @test renewed.renew === nothing - @test renewed.expiry == typemax(DateTime) - @test !AWS._will_expire(renewed) - renew = creds.renew - - # Check renewal on time out - newcreds = check_credentials(creds; force_refresh=false) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_2" - @test creds.secret_key == "NEW_KEY_2" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - @test !AWS._will_expire(creds) - - # Check renewal doesn't happen if not forced or timed out - newcreds = check_credentials(creds; force_refresh=false) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_2" - @test creds.secret_key == "NEW_KEY_2" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - - # Check forced renewal works - newcreds = check_credentials(creds; force_refresh=true) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_3" - @test creds.secret_key == "NEW_KEY_3" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - end - - mktempdir() do dir - config_file = joinpath(dir, "config") - creds_file = joinpath(dir, "creds") - write( - config_file, - """ - [profile test] - output = json - - [profile test:dev] - source_profile = test - role_arn = arn:aws:iam::123456789000:role/Dev - - [profile test:sub-dev] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/SubDev - - [profile test2] - aws_access_key_id = WRONG_ACCESS_ID - aws_secret_access_key = WRONG_ACCESS_KEY - - [profile test3] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/test3 - - [profile test4] - aws_access_key_id = RIGHT_ACCESS_ID4 - aws_secret_access_key = RIGHT_ACCESS_KEY4 - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/test3 - """, - ) - - write( - creds_file, - """ - [test] - aws_access_key_id = TEST_ACCESS_ID - aws_secret_access_key = TEST_ACCESS_KEY - - [test2] - aws_access_key_id = RIGHT_ACCESS_ID2 - aws_secret_access_key = RIGHT_ACCESS_KEY2 - - [test3] - aws_access_key_id = RIGHT_ACCESS_ID3 - aws_secret_access_key = RIGHT_ACCESS_KEY3 - """, - ) - - withenv( - "AWS_SHARED_CREDENTIALS_FILE" => creds_file, - "AWS_CONFIG_FILE" => config_file, - "AWS_DEFAULT_PROFILE" => "test", - "AWS_PROFILE" => nothing, - "AWS_ACCESS_KEY_ID" => nothing, - "AWS_REGION" => "us-east-1", - ) do - @testset "Loading" begin - # Check credentials load - config = AWSConfig() - creds = config.credentials - - @test creds isa AWSCredentials - - @test creds.access_key_id == "TEST_ACCESS_ID" - @test creds.secret_key == "TEST_ACCESS_KEY" - @test creds.renew !== nothing - - # Check credential file takes precedence over config - withenv("AWS_DEFAULT_PROFILE" => "test2") do - config = AWSConfig() - creds = config.credentials - - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - end - - # Check credentials take precedence over role - withenv("AWS_DEFAULT_PROFILE" => "test3") do - config = AWSConfig() - creds = config.credentials - - @test creds.access_key_id == "RIGHT_ACCESS_ID3" - @test creds.secret_key == "RIGHT_ACCESS_KEY3" - end - - withenv("AWS_DEFAULT_PROFILE" => "test4") do - config = AWSConfig() - creds = config.credentials - - @test creds.access_key_id == "RIGHT_ACCESS_ID4" - @test creds.secret_key == "RIGHT_ACCESS_KEY4" - end - end - - @testset "Refresh" begin - withenv("AWS_DEFAULT_PROFILE" => "test") do - # Check credentials refresh on timeout - config = AWSConfig() - creds = config.credentials - creds.access_key_id = "EXPIRED_ACCESS_ID" - creds.secret_key = "EXPIRED_ACCESS_KEY" - creds.expiry = now(UTC) - - @test creds.renew !== nothing - renew = creds.renew - - @test renew() isa AWSCredentials - - creds = check_credentials(config.credentials) - - @test creds.access_key_id == "TEST_ACCESS_ID" - @test creds.secret_key == "TEST_ACCESS_KEY" - @test creds.expiry > now(UTC) - - # Check renew function remains unchanged - @test creds.renew !== nothing - @test creds.renew === renew - - # Check force_refresh - creds.access_key_id = "WRONG_ACCESS_KEY" - creds = check_credentials(creds; force_refresh=true) - @test creds.access_key_id == "TEST_ACCESS_ID" - end - end - - @testset "Profile" begin - # Check profile kwarg - withenv("AWS_DEFAULT_PROFILE" => "test") do - creds = AWSCredentials(; profile="test2") - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - - config = AWSConfig(; profile="test2") - creds = config.credentials - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - - # Check profile persists on renewal - creds.access_key_id = "WRONG_ACCESS_ID2" - creds.secret_key = "WRONG_ACCESS_KEY2" - creds = check_credentials(creds; force_refresh=true) - - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - end - end - - @testset "Assume Role" begin - # Check we try to assume a role - withenv("AWS_DEFAULT_PROFILE" => "test:dev") do - @test_ecode("InvalidClientTokenId", AWSConfig()) - end - - # Check we try to assume a role - withenv("AWS_DEFAULT_PROFILE" => "test:sub-dev") do - oldout = stdout - r, w = redirect_stdout() - - @test_ecode("InvalidClientTokenId", AWSConfig()) - redirect_stdout(oldout) - close(w) - output = String(read(r)) - occursin("Assuming \"test:dev\"", output) - occursin("Assuming \"test\"", output) - close(r) - end - end - end - end - - # Verify that the search order for credentials mirrors the behavior of the AWS CLI - # (version 2.11.13). Whenever support is added for new credential types new tests should - # be added to this test set. To determine the credential preference order used by AWS - # CLI it is recommended you use a set of valid credentials and a set of invalid - # credentials to determine the precedence. - # - # Documentation on credential precedence: - # - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence - # - https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html - # - https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html - @testset "Credential Precedence" begin - mktempdir() do dir - config_file = joinpath(dir, "config") - creds_file = joinpath(dir, "creds") - - basic_creds_content = """ - [profile1] - aws_access_key_id = AKI1 - aws_secret_access_key = SAK1 - - [profile2] - aws_access_key_id = AKI2 - aws_secret_access_key = SAK2 - """ - - ec2_json = Dict( - "AccessKeyId" => "AKI_EC2", - "SecretAccessKey" => "SAK_EC2", - "Token" => "TOK_EC2", - "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), - ) - - function ec2_metadata(url::AbstractString) - name = "local-credentials" - metadata_uri = "http://169.254.169.254/latest/meta-data" - if url == "$metadata_uri/iam/info" - return HTTP.Response(200, JSON.json("InstanceProfileArn" => "ARN0")) - elseif url == "$metadata_uri/iam/security-credentials/" - return HTTP.Response(200, name) - elseif url == "$metadata_uri/iam/security-credentials/$name" - return HTTP.Response(200, JSON.json(ec2_json)) - else - return HTTP.Response(404) - end - end - - ecs_json = Dict( - "AccessKeyId" => "AKI_ECS", - "SecretAccessKey" => "SAK_ECS", - "Token" => "TOK_ECS", - "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), - ) - - function ecs_metadata(url::AbstractString) - if startswith(url, "http://169.254.170.2/") - return HTTP.Response(200, JSON.json(ecs_json)) - else - return HTTP.Response(404) - end - end - - function ecs_metadata_localhost(url::AbstractString) - if startswith(url, "http://localhost:8080") - return HTTP.Response(200, JSON.json(ecs_json)) - else - return HTTP.Response(404) - end - end - - function http_request_patcher(funcs) - @patch function HTTP.request(method, url, args...; kwargs...) - local r - for f in funcs - r = f(string(url)) - r.status != 404 && break - end - return r - end - end - - withenv( - [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., - "AWS_SHARED_CREDENTIALS_FILE" => creds_file, - "AWS_CONFIG_FILE" => config_file, - ) do - @testset "explicit profile preferred" begin - isfile(config_file) && rm(config_file) - write(creds_file, basic_creds_content) - - withenv("AWS_PROFILE" => "profile1") do - creds = AWSCredentials(; profile="profile2") - @test creds.access_key_id == "AKI2" - end - - withenv( - "AWS_ACCESS_KEY_ID" => "AKI0", - "AWS_SECRET_ACCESS_KEY" => "SAK0", - # format trick: using this comment to force use of multiple lines - ) do - creds = AWSCredentials(; profile="profile2") - @test creds.access_key_id == "AKI2" - end - end - - @testset "AWS_ACCESS_KEY_ID preferred over AWS_PROFILE" begin - isfile(config_file) && rm(config_file) - write(creds_file, basic_creds_content) - - withenv( - "AWS_PROFILE" => "profile1", - "AWS_ACCESS_KEY_ID" => "AKI0", - "AWS_SECRET_ACCESS_KEY" => "SAK0", - ) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI0" - end - end - - # The AWS CLI used to use `AWS_DEFAULT_PROFILE` to set the AWS profile via the - # command line but this was deprecated in favor of `AWS_PROFILE`. We'll probably - # keeps support for this as long as AWS CLI continues to support it. - # https://github.com/aws/aws-cli/issues/2597 - @testset "AWS_PROFILE preferred over AWS_DEFAULT_PROFILE" begin - isfile(config_file) && rm(config_file) - write(creds_file, basic_creds_content) - - withenv( - "AWS_DEFAULT_PROFILE" => "profile1", - "AWS_PROFILE" => "profile2", - # format trick: using this comment to force use of multiple lines - ) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI2" - end - end - - @testset "Web identity preferred over SSO" begin - write( - config_file, - """ - [default] - sso_start_url = https://my-sso-portal.awsapps.com/start - sso_role_name = role1 - """, - ) - isfile(creds_file) && rm(creds_file) - - web_identity_file = joinpath(dir, "web_identity") - write(web_identity_file, "webid") - - patches = [ - Patches._assume_role_patch( - "AssumeRoleWithWebIdentity"; - access_key="AKI_WEB", - secret_key="SAK_WEB", - session_token="TOK_WEB", - ), - Patches.sso_service_patches("AKI_SSO", "SAK_SSO"), - Patches._imds_region_patch(nothing), - ] - - withenv( - "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, - "AWS_ROLE_ARN" => "webid", - ) do - apply(patches) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI_WEB" - end - end - end - - # TODO: Additional, precedence tests should be added for IAM Identity Center - # once support has been introduced. - @testset "IAM Identity Center preferred over legacy SSO" begin - write( - config_file, - """ - [sso-session my-sso] - sso_region = us-east-1 - sso_start_url = https://my-sso-portal.awsapps.com/start - - [default] - sso_session = my-sso - sso_start_url = https://my-legacy-sso-portal.awsapps.com/start - sso_role_name = role1 - """, - ) - isfile(creds_file) && rm(creds_file) - - apply(Patches.sso_service_patches("AKI_SSO", "SAK_SSO")) do - @test_throws ErrorException AWSCredentials() - end - end - - @testset "SSO preferred over credentials file" begin - write( - config_file, - """ - [profile profile1] - sso_start_url = https://my-sso-portal.awsapps.com/start - sso_role_name = role1 - """, - ) - write(creds_file, basic_creds_content) - - apply(Patches.sso_service_patches("AKI_SSO", "SAK_SSO")) do - creds = AWSCredentials(; profile="profile1") - @test creds.access_key_id == "AKI_SSO" - end - end - - @testset "Credential file over credential_process" begin - json = Dict( - "Version" => 1, - "AccessKeyId" => "AKI0", - "SecretAccessKey" => "SAK0", - # format trick: using this comment to force use of multiple lines - ) - write( - config_file, - """ - [profile profile1] - credential_process = echo '$(JSON.json(json))' - """, - ) - write(creds_file, basic_creds_content) - - creds = AWSCredentials(; profile="profile1") - @test creds.access_key_id == "AKI1" - end - - @testset "credential_process over config credentials" begin - json = Dict( - "Version" => 1, - "AccessKeyId" => "AKI0", - "SecretAccessKey" => "SAK0", - # format trick: using this comment to force use of multiple lines - ) - write( - config_file, - """ - [profile profile1] - aws_access_key_id = AKI1 - aws_secret_access_key = SAK1 - credential_process = echo '$(JSON.json(json))' - """, - ) - isfile(creds_file) && rm(creds_file) - - creds = AWSCredentials(; profile="profile1") - @test creds.access_key_id == "AKI0" - end - - @testset "default config credentials over ECS container credentials ENV variables" begin - write( - config_file, - """ - [default] - aws_access_key_id = AKI1 - aws_secret_access_key = SAK1 - """, - ) - isfile(creds_file) && rm(creds_file) - - withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do - apply(http_request_patcher([ecs_metadata])) do - @test isnothing(AWS._aws_get_profile(; default=nothing)) - - creds = AWSCredentials() - @test creds.access_key_id == "AKI1" - end - end - - withenv( - "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost:8080" - ) do - apply(http_request_patcher([ecs_metadata_localhost])) do - @test isnothing(AWS._aws_get_profile(; default=nothing)) - - creds = AWSCredentials() - @test creds.access_key_id == "AKI1" - end - end - end - - @testset "default config credentials over EC2 instance credentials" begin - write( - config_file, - """ - [default] - aws_access_key_id = AKI1 - aws_secret_access_key = SAK1 - """, - ) - isfile(creds_file) && rm(creds_file) - - apply(http_request_patcher([ec2_metadata])) do - @test isnothing(AWS._aws_get_profile(; default=nothing)) - - creds = AWSCredentials() - @test creds.access_key_id == "AKI1" - end - end - - @testset "ECS container credentials ENV variables over EC2 instance credentials" begin - isfile(config_file) && rm(config_file) - isfile(creds_file) && rm(creds_file) - - withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do - apply(http_request_patcher([ec2_metadata, ecs_metadata])) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI_ECS" - end - end - - withenv( - "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost:8080" - ) do - p = http_request_patcher([ec2_metadata, ecs_metadata_localhost]) - apply(p) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI_ECS" - end - end - end - - # Note: It appears that the ECS container credentials are only used when - # a `AWS_CONTAINER_*` environmental variable is set. However, this test - # ensures that if we do add implicit support that the documented precedence - # order is not violated. - @testset "EC2 instance credentials over ECS container credentials" begin - isfile(config_file) && rm(config_file) - isfile(creds_file) && rm(creds_file) - - apply(http_request_patcher([ec2_metadata, ecs_metadata])) do - creds = AWSCredentials() - @test creds.access_key_id == "AKI_EC2" - end - end - end - end - end -end - -@testset "Retrieving AWS Credentials" begin - test_values = Dict{String,Any}( - "Default-Profile" => "default", - "Test-Profile" => "test", - "Test-Config-Profile" => "test", - "AccessKeyId" => "Default-Key", - "SecretAccessKey" => "Default-Secret", - "Test-AccessKeyId" => "Test-Key", - "Test-SecretAccessKey" => "Test-Secret", - "Token" => "Test-Token", - "InstanceProfileArn" => "Test-Arn", - "RoleArn" => "Test-Arn", - "Expiration" => now(UTC), - "Security-Credentials" => "Test-Security-Credentials", - "Test-SSO-Profile" => "sso-test", - "Test-SSO-start-url" => "https://test-sso.com/start", - "Test-SSO-Role" => "SSORoleName", - ) - - @testset "~/.aws/config - Default Profile" begin - mktemp() do config_file, config_io - write( - config_io, - """ - [$(test_values["Default-Profile"])] - aws_access_key_id=$(test_values["AccessKeyId"]) - aws_secret_access_key=$(test_values["SecretAccessKey"]) - """, - ) - close(config_io) - - withenv("AWS_CONFIG_FILE" => config_file) do - default_profile = dot_aws_config(test_values["Default-Profile"]) - - @test default_profile.access_key_id == test_values["AccessKeyId"] - @test default_profile.secret_key == test_values["SecretAccessKey"] - end - end - end - - @testset "~/.aws/config - Specified Profile" begin - mktemp() do config_file, config_io - write( - config_io, - """ - [profile $(test_values["Test-Config-Profile"])] - aws_access_key_id=$(test_values["Test-AccessKeyId"]) - aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) - """, - ) - close(config_io) - - withenv("AWS_CONFIG_FILE" => config_file) do - specified_result = dot_aws_config(test_values["Test-Profile"]) - - @test specified_result.access_key_id == test_values["Test-AccessKeyId"] - @test specified_result.secret_key == test_values["Test-SecretAccessKey"] - end - end - end - - @testset "~/.aws/config - Specified SSO Profile" begin - mktemp() do config_file, config_io - write( - config_io, - """ - [profile $(test_values["Test-SSO-Profile"])] - sso_start_url=$(test_values["Test-SSO-start-url"]) - sso_role_name=$(test_values["Test-SSO-Role"]) - """, - ) - close(config_io) - - withenv("AWS_CONFIG_FILE" => config_file) do - apply( - Patches.sso_service_patches( - test_values["AccessKeyId"], test_values["SecretAccessKey"] - ), - ) do - specified_result = sso_credentials(test_values["Test-SSO-Profile"]) - - @test specified_result.access_key_id == test_values["AccessKeyId"] - @test specified_result.secret_key == test_values["SecretAccessKey"] - end - end - end - end - - @testset "~/.aws/config - Credential Process" begin - mktempdir() do dir - config_file = joinpath(dir, "config") - credential_process_file = joinpath(dir, "cred_process") - open(credential_process_file, "w") do io - println(io, "#!/bin/sh") - println(io, "cat < 1, - "AccessKeyId" => test_values["Test-AccessKeyId"], - "SecretAccessKey" => test_values["Test-SecretAccessKey"], - ) - JSON.print(io, json) - println(io, "\nEOF") - end - chmod(credential_process_file, 0o700) - - withenv("AWS_CONFIG_FILE" => config_file) do - open(config_file, "w") do io - write( - io, - """ - [profile $(test_values["Test-Config-Profile"])] - credential_process = $(abspath(credential_process_file)) - """, - ) - end - - result = dot_aws_config(test_values["Test-Config-Profile"]) - - @test result.access_key_id == test_values["Test-AccessKeyId"] - @test result.secret_key == test_values["Test-SecretAccessKey"] - @test isempty(result.token) - @test result.expiry == typemax(DateTime) - end - end - end - - @testset "~/.aws/creds - Default Profile" begin - mktemp() do creds_file, creds_io - write( - creds_io, - """ - [$(test_values["Default-Profile"])] - aws_access_key_id=$(test_values["AccessKeyId"]) - aws_secret_access_key=$(test_values["SecretAccessKey"]) - """, - ) - close(creds_io) - - withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do - specified_result = dot_aws_credentials(test_values["Default-Profile"]) - - @test specified_result.access_key_id == test_values["AccessKeyId"] - @test specified_result.secret_key == test_values["SecretAccessKey"] - end - end - end - - @testset "~/.aws/creds - Specified Profile" begin - mktemp() do creds_file, creds_io - write( - creds_io, - """ - [$(test_values["Test-Profile"])] - aws_access_key_id=$(test_values["Test-AccessKeyId"]) - aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) - """, - ) - close(creds_io) - - withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do - specified_result = dot_aws_credentials(test_values["Test-Profile"]) - - @test specified_result.access_key_id == test_values["Test-AccessKeyId"] - @test specified_result.secret_key == test_values["Test-SecretAccessKey"] - end - end - end - - @testset "Environment Variables" begin - withenv( - "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], - "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"], - ) do - aws_creds = env_var_credentials() - @test aws_creds.access_key_id == test_values["AccessKeyId"] - @test aws_creds.secret_key == test_values["SecretAccessKey"] - end - end - - @testset "Instance - EC2" begin - role_name = "foobar" - role_arn = "arn:aws:sts::1234:assumed-role/$role_name" - access_key = "access-key-$(randstring(6))" - secret_key = "secret-key-$(randstring(6))" - session_token = "session-token-$(randstring(6))" - session_name = "$role_name-session" - - assume_role_patch = Patches._assume_role_patch( - "AssumeRole"; - access_key=access_key, - secret_key=secret_key, - session_token=session_token, - role_arn=role_arn, - ) - ec2_metadata_patch = @patch function HTTP.request(method, url, args...; kwargs...) - url = string(url) - security_credentials = test_values["Security-Credentials"] - - metadata_uri = "http://169.254.169.254/latest/meta-data" - if url == "$metadata_uri/iam/info" - json = JSON.json("InstanceProfileArn" => test_values["InstanceProfileArn"]) - return HTTP.Response(200, json) - elseif url == "$metadata_uri/iam/security-credentials/" - return HTTP.Response(200, security_credentials) - elseif url == "$metadata_uri/iam/security-credentials/$security_credentials" - return HTTP.Response(200, JSON.json(test_values)) - else - return HTTP.Response(404) - end - end - - apply([assume_role_patch, ec2_metadata_patch]) do - result = ec2_instance_credentials("default") - @test result.access_key_id == test_values["AccessKeyId"] - @test result.secret_key == test_values["SecretAccessKey"] - @test result.token == test_values["Token"] - @test result.user_arn == test_values["InstanceProfileArn"] - @test result.expiry == test_values["Expiration"] - @test result.renew !== nothing - - result = mktemp() do config_file, config_io - write( - config_io, - """ - [profile $role_name] - credential_source = Ec2InstanceMetadata - role_arn = $role_arn - """, - ) - close(config_io) - - withenv( - "AWS_CONFIG_FILE" => config_file, - "AWS_ROLE_SESSION_NAME" => session_name, - ) do - ec2_instance_credentials(role_name) - end - end - - @test result.access_key_id == access_key - @test result.secret_key == secret_key - @test result.token == session_token - @test result.user_arn == "$(role_arn)/$(session_name)" - @test result.renew !== nothing - end - end - - @testset "Instance - ECS" begin - expiration = floor(now(UTC), Second) - rel_uri_json = Dict( - "AccessKeyId" => "AKI_REL_ECS", - "SecretAccessKey" => "SAK_REL_ECS", - "Token" => "TOK_REL_ECS", - "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), - "RoleArn" => "ROLE_REL_ECS", - ) - - rel_uri_patch = @patch function HTTP.request(::String, url, headers=[]; kwargs...) - url = string(url) - - @test url == "http://169.254.170.2/get-credentials" - @test isempty(headers) - - if url == "http://169.254.170.2/get-credentials" - return HTTP.Response(200, JSON.json(rel_uri_json)) - else - return HTTP.Response(404) - end - end - - withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-credentials") do - apply(rel_uri_patch) do - result = ecs_instance_credentials() - @test result.access_key_id == rel_uri_json["AccessKeyId"] - @test result.secret_key == rel_uri_json["SecretAccessKey"] - @test result.token == rel_uri_json["Token"] - @test result.user_arn == rel_uri_json["RoleArn"] - @test result.expiry == expiration - @test result.renew == ecs_instance_credentials - end - end - - # When the environmental variable isn't set then the ECS credential provider is - # unavailable. - withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing) do - @test ecs_instance_credentials() === nothing - end - - # Specifying the environmental variable results in us attempting to connect to the - # ECS credential provider. - withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/invalid") do - # Internally throws a `ConnectError` exception - @test ecs_instance_credentials() === nothing - end - - full_uri_json = Dict( - "AccessKeyId" => "AKI_FULL_ECS", - "SecretAccessKey" => "SAK_FULL_ECS", - "Token" => "TOK_FULL_ECS", - "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), - "RoleArn" => "ROLE_FULL_ECS", - ) - - full_uri_patch = @patch function HTTP.request(::String, url, headers=[]; kwargs...) - url = string(url) - authorization = http_header(headers, "Authorization") - - @test url == "http://localhost/get-credentials" - @test authorization == "Basic abcd" - - if url == "http://localhost/get-credentials" && authorization == "Basic abcd" - return HTTP.Response(200, JSON.json(full_uri_json)) - else - return HTTP.Response(403) - end - end - - withenv( - "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost/get-credentials", - "AWS_CONTAINER_AUTHORIZATION_TOKEN" => "Basic abcd", - ) do - apply(full_uri_patch) do - result = ecs_instance_credentials() - @test result.access_key_id == full_uri_json["AccessKeyId"] - @test result.secret_key == full_uri_json["SecretAccessKey"] - @test result.token == full_uri_json["Token"] - @test result.user_arn == full_uri_json["RoleArn"] - @test result.expiry == expiration - @test result.renew == ecs_instance_credentials - end - end - - # `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` should be preferred over - # `AWS_CONTAINER_CREDENTIALS_FULL_URI`. - withenv( - "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-credentials", - "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost/get-credentials", - ) do - apply(rel_uri_patch) do - result = ecs_instance_credentials() - @test result.access_key_id == rel_uri_json["AccessKeyId"] - end - end - end - - @testset "Web Identity File" begin - @test credentials_from_webtoken() == nothing - - mktempdir() do dir - web_identity_file = joinpath(dir, "web_identity") - write(web_identity_file, "foobar") - session_name = "foobar-session" - - access_key = "access-key-$(randstring(6))" - secret_key = "secret-key-$(randstring(6))" - session_token = "session-token-$(randstring(6))" - role_arn = "arn:aws:sts::1234:assumed-role/foobar" - - patch = Patches._assume_role_patch( - "AssumeRoleWithWebIdentity"; - access_key=access_key, - secret_key=secret_key, - session_token=session_token, - role_arn=role_arn, - expiry=duration -> now(UTC), # expire immediately to check renewal - ) - - withenv( - "AWS_ROLE_ARN" => "foobar", - "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, - "AWS_ROLE_SESSION_NAME" => session_name, - ) do - apply(patch) do - result = credentials_from_webtoken() - - @test result.access_key_id == access_key - @test result.secret_key == secret_key - @test result.token == session_token - @test result.user_arn == "$(role_arn)/$(session_name)" - @test result.renew == credentials_from_webtoken - expiry = result.expiry - sleep(0.1) - result = check_credentials(result) - - @test result.access_key_id == access_key - @test result.secret_key == secret_key - @test result.token == session_token - @test result.user_arn == "$(role_arn)/$(session_name)" - @test result.renew == credentials_from_webtoken - @test expiry != result.expiry - end - end - - session_name = "AWS.jl-role-foobar-20210101T000000Z" - patches = [ - patch - @patch Dates.now(::Type{UTC}) = DateTime(2021) - ] - - withenv( - "AWS_ROLE_ARN" => "foobar", - "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, - "AWS_ROLE_SESSION_NAME" => nothing, - ) do - apply(patches) do - result = credentials_from_webtoken() - @test result.user_arn == "$(role_arn)/$(session_name)" - end - end - end - end - - @testset "Credential Process" begin - gen_process(json) = Cmd(["echo", JSON.json(json)]) - - long_term_resp = Dict( - "Version" => 1, - "AccessKeyId" => "access-key", - "SecretAccessKey" => "secret-key", - # format trick: using this comment to force use of multiple lines - ) - creds = external_process_credentials(gen_process(long_term_resp)) - @test creds.access_key_id == long_term_resp["AccessKeyId"] - @test creds.secret_key == long_term_resp["SecretAccessKey"] - @test isempty(creds.token) - @test creds.expiry == typemax(DateTime) - - expiration = floor(now(UTC), Second) - temporary_resp = Dict( - "Version" => 1, - "AccessKeyId" => "access-key", - "SecretAccessKey" => "secret-key", - "SessionToken" => "session-token", - "Expiration" => Dates.format(expiration, EXPIRATION_FMT), - ) - creds = external_process_credentials(gen_process(temporary_resp)) - @test creds.access_key_id == temporary_resp["AccessKeyId"] - @test creds.secret_key == temporary_resp["SecretAccessKey"] - @test creds.token == temporary_resp["SessionToken"] - @test creds.expiry == expiration - - unhandled_version_resp = Dict("Version" => 2) - json = sprint(JSON.print, unhandled_version_resp, 2) - ex = ErrorException("Credential process returned unhandled version 2:\n$json") - @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) - - missing_token_resp = Dict( - "Version" => 1, - "AccessKeyId" => "access-key", - "SecretAccessKey" => "secret-key", - "Expiration" => Dates.format(expiration, EXPIRATION_FMT), - ) - ex = KeyError("SessionToken") - @test_throws ex external_process_credentials(gen_process(missing_token_resp)) - - missing_expiration_resp = Dict( - "Version" => 1, - "AccessKeyId" => "access-key", - "SecretAccessKey" => "secret-key", - "SessionToken" => "session-token", - ) - ex = KeyError("Expiration") - @test_throws ex external_process_credentials(gen_process(missing_expiration_resp)) - end - - @testset "Credentials Not Found" begin - patches = [ - @patch function HTTP.request(method::String, url, args...; kwargs...) - throw(HTTP.Exceptions.ConnectError(string(url), "host is unreachable")) - end - Patches._cred_file_patch - Patches._config_file_patch - ] - - withenv( - "AWS_ACCESS_KEY_ID" => nothing, - "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing, - ) do - apply(patches) do - @test_throws NoCredentials AWSConfig() - end - end - end - - @testset "Helper functions" begin - @testset "Check Credentials - EnvVars" begin - withenv( - "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], - "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"], - ) do - testAWSCredentials = AWSCredentials( - test_values["AccessKeyId"], - test_values["SecretAccessKey"]; - expiry=Dates.now(UTC) - Minute(10), - renew=env_var_credentials, - ) - - result = check_credentials(testAWSCredentials; force_refresh=true) - @test result.access_key_id == testAWSCredentials.access_key_id - @test result.secret_key == testAWSCredentials.secret_key - @test result.expiry == typemax(DateTime) - @test result.renew == testAWSCredentials.renew - end - end - end -end - -@testset "aws_get_region" begin - mktempdir() do dir - config_str = """ - [default] - region = us-west-2 - - [profile test] - region = ap-northeast-1 - """ - config_file = joinpath(dir, "config") - write(config_file, config_str) - ini = read(Inifile(), IOBuffer(config_str)) - - @testset "environmental variable (AWS_REGION)" begin - withenv("AWS_REGION" => "eu-west-1", "AWS_DEFAULT_REGION" => nothing) do - @test aws_get_region(; config=ini, profile="default") == "eu-west-1" - @test aws_get_region() == "eu-west-1" - end - - withenv("AWS_REGION" => nothing, "AWS_DEFAULT_REGION" => "us-gov-east-1") do - @test aws_get_region(; config=ini, profile="default") == "us-gov-east-1" - @test aws_get_region() == "us-gov-east-1" - end - - withenv("AWS_REGION" => "eu-west-1", "AWS_DEFAULT_REGION" => "us-gov-east-1") do - @test aws_get_region(; config=ini, profile="default") == "eu-west-1" - @test aws_get_region() == "eu-west-1" - end - end - - @testset "default profile" begin - withenv("AWS_REGION" => nothing, "AWS_DEFAULT_REGION" => nothing) do - @test aws_get_region(; config=ini, profile="default") == "us-west-2" - @test aws_get_region(; config=config_file, profile="default") == "us-west-2" - end - - withenv( - "AWS_REGION" => nothing, - "AWS_DEFAULT_REGION" => nothing, - "AWS_CONFIG_FILE" => config_file, - "AWS_PROFILE" => nothing, - "AWS_DEFAULT_PROFILE" => nothing, - ) do - @test aws_get_region() == "us-west-2" - end - end - - @testset "specified profile" begin - withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do - @test aws_get_region(; config=ini, profile="test") == "ap-northeast-1" - @test aws_get_region(; config=config_file, profile="test") == - "ap-northeast-1" - end - - withenv( - "AWS_REGION" => nothing, - "AWS_DEFAULT_REGION" => nothing, - "AWS_CONFIG_FILE" => config_file, - "AWS_PROFILE" => "test", - ) do - @test aws_get_region() == "ap-northeast-1" - end - end - - @testset "unknown profile" begin - withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do - apply(Patches._imds_region_patch(nothing)) do - @test aws_get_region(; config=ini, profile="unknown") == - AWS.DEFAULT_REGION - @test aws_get_region(; config=config_file, profile="unknown") == - AWS.DEFAULT_REGION - end - end - - withenv( - "AWS_REGION" => nothing, - "AWS_DEFAULT_REGION" => nothing, - "AWS_CONFIG_FILE" => config_file, - "AWS_PROFILE" => "unknown", - ) do - apply(Patches._imds_region_patch(nothing)) do - @test aws_get_region() == AWS.DEFAULT_REGION - end - end - end - - @testset "default keyword" begin - default = nothing - withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do - apply(Patches._imds_region_patch(nothing)) do - @test aws_get_region(; config=ini, profile="unknown", default) === - default - @test aws_get_region(; - config=config_file, profile="unknown", default - ) === default - end - end - - withenv( - "AWS_REGION" => nothing, - "AWS_DEFAULT_REGION" => nothing, - "AWS_CONFIG_FILE" => config_file, - "AWS_PROFILE" => "unknown", - ) do - apply(Patches._imds_region_patch(nothing)) do - @test aws_get_region(; default=default) === default - end - end - end - - @testset "no such config file" begin - withenv( - "AWS_DEFAULT_REGION" => nothing, - "AWS_REGION" => nothing, - "AWS_CONFIG_FILE" => tempname(), - ) do - apply(Patches._imds_region_patch(nothing)) do - @test aws_get_region() == AWS.DEFAULT_REGION - end - end - end - - @testset "instance profile" begin - withenv( - "AWS_DEFAULT_REGION" => nothing, - "AWS_REGION" => nothing, - "AWS_CONFIG_FILE" => tempname(), - ) do - apply(Patches._imds_region_patch("ap-atlantis-1")) do - @test aws_get_region() == "ap-atlantis-1" - end - end - end - end -end diff --git a/test/configs/default-role/config b/test/config/default-role/config similarity index 100% rename from test/configs/default-role/config rename to test/config/default-role/config diff --git a/test/configs/default-role/credentials b/test/config/default-role/credentials similarity index 100% rename from test/configs/default-role/credentials rename to test/config/default-role/credentials diff --git a/test/configs/role-with-mfa/config b/test/config/role-with-mfa/config similarity index 100% rename from test/configs/role-with-mfa/config rename to test/config/role-with-mfa/config diff --git a/test/configs/role-with-mfa/credentials b/test/config/role-with-mfa/credentials similarity index 100% rename from test/configs/role-with-mfa/credentials rename to test/config/role-with-mfa/credentials diff --git a/test/integration/AWS.jl b/test/integration/AWS.jl new file mode 100644 index 000000000..11f02aac5 --- /dev/null +++ b/test/integration/AWS.jl @@ -0,0 +1,541 @@ +@testset "STS" begin + @testset "high-level" begin + @service STS + + response = STS.get_caller_identity() + d = response["GetCallerIdentityResult"] + + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) + end + + @testset "low-level" begin + response = AWSServices.sts("GetCallerIdentity") + d = response["GetCallerIdentityResult"] + + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test occursin(r"^arn:aws:(iam|sts):", d["Arn"]) + @test all(isdigit, d["Account"]) + end +end + +@testset "json" begin + @testset "high-level secrets manager" begin + @service Secrets_Manager + + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" + + function _get_secret_string(secret_name) + response = Secrets_Manager.get_secret_value(secret_name) + + return response["SecretString"] + end + + Secrets_Manager.create_secret( + secret_name, + LittleDict( + "SecretString" => secret_string, "ClientRequestToken" => string(uuid4()) + ), + ) + + try + @test _get_secret_string(secret_name) == secret_string + finally + Secrets_Manager.delete_secret( + secret_name, LittleDict("ForceDeleteWithoutRecovery" => "true") + ) + end + + @test_throws AWSException _get_secret_string(secret_name) + end + + @testset "low-level secrets manager" begin + secret_name = "aws-jl-test---" * _now_formatted() + secret_string = "sshhh it is a secret!" + + function _get_secret_string(secret_name) + response = AWSServices.secrets_manager( + "GetSecretValue", LittleDict("SecretId" => secret_name) + ) + + return response["SecretString"] + end + + resp = AWSServices.secrets_manager( + "CreateSecret", + LittleDict( + "Name" => secret_name, + "SecretString" => secret_string, + "ClientRequestToken" => string(uuid4()), + ), + ) + + try + @test _get_secret_string(secret_name) == secret_string + finally + AWSServices.secrets_manager( + "DeleteSecret", + LittleDict( + "SecretId" => secret_name, "ForceDeleteWithoutRecovery" => "true" + ), + ) + end + + @test_throws AWSException _get_secret_string(secret_name) + end +end + +@testset "query" begin + @testset "high-level iam" begin + @service IAM + + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], + ) + expected_policy_document = JSON.json(expected_policy_document) + + response = IAM.create_policy(expected_policy_document, expected_policy_name) + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] + + try + response_policy_version = IAM.get_policy_version(policy_arn, "v1") + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + IAM.delete_policy(policy_arn) + end + + @test_throws AWSException IAM.get_policy(policy_arn) + end + + @testset "low-level iam" begin + policy_arn = "" + expected_policy_name = "aws-jl-test---" * _now_formatted() + expected_policy_document = LittleDict( + "Version" => "2012-10-17", + "Statement" => [ + LittleDict( + "Effect" => "Allow", + "Action" => ["s3:Get*", "s3:List*"], + "Resource" => ["arn:aws:s3:::my-bucket/shared/*"], + ), + ], + ) + expected_policy_document = JSON.json(expected_policy_document) + + response = AWSServices.iam( + "CreatePolicy", + LittleDict( + "PolicyName" => expected_policy_name, + "PolicyDocument" => expected_policy_document, + ), + ) + policy_arn = response["CreatePolicyResult"]["Policy"]["Arn"] + + try + response_policy_version = AWSServices.iam( + "GetPolicyVersion", + LittleDict("PolicyArn" => policy_arn, "VersionId" => "v1"), + ) + response_document = response_policy_version["GetPolicyVersionResult"]["PolicyVersion"]["Document"] + @test HTTP.unescapeuri(response_document) == expected_policy_document + finally + AWSServices.iam("DeletePolicy", LittleDict("PolicyArn" => policy_arn)) + end + + @test_throws AWSException AWSServices.iam( + "GetPolicy", LittleDict("PolicyArn" => policy_arn) + ) + end + + @testset "high-level sqs" begin + @service SQS + + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" + + function _get_queue_url(queue_name) + result = SQS.get_queue_url(queue_name) + + return result["QueueUrl"] + end + + # Create Queue + SQS.create_queue(queue_name) + queue_url = _get_queue_url(queue_name) + + try + # Get Queues + @test !isempty(queue_url) + + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" + + SQS.send_message(expected_message, queue_url) + + response = SQS.receive_message(queue_url) + receipt_handle = only(response["Messages"])["ReceiptHandle"] + + response = SQS.delete_message_batch( + [ + LittleDict( + "Id" => expected_message_id, "ReceiptHandle" => receipt_handle + ), + ], + queue_url, + ) + + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id + + SQS.send_message(expected_message, queue_url) + + result = SQS.receive_message(queue_url) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + SQS.delete_queue(queue_url) + end + + @test_throws AWSException _get_queue_url(queue_name) + end + + @testset "low-level sqs" begin + queue_name = "aws-jl-test---" * _now_formatted() + expected_message = "Hello for AWS.jl" + + function _get_queue_url(queue_name) + result = AWSServices.sqs("GetQueueUrl", LittleDict("QueueName" => queue_name)) + + return result["QueueUrl"] + end + + # Create Queue + AWSServices.sqs("CreateQueue", LittleDict("QueueName" => queue_name)) + + queue_url = _get_queue_url(queue_name) + @test !isempty(queue_url) + + try + # Change Message Visibility Batch Request + expected_message_id = "aws-jl-test" + + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) + + response = AWSServices.sqs( + "ReceiveMessage", LittleDict("QueueUrl" => queue_url) + ) + receipt_handle = only(response["Messages"])["ReceiptHandle"] + + response = AWSServices.sqs( + "DeleteMessageBatch", + LittleDict( + "QueueUrl" => queue_url, + "Entries" => [ + LittleDict( + "Id" => expected_message_id, + "ReceiptHandle" => receipt_handle, + ), + ], + ), + ) + + message_id = only(response["Successful"])["Id"] + @test message_id == expected_message_id + + # Send message + AWSServices.sqs( + "SendMessage", + LittleDict("QueueUrl" => queue_url, "MessageBody" => expected_message), + ) + + # Receive Message + result = AWSServices.sqs("ReceiveMessage", LittleDict("QueueUrl" => queue_url)) + message = only(result["Messages"])["Body"] + @test message == expected_message + finally + AWSServices.sqs("DeleteQueue", LittleDict("QueueUrl" => queue_url)) + end + + @test_throws AWSException _get_queue_url(queue_name) + end +end + +@testset "rest-xml" begin + @testset "high-level s3" begin + @service S3 + + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = string(uuid4()) + + function _bucket_exists(bucket_name) + try + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end + end + end + + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = S3.list_objects(bucket_name, Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # GET with an IO target + mktemp() do f, io + S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) + flush(io) + @test read(f, String) == body + end + finally + # DELETE with parameters operation + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) + + # DELETE operation + S3.delete_bucket(bucket_name) + + sleep(2) + end + + @test _bucket_exists(bucket_name) == false + end + + @testset "low-level s3" begin + bucket_name = "aws-jl-test---" * _now_formatted() + file_name = "*)=('! +@,:.txt" # Special characters which S3 allows + + function _bucket_exists(bucket_name) + try + AWSServices.s3("HEAD", "/$bucket_name") + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end + end + end + + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + AWSServices.s3("PUT", "/$bucket_name") + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = Array{UInt8}("sample-file-body") + AWSServices.s3("PUT", "/$bucket_name/$file_name", Dict("body" => body)) + @test AWSServices.s3("GET", "/$bucket_name/$file_name") == body + + # GET operation + result = AWSServices.s3("GET", "/$bucket_name") + @test result["Contents"]["Key"] == file_name + + # GET with parameters operation + max_keys = 1 + result = AWSServices.s3("GET", "/$bucket_name", Dict("max_keys" => max_keys)) + @test length([result["Contents"]]) == max_keys + + # POST with parameters operation + body = """ + + + $file_name + + + """ + + AWSServices.s3("POST", "/$bucket_name?delete", Dict("body" => body)) + @test_throws AWSException AWSServices.s3("GET", "/$bucket_name/$file_name") + finally + # DELETE operation + AWSServices.s3("DELETE", "/$bucket_name") + + sleep(2) + end + + @test _bucket_exists(bucket_name) == false + end + + @testset "additional S3 operations" begin + @service S3 + + bucket_name = "aws-jl-test---" * _now_formatted() + + # Testing a file name with various special & Unicode characters + file_name = "$(uuid4())/📁!!/@ +*" + + function _bucket_exists(bucket_name) + try + S3.head_bucket(bucket_name) + return true + catch e + if e isa AWSException && e.cause.status == 404 + return false + else + rethrow(e) + end + end + end + + # HEAD operation + @test _bucket_exists(bucket_name) == false + + # PUT operation + S3.create_bucket(bucket_name) + @test _bucket_exists(bucket_name) + + try + # PUT with parameters operation + body = "sample-file-body" + S3.put_object(bucket_name, file_name, Dict("body" => body)) + @test !isempty(S3.get_object(bucket_name, file_name)) + + # GET operation + result = S3.list_objects(bucket_name) + @test result["Contents"]["Key"] == file_name + finally + # DELETE the file, check that it's gone, and then DELETE the bucket + S3.delete_object(bucket_name, file_name) + @test_throws AWSException S3.get_object(bucket_name, file_name) + S3.delete_bucket(bucket_name) + sleep(2) + end + + @test _bucket_exists(bucket_name) == false + end +end + +@testset "rest-json" begin + @testset "high-level glacier" begin + @service Glacier + + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + + # PUT + for vault in vault_names + Glacier.create_vault("-", vault) + end + + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + + for vault in vault_names + Glacier.add_tags_to_vault("-", vault, tags) + end + + for vault in vault_names + result_tags = Glacier.list_tags_for_vault("-", vault) + @test result_tags == tags + end + + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + args = LittleDict("limit" => limit) + result = Glacier.list_vaults("-", args) + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE + for vault in vault_names + Glacier.delete_vault("-", vault) + end + end + + result = Glacier.list_vaults("-") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] + + for vault in vault_names + @test !(vault in res_vault_names) + end + end + + @testset "low-level glacier" begin + timestamp = _now_formatted() + vault_names = ["aws-jl-test-01---$timestamp", "aws-jl-test-02---$timestamp"] + + # PUT + for vault in vault_names + AWSServices.glacier("PUT", "/-/vaults/$vault") + end + + try + # POST + tags = Dict("Tags" => LittleDict("Tag-01" => "Tag-01", "Tag-02" => "Tag-02")) + + for vault in vault_names + AWSServices.glacier("POST", "/-/vaults/$vault/tags?operation=add", tags) + end + + for vault in vault_names + result_tags = AWSServices.glacier("GET", "/-/vaults/$vault/tags") + + @test result_tags == tags + end + + # GET + # If this is an Integer AWS Coral cannot convert it to a String + # "class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String" + limit = "1" + params = LittleDict("limit" => limit) + result = AWSServices.glacier("GET", "/-/vaults/", params) + + @test length(result["VaultList"]) == parse(Int, limit) + finally + # DELETE + for vault in vault_names + AWSServices.glacier("DELETE", "/-/vaults/$vault") + end + end + + result = AWSServices.glacier("GET", "/-/vaults") + res_vault_names = [v["VaultName"] for v in result["VaultList"]] + + for vault in vault_names + @test !(vault in res_vault_names) + end + end +end diff --git a/test/integration/AWSCredentials.jl b/test/integration/AWSCredentials.jl new file mode 100644 index 000000000..b1e790305 --- /dev/null +++ b/test/integration/AWSCredentials.jl @@ -0,0 +1,21 @@ +@testset "Load Credentials" begin + config = global_aws_config() + user = aws_user_arn(config) + @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) + config.region = "us-east-1" + + @test_ecode("InvalidAction", AWSServices.iam("GetFoo")) + + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSServices.iam("GetUser", Dict("UserName" => "notauser")) + ) + + @test_ecode("ValidationError", AWSServices.iam("GetUser", Dict("UserName" => "@#!%%!"))) + + # Please note: If testing in a managed Corporate AWS environment, this can set off alarms... + @test_ecode( + ["AccessDenied", "EntityAlreadyExists"], + AWSServices.iam("CreateUser", Dict("UserName" => "root")) + ) +end diff --git a/test/issues.jl b/test/integration/issues.jl similarity index 61% rename from test/issues.jl rename to test/integration/issues.jl index f0821da76..61542b725 100644 --- a/test/issues.jl +++ b/test/integration/issues.jl @@ -151,82 +151,6 @@ try S3.delete_object(BUCKET_NAME, file_name) end end - - # https://github.com/JuliaCloud/AWS.jl/issues/515 - @testset "issue 515" begin - function _incomplete_patch(; data, num_attempts_to_fail=4) - attempt_num = 0 - n = length(data) - - function _downloads_response(content_length) - headers = ["content-length" => string(content_length)] - return Downloads.Response("http", "", 200, "HTTP/1.1 200 OK", headers) - end - - patch = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend - @patch function HTTP.request(args...; response_stream, kwargs...) - attempt_num += 1 - if attempt_num <= num_attempts_to_fail - write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained - throw(HTTP.RequestError(HTTP.Request(), EOFError())) - else - write(response_stream, data) - return HTTP.Response(200, "{\"Location\": \"us-east-1\"}") - end - end - elseif AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend - @patch function Downloads.request(args...; output, kwargs...) - attempt_num += 1 - if attempt_num <= num_attempts_to_fail - write(output, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained - message = "transfer closed with 1 bytes remaining to read" - e = Downloads.RequestError("", 18, message, _downloads_response(n)) - throw(e) - else - write(output, data) - return _downloads_response(n) - end - end - end - - return patch - end - - n = 100 - data = rand(UInt8, n) - bucket = "julialang2" # use public bucket as dummy - key = "bin/versions.json" - config = AWSConfig(; creds=nothing) - - @testset "Fail 2 attempts then succeed" begin - apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do - retrieved = S3.get_object(bucket, key; aws_config=config) - - @test length(retrieved) == n - @test retrieved == data - end - end - - @testset "Fail all 4 attempts then throw" begin - err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend - HTTP.RequestError - else - Downloads.RequestError - end - io = IOBuffer() - - apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do - params = Dict("response_stream" => io) - @test_throws err_t S3.get_object(bucket, key, params; aws_config=config) - - seekstart(io) - retrieved = read(io) - @test length(retrieved) == n - 1 - @test retrieved == data[1:(n - 1)] - end - end - end - finally S3.delete_bucket(BUCKET_NAME) end diff --git a/test/minio.jl b/test/integration/minio.jl similarity index 100% rename from test/minio.jl rename to test/integration/minio.jl diff --git a/test/role.jl b/test/integration/role.jl similarity index 99% rename from test/role.jl rename to test/integration/role.jl index 310923fb9..d4fb83c3e 100644 --- a/test/role.jl +++ b/test/integration/role.jl @@ -48,7 +48,7 @@ end @testset "assume_role / assume_role_creds" begin # In order to mitigate the effects of using `assume_role` in order to test itself we'll # use the lowest-level call with as many defaults as possible. - base_config = AWS_CONFIG[] + base_config = global_aws_config() creds = assume_role_creds(base_config, testset_role("AssumeRoleTestset")) config = AWSConfig(; creds) @test get_assumed_role(config) == testset_role("AssumeRoleTestset") diff --git a/test/resources/Manifest.toml b/test/resource/Manifest.toml similarity index 100% rename from test/resources/Manifest.toml rename to test/resource/Manifest.toml diff --git a/test/resources/Project.toml b/test/resource/Project.toml similarity index 100% rename from test/resources/Project.toml rename to test/resource/Project.toml diff --git a/test/resources/TestPkg/Project.toml b/test/resource/TestPkg/Project.toml similarity index 100% rename from test/resources/TestPkg/Project.toml rename to test/resource/TestPkg/Project.toml diff --git a/test/resources/TestPkg/src/TestPkg.jl b/test/resource/TestPkg/src/TestPkg.jl similarity index 100% rename from test/resources/TestPkg/src/TestPkg.jl rename to test/resource/TestPkg/src/TestPkg.jl diff --git a/test/resources/aws_jl_test.yaml b/test/resource/aws_jl_test.yaml similarity index 100% rename from test/resources/aws_jl_test.yaml rename to test/resource/aws_jl_test.yaml diff --git a/test/resources/operations.json b/test/resource/operations.json similarity index 100% rename from test/resources/operations.json rename to test/resource/operations.json diff --git a/test/resources/services.json b/test/resource/services.json similarity index 100% rename from test/resources/services.json rename to test/resource/services.json diff --git a/test/resources/setup.jl b/test/resource/setup.jl similarity index 100% rename from test/resources/setup.jl rename to test/resource/setup.jl diff --git a/test/resources/shapes.json b/test/resource/shapes.json similarity index 100% rename from test/resources/shapes.json rename to test/resource/shapes.json diff --git a/test/resources/totp.jl b/test/resource/totp.jl similarity index 100% rename from test/resources/totp.jl rename to test/resource/totp.jl diff --git a/test/runtests.jl b/test/runtests.jl index bd2bee277..a9f8e0617 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,11 +39,7 @@ using StableRNGs Mocking.activate() include("patch.jl") -include("resources/totp.jl") - -const TEST_MINIO = begin - all(k -> haskey(ENV, k), ("MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MINIO_REGION_NAME")) -end +include("resource/totp.jl") function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) @@ -52,7 +48,10 @@ end testset_role(role_name) = "AWS.jl-$role_name" const RUN_UNIT_TESTS = get(ENV, "RUN_UNIT_TESTS", "true") == "true" -const RUN_INTEGRATION_TESTS = get(ENV, "RUN_INTEGRATION_TESTS", "true") == "true" +const RUN_INTEGRATION_TESTS = get(ENV, "RUN_INTEGRATION_TESTS", "false") == "true" +const RUN_MINIO_INTEGRATION_TESTS = begin + all(k -> haskey(ENV, k), ("MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MINIO_REGION_NAME")) +end # Avoid the situation where we all tests are silently skipped. Most likely due to the wrong # environmental variables being used. @@ -60,40 +59,53 @@ if !RUN_UNIT_TESTS && !RUN_INTEGRATION_TESTS error("All tests have been disabled") end -const AWS_CONFIG = Ref{AbstractAWSConfig}() - @testset "AWS.jl" begin + # Unit tests do not requires access to an AWS account @testset "Unit Tests" begin if RUN_UNIT_TESTS - include("unit/AWS.jl") - include("unit/AWSCredentials.jl") + # Force unit tests to run without a valid AWS configuration being present + withenv( + [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., + "AWS_CONFIG_FILE" => "/dev/null", + "AWS_SHARED_CREDENTIALS_FILE" => "/dev/null", + ) do + include("unit/AWS.jl") + include("unit/AWSExceptions.jl") + include("unit/AWSMetadataUtilities.jl") + include("unit/test_pkg.jl") + include("unit/utilities.jl") + include("unit/AWSConfig.jl") + include("unit/IMDS.jl") + include("unit/AWSCredentials.jl") + include("unit/issues.jl") + end else @warn "Skipping unit tests" end end - # TODO: Some of these tests are actually unit tests and need to be refactored + # Integration tests require access to an AWS account @testset "Integration Tests" begin if RUN_INTEGRATION_TESTS - AWS_CONFIG[] = AWSConfig() - - include("AWSExceptions.jl") - include("AWSMetadataUtilities.jl") - include("test_pkg.jl") - include("utilities.jl") - include("AWSConfig.jl") - + config = AWSConfig() backends = [AWS.HTTPBackend, AWS.DownloadsBackend] + @testset "Backend: $(nameof(backend))" for backend in backends AWS.DEFAULT_BACKEND[] = backend() - include("AWS.jl") - include("IMDS.jl") - include("AWSCredentials.jl") - include("role.jl") - include("issues.jl") - if TEST_MINIO - include("minio.jl") + # Reset the default AWS configuration as the unit tests may have messed with + # the global default. + global_aws_config(config) + + include("integration/AWS.jl") + include("integration/AWSCredentials.jl") + include("integration/role.jl") + include("integration/issues.jl") + + if RUN_MINIO_INTEGRATION_TESTS + include("integration/minio.jl") + else + @warn "Skipping MinIO integration tests" end end else diff --git a/test/unit/AWS.jl b/test/unit/AWS.jl index 46d5c32e0..49c6e9c17 100644 --- a/test/unit/AWS.jl +++ b/test/unit/AWS.jl @@ -111,3 +111,656 @@ @test !(Symbol("STS.X") in names(@__MODULE__; all=true)) end end + +@testset "global config, kwargs" begin + # Fake AWS credentials as shown in the AWS documentation: + # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html + access_key_id = "AKIAIOSFODNN7EXAMPLE" + secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + creds = AWS.AWSCredentials(access_key_id, secret_access_key) + region = "us-example-1" + + old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing + try + AWS.global_aws_config(; creds, region) + + result = AWS.global_aws_config() + @test result.region == region + @test result.credentials == creds + finally + if !isnothing(old_config) + AWS.aws_config[] = old_config + end + end +end + +@testset "set global aws config" begin + config = AWSConfig(; creds=nothing, region="us-example-2") + + old_config = isassigned(AWS.aws_config) ? AWS.aws_config[] : nothing + try + AWS.global_aws_config(config) + + result = AWS.global_aws_config() + @test result.region == config.region + @test result.credentials == config.credentials + finally + if !isnothing(old_config) + AWS.aws_config[] = old_config + end + end +end + +@testset "set user agent" begin + old_user_agent = AWS.user_agent[] + new_user_agent = "new user agent" + + try + @test AWS.user_agent[] == "AWS.jl/$(pkgversion(AWS))" + set_user_agent(new_user_agent) + @test AWS.user_agent[] == new_user_agent + finally + set_user_agent(old_user_agent) + end +end + +@testset "sign" begin + access_key = "access-key" + secret_key = "ssh... it is a secret" + + creds = AWSCredentials(access_key, secret_key) + aws = AWSConfig(; creds, region="us-east-1") + + time = DateTime(2020) + date = Dates.format(time, dateformat"yyyymmdd") + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + headers=LittleDict( + "Host" => "s3.us-east-1.amazonaws.com", "User-Agent" => "AWS.jl/1.0.0" + ), + resource="/test-resource", + url="https://s3.us-east-1.amazonaws.com/test-resource", + ) + + @testset "sign v2" begin + result = AWS.sign_aws2!(aws, request, time) + @test result === request + content = result.content + content_type = result.headers["Content-Type"] + + expected_access_key = "AWSAccessKeyId=$access_key" + expected_expires = "Expires=2020-01-01T00%3A02%3A00Z" + expected_signature_method = "SignatureMethod=HmacSHA256" + expected_signature_version = "SignatureVersion=2" + expected_signature = "Signature=O0MLzMKpEcfVZeHy0tyxVAuZF%2BvrvbgIGgqbWtJLTQ0%3D" + + expected_content = join( + [ + expected_access_key, + expected_expires, + expected_signature_method, + expected_signature_version, + expected_signature, + ], + '&', + ) + + @test content == expected_content + end + + @testset "sign v4" begin + @testset "basic" begin + expected_x_amz_content_sha256 = bytes2hex(digest(MD_SHA256, request.content)) + expected_content_md5 = base64encode(digest(MD_MD5, request.content)) + expected_x_amz_date = Dates.format(time, dateformat"yyyymmdd\THHMMSS\Z") + + result = AWS.sign_aws4!(aws, request, time) + @test result === request + headers = result.headers + + @test headers["x-amz-content-sha256"] == expected_x_amz_content_sha256 + @test headers["Content-MD5"] == expected_content_md5 + @test headers["x-amz-date"] == expected_x_amz_date + + authorization_header = split(headers["Authorization"], ' ') + @test length(authorization_header) == 4 + @test authorization_header[1] == "AWS4-HMAC-SHA256" + @test authorization_header[2] == + "Credential=$access_key/$date/us-east-1/$(request.service)/aws4_request," + @test authorization_header[3] == + "SignedHeaders=content-md5;content-type;host;user-agent;x-amz-content-sha256;x-amz-date," + @test authorization_header[4] == + "Signature=0f292eaf0b66cf353bafcb1b9b6d90ee27064236a60f17f6fc5bd7d40173a0be" + end + + @testset "duplicate query params" begin + request_query = deepcopy(request) + request_query.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions" + + request_dup = deepcopy(request) + request_dup.url = "https://s3.us-east-1.amazonaws.com/test-resource?versions&versions=" + + AWS.sign_aws4!(aws, request_query, time) + AWS.sign_aws4!(aws, request_dup, time) + + authorization_header_query = split(request_query.headers["Authorization"], ' ') + authorization_header_dup = split(request_dup.headers["Authorization"], ' ') + + @test length(authorization_header_query) == 4 + @test length(authorization_header_dup) == 4 + + # Signatures should differ + @test authorization_header_query[4] != authorization_header_dup[4] + end + end +end + +@testset "submit_request" begin + aws = AWSConfig(; creds=nothing, region="us-east-1") + + function _expected_xml(body::AbstractString, dict_type::Type) + parsed = parse_xml(body) + return xml_dict(XMLDict.root(parsed.x), dict_type) + end + + @testset "301 redirect" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="HEAD", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + apply(Patches._aws_http_request_patch(Patches._response(; status=301))) do + @test_throws AWSException AWS.submit_request(aws, request) + end + end + + @testset "HEAD response" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="HEAD", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + response = apply(Patches._aws_http_request_patch()) do + AWS.submit_request(aws, request) + end + + # Access to response headers + @test response.response.headers == Patches.headers + @test response.response.headers isa Vector + + # Access to streaming content + @test response.io isa IO + + # Content as a string + @test String(take!(response.io)) == Patches.body + + # Backwards compatibility with those expecting an `HTTP.Response` + @test response.headers == Patches.headers + @test response.headers isa Vector + @test String(response.body) == Patches.body + end + + @testset "GET response" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + response = apply(Patches._aws_http_request_patch()) do + AWS.submit_request(aws, request) + end + + # Access to response headers + @test response.response.headers == Patches.headers + @test response.response.headers isa Vector + + # Access to streaming content + @test response.io isa IO + + # Content as a string + @test String(take!(response.io)) == Patches.body + + # Backwards compatibility with those expecting an `HTTP.Response` + @test response.headers == Patches.headers + @test response.headers isa Vector + @test String(response.body) == Patches.body + end + + @testset "Default throttling" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + + @testset "Custom throttling" begin + aws = AWSConfig(; creds=nothing, region="us-east-1", max_attempts=1) + @test AWS.max_attempts(aws) == 1 + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + + @testset "Not authorized" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + message = "User is not authorized to perform: action on resource with an explicit deny" + + # Simulate the HTTP.request behaviour with a HTTP 400 response + exception = apply(Patches.gen_http_options_400_patches(message)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + + # If handled incorrectly using a `response_stream` may result in the body data being + # lost. Mainly, this is a problem when using a temporary I/O stream instead of + # writing directly to the `response_stream`. + @test exception.message == message + @test exception.streamed_body !== nothing + end + + @testset "Not authorized with BufferStream response_stream" begin + buf = Base.BufferStream() + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + response_stream=buf, + use_response_type=true, + ) + message = "User is not authorized to perform: action on resource with an explicit deny" + # Simulate the HTTP.request behaviour with a HTTP 400 response + exception = apply(Patches.gen_http_options_400_patches(message)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + @test exception isa AWSException + # If handled incorrectly using a `response_stream` may result in the body data being + # lost. Mainly, this is a problem when using a temporary I/O stream instead of + # writing directly to the `response_stream`. + @test exception.message == message + @test exception.streamed_body !== nothing + end + + @testset "read MIME-type" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + @testset "invalid content type" begin + headers = Pair["Content-Type" => ""] + body = "" + expected_body_type = Vector{UInt8} + expected_body = b"" + + r = Patches._response(; headers=headers, body=body) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "text/xml" begin + headers = Pair["Content-Type" => "text/xml"] + expected_body_type = LittleDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "application/xml" begin + headers = Pair["Content-Type" => "application/xml"] + expected_body_type = LittleDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "application/json" begin + headers = ["Content-Type" => "application/json"] + body = JSON.json( + Dict{String,Any}( + "Marker" => nothing, + "VaultList" => Any[Dict{String,Any}( + "VaultName" => "test", + "SizeInBytes" => 0, + "NumberOfArchives" => 0, + "CreationDate" => "2020-06-22T03:14:41.754Z", + "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", + "LastInventoryDate" => nothing, + )], + ), + ) + + expected_body_type = LittleDict{String,Any} + expected_body = JSON.parse(body; dicttype=expected_body_type) + + r = Patches._response(; body=body, headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + end + + @testset "text/html" begin + headers = ["Content-Type" => "text/html"] + expected_body = Patches.body + + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa String + @test content == expected_body + end + + # Note: `S3.create_multipart_upload` is an example of a response type that doesn't + # specify a Content-Type. + @testset "missing content type" begin + headers = Pair[] + body = """ + + text + """ + expected_body_type = AbstractDict + expected_body = Dict{String,Any}("body" => "text") + + r = Patches._response(; headers=headers, body=body) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end + + content = parse(response) + @test content isa expected_body_type + @test content == expected_body + + content = parse(response, MIME"text/plain"()) + @test content isa String + @test content == body + end + end +end + +struct TestBackend <: AWS.AbstractBackend + param::Int +end + +function AWS._http_request(backend::TestBackend, ::AWS.Request, ::IO) + return backend.param +end + +@testset "HTTPBackend" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + backend=AWS.HTTPBackend(), + ) + io = IOBuffer() + + apply(Patches._http_options_patches) do + # No default options + @test isempty(AWS._http_request(request.backend, request, io)) + + # We can pass HTTP options via the backend + custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5)) + @test custom_backend isa AWS.AbstractBackend + @test AWS._http_request(custom_backend, request, io) == Dict(:connection_limit => 5) + + # We can pass options per-request + request.http_options = Dict(:pipeline_limit => 20) + @test AWS._http_request(request.backend, request, io) == Dict(:pipeline_limit => 20) + @test AWS._http_request(custom_backend, request, io) == + Dict(:pipeline_limit => 20, :connection_limit => 5) + + # per-request options override backend options: + custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5)) + @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20) + end + + request.backend = TestBackend(2) + @test AWS._http_request(request.backend, request, io) == 2 + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + backend=TestBackend(4), + ) + @test AWS._http_request(request.backend, request, io) == 4 + + # Let's test setting the default backend + prev_backend = AWS.DEFAULT_BACKEND[] + try + AWS.DEFAULT_BACKEND[] = TestBackend(3) + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + ) + @test AWS._http_request(request.backend, request, io) == 3 + finally + AWS.DEFAULT_BACKEND[] = prev_backend + end +end + +@testset "_generate_rest_resource" begin + request_uri = "/{Bucket}/{Key+}" + args = Dict{String,Any}("Bucket" => "aws.jl-test", "Key" => "Test-Key") + + expected = "/$(args["Bucket"])/$(args["Key"])" + result = AWS._generate_rest_resource(request_uri, args) + @test result == expected +end + +@testset "generate_service_url" begin + region = "us-east-2" + resource = "/aws.jl-test---timestamp" + config = AWSConfig(; creds=nothing, region) + + request = Request(; + service="service", + api_version="api_version", + request_method="GET", + resource=resource, + ) + + @testset "regionless endpoints" for regionless_endpoint in ("iam", "route53") + endpoint = "sdb" + request.service = regionless_endpoint + expected_result = "https://$regionless_endpoint.amazonaws.com$resource" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end + + @testset "region service" begin + endpoint = "sdb" + request.service = endpoint + expected_result = "https://$endpoint.$region.amazonaws.com$resource" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end + + @testset "sdb -- us-east-1 region exception" begin + endpoint = "sdb" + request.service = endpoint + expected_result = "https://$endpoint.amazonaws.com$resource" + config.region = "us-east-1" + result = AWS.generate_service_url(config, request.service, request.resource) + + @test result == expected_result + end +end + +@testset "_flatten_query" begin + high_level_value = "high_level_value" + entry_1 = LittleDict( + "low_level_key_1" => "low_level_value_1", "low_level_key_2" => "low_level_value_2" + ) + entry_2 = LittleDict( + "low_level_key_3" => "low_level_value_3", "low_level_key_4" => "low_level_value_4" + ) + + args = LittleDict( + "high_level_key" => high_level_value, "high_level_array" => [entry_1, entry_2] + ) + + @testset "non-special case suffix" begin + service = "sts" + result = AWS._flatten_query(service, args) + + expected = Pair{String,String}[ + "high_level_key" => "high_level_value", + "high_level_array.member.1.low_level_key_1" => "low_level_value_1", + "high_level_array.member.1.low_level_key_2" => "low_level_value_2", + "high_level_array.member.2.low_level_key_3" => "low_level_value_3", + "high_level_array.member.2.low_level_key_4" => "low_level_value_4", + ] + + @test result == expected + end + + @testset "sqs - special casing suffix" begin + service = "sqs" + result = AWS._flatten_query(service, args) + + expected = Pair{String,String}[ + "high_level_key" => "high_level_value", + "high_level_array.1.low_level_key_1" => "low_level_value_1", + "high_level_array.1.low_level_key_2" => "low_level_value_2", + "high_level_array.2.low_level_key_3" => "low_level_value_3", + "high_level_array.2.low_level_key_4" => "low_level_value_4", + ] + + @test result == expected + end +end + +@testset "_clean_s3_uri" begin + uri = "/test-bucket/*)=('! +@,:.txt?list-objects=v2" + expected_uri = "/test-bucket/%2A%29%3D%28%27%21%20%2B%40%2C%3A.txt?list-objects=v2" + @test AWS._clean_s3_uri(uri) == expected_uri + + # make sure that other parts of the uri aren't changed by `_clean_s3_uri` + for uri in ( + "https://julialang.org", + "http://julialang.org", + "http://julialang.org:8080", + "/onlypath", + "/path?query= +99", + "/anchor?query=yes#anchor1", + ) + @test AWS._clean_s3_uri(uri) == uri + end +end diff --git a/test/AWSConfig.jl b/test/unit/AWSConfig.jl similarity index 56% rename from test/AWSConfig.jl rename to test/unit/AWSConfig.jl index e7db09245..7bcd6d9a8 100644 --- a/test/AWSConfig.jl +++ b/test/unit/AWSConfig.jl @@ -1,13 +1,12 @@ @testset "AWSConfig" begin @testset "default profile assumes role" begin access_key_id = "assumed_access_key_id" - config_dir = joinpath(@__DIR__, "configs", "default-role") + config_dir = joinpath(@__DIR__, "..", "config", "default-role") # Avoid calling out to STS with invalid credentials patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) config = withenv( - [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), ) do @@ -22,16 +21,14 @@ @testset "default profile section names" begin allowed_default_sections = ["default", "profile default"] mktemp() do config_path, _ - withenv([k => nothing for k in filter(startswith("AWS_"), keys(ENV))]...) do - for default_section_str in allowed_default_sections - config = """ - [$default_section_str] - region = xx-yy-1 - """ - write(config_path, config) - region = aws_get_region(; profile="default", config=config_path) - @test region == "xx-yy-1" - end + for default_section_str in allowed_default_sections + config = """ + [$default_section_str] + region = xx-yy-1 + """ + write(config_path, config) + region = aws_get_region(; profile="default", config=config_path) + @test region == "xx-yy-1" end end end diff --git a/test/unit/AWSCredentials.jl b/test/unit/AWSCredentials.jl index 2baa12bf7..3a9479c8a 100644 --- a/test/unit/AWSCredentials.jl +++ b/test/unit/AWSCredentials.jl @@ -1,3 +1,28 @@ +macro test_ecode(error_codes, expr) + quote + try + $expr + @test false + catch e + if e isa AWSException + @test e.code in [$error_codes;] + else + rethrow(e) + end + end + end +end + +const EXPIRATION_FMT = dateformat"yyyy-mm-dd\THH:MM:SS\Z" + +http_header(h::Vector, k, d="") = get(Dict(h), k, d) +http_header(args...) = HTTP.header(args...) + +@testset "_role_session_name" begin + @test AWS._role_session_name("prefix-", "name", "-suffix") == "prefix-name-suffix" + @test AWS._role_session_name("a"^22, "b"^22, "c"^22) == "a"^22 * "b"^20 * "c"^22 +end + @testset "_aws_profile_config" begin using AWS: _aws_profile_config @@ -61,3 +86,1324 @@ @test config["region"] == "default-2" end end + +@testset "aws_get_profile_settings" begin + @testset "no profile" begin + @test aws_get_profile_settings("foo", Inifile()) === nothing + end +end + +@testset "_aws_get_role" begin + profile = "foobar" + ini = Inifile() + + @testset "settings early exit" begin + apply(Patches.get_profile_settings_empty_patch) do + @test AWS._aws_get_role(profile, ini) === nothing + end + end + + @testset "source_profile early exit" begin + apply(Patches.get_profile_settings_empty_patch) do + @test AWS._aws_get_role(profile, ini) === nothing + end + end + + @testset "default profile" begin + access_key_id = "assumed_access_key_id" + config_dir = joinpath(@__DIR__, "..", "config", "default-role") + + patch = Patches._assume_role_patch("AssumeRole"; access_key=access_key_id) + + cred = withenv( + "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), + "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_SECRET_ACCESS_KEY" => nothing, + ) do + ini = read(Inifile(), ENV["AWS_CONFIG_FILE"]) + apply(patch) do + AWS._aws_get_role("default", ini) + end + end + + @test cred.access_key_id == access_key_id + end + + @testset "profile with role and MFA" begin + access_key_id = "assumed_access_key_id" + config_dir = joinpath(@__DIR__, "..", "config", "role-with-mfa") + + mfa_token = "123456" + sent_token = Ref("") + server_time = DateTime(0) + patches = [ + Patches._assume_role_patch( + "AssumeRole"; + access_key=access_key_id, + expiry=duration -> server_time + duration, + token_code_ref=sent_token, + ), + Patches._getpass_patch(; secret=mfa_token), + ] + + cred = withenv( + "AWS_CONFIG_FILE" => joinpath(config_dir, "config"), + "AWS_SHARED_CREDENTIALS_FILE" => joinpath(config_dir, "credentials"), + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_SECRET_ACCESS_KEY" => nothing, + ) do + ini = read(Inifile(), ENV["AWS_CONFIG_FILE"]) + apply(patches) do + AWS._aws_get_role("role_and_mfa", ini) + end + end + + @test cred.access_key_id == access_key_id + @test cred.expiry == server_time + Second(1234) + @test sent_token[] == mfa_token + end +end + +@testset "AWSCredentials" begin + @testset "Defaults" begin + creds = AWSCredentials("access_key_id", "secret_key") + @test creds.token == "" + @test creds.user_arn == "" + @test creds.account_number == "" + @test creds.expiry == typemax(DateTime) + @test creds.renew === nothing + end + + @testset "Renewal" begin + # Credentials shouldn't throw an error if no renew function is supplied + creds = AWSCredentials("access_key_id", "secret_key"; renew=nothing) + newcreds = check_credentials(creds; force_refresh=true) + + # Creds should remain unchanged if no renew function exists + @test creds === newcreds + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.renew === nothing + + # Creds should error if the renew function returns nothing + creds = AWSCredentials("access_key_id", "secret_key"; renew=() -> nothing) + @test_throws NoCredentials check_credentials(creds; force_refresh=true) + + # Creds should remain unchanged + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + + # Creds should take on value of a returned AWSCredentials except renew function + function gen_credentials() + i = 0 + return () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i")) + end + + creds = AWSCredentials( + "access_key_id", "secret_key"; renew=gen_credentials(), expiry=now(UTC) + ) + + @test creds.renew !== nothing + renewed = creds.renew() + + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.expiry <= now(UTC) + @test AWS._will_expire(creds) + + @test renewed.access_key_id === "NEW_ID_1" + @test renewed.secret_key == "NEW_KEY_1" + @test renewed.renew === nothing + @test renewed.expiry == typemax(DateTime) + @test !AWS._will_expire(renewed) + renew = creds.renew + + # Check renewal on time out + newcreds = check_credentials(creds; force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + @test !AWS._will_expire(creds) + + # Check renewal doesn't happen if not forced or timed out + newcreds = check_credentials(creds; force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + + # Check forced renewal works + newcreds = check_credentials(creds; force_refresh=true) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_3" + @test creds.secret_key == "NEW_KEY_3" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + end + + mktempdir() do dir + config_file = joinpath(dir, "config") + creds_file = joinpath(dir, "creds") + write( + config_file, + """ + [profile test] + output = json + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + + [profile test2] + aws_access_key_id = WRONG_ACCESS_ID + aws_secret_access_key = WRONG_ACCESS_KEY + + [profile test3] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + + [profile test4] + aws_access_key_id = RIGHT_ACCESS_ID4 + aws_secret_access_key = RIGHT_ACCESS_KEY4 + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + """, + ) + + write( + creds_file, + """ + [test] + aws_access_key_id = TEST_ACCESS_ID + aws_secret_access_key = TEST_ACCESS_KEY + + [test2] + aws_access_key_id = RIGHT_ACCESS_ID2 + aws_secret_access_key = RIGHT_ACCESS_KEY2 + + [test3] + aws_access_key_id = RIGHT_ACCESS_ID3 + aws_secret_access_key = RIGHT_ACCESS_KEY3 + """, + ) + + withenv( + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + "AWS_CONFIG_FILE" => config_file, + "AWS_DEFAULT_PROFILE" => "test", + "AWS_PROFILE" => nothing, + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_REGION" => "us-east-1", + ) do + @testset "Loading" begin + # Check credentials load + config = AWSConfig() + creds = config.credentials + + @test creds isa AWSCredentials + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.renew !== nothing + + # Check credential file takes precedence over config + withenv("AWS_DEFAULT_PROFILE" => "test2") do + config = AWSConfig() + creds = config.credentials + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + end + + # Check credentials take precedence over role + withenv("AWS_DEFAULT_PROFILE" => "test3") do + config = AWSConfig() + creds = config.credentials + + @test creds.access_key_id == "RIGHT_ACCESS_ID3" + @test creds.secret_key == "RIGHT_ACCESS_KEY3" + end + + withenv("AWS_DEFAULT_PROFILE" => "test4") do + config = AWSConfig() + creds = config.credentials + + @test creds.access_key_id == "RIGHT_ACCESS_ID4" + @test creds.secret_key == "RIGHT_ACCESS_KEY4" + end + end + + @testset "Refresh" begin + withenv("AWS_DEFAULT_PROFILE" => "test") do + # Check credentials refresh on timeout + config = AWSConfig() + creds = config.credentials + creds.access_key_id = "EXPIRED_ACCESS_ID" + creds.secret_key = "EXPIRED_ACCESS_KEY" + creds.expiry = now(UTC) + + @test creds.renew !== nothing + renew = creds.renew + + @test renew() isa AWSCredentials + + creds = check_credentials(config.credentials) + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.expiry > now(UTC) + + # Check renew function remains unchanged + @test creds.renew !== nothing + @test creds.renew === renew + + # Check force_refresh + creds.access_key_id = "WRONG_ACCESS_KEY" + creds = check_credentials(creds; force_refresh=true) + @test creds.access_key_id == "TEST_ACCESS_ID" + end + end + + @testset "Profile" begin + # Check profile kwarg + withenv("AWS_DEFAULT_PROFILE" => "test") do + creds = AWSCredentials(; profile="test2") + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + config = AWSConfig(; profile="test2") + creds = config.credentials + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + # Check profile persists on renewal + creds.access_key_id = "WRONG_ACCESS_ID2" + creds.secret_key = "WRONG_ACCESS_KEY2" + creds = check_credentials(creds; force_refresh=true) + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + end + end + + @testset "Assume Role" begin + # Check we try to assume a role + withenv("AWS_DEFAULT_PROFILE" => "test:dev") do + @test_ecode("InvalidClientTokenId", AWSConfig()) + end + + # Check we try to assume a role + withenv("AWS_DEFAULT_PROFILE" => "test:sub-dev") do + oldout = stdout + r, w = redirect_stdout() + + @test_ecode("InvalidClientTokenId", AWSConfig()) + redirect_stdout(oldout) + close(w) + output = String(read(r)) + occursin("Assuming \"test:dev\"", output) + occursin("Assuming \"test\"", output) + close(r) + end + end + end + end + + # Verify that the search order for credentials mirrors the behavior of the AWS CLI + # (version 2.11.13). Whenever support is added for new credential types new tests should + # be added to this test set. To determine the credential preference order used by AWS + # CLI it is recommended you use a set of valid credentials and a set of invalid + # credentials to determine the precedence. + # + # Documentation on credential precedence: + # - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence + # - https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html + # - https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html + @testset "Credential Precedence" begin + mktempdir() do dir + config_file = joinpath(dir, "config") + creds_file = joinpath(dir, "creds") + + basic_creds_content = """ + [profile1] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + + [profile2] + aws_access_key_id = AKI2 + aws_secret_access_key = SAK2 + """ + + ec2_json = Dict( + "AccessKeyId" => "AKI_EC2", + "SecretAccessKey" => "SAK_EC2", + "Token" => "TOK_EC2", + "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), + ) + + function ec2_metadata(url::AbstractString) + name = "local-credentials" + metadata_uri = "http://169.254.169.254/latest/meta-data" + if url == "$metadata_uri/iam/info" + return HTTP.Response(200, JSON.json("InstanceProfileArn" => "ARN0")) + elseif url == "$metadata_uri/iam/security-credentials/" + return HTTP.Response(200, name) + elseif url == "$metadata_uri/iam/security-credentials/$name" + return HTTP.Response(200, JSON.json(ec2_json)) + else + return HTTP.Response(404) + end + end + + ecs_json = Dict( + "AccessKeyId" => "AKI_ECS", + "SecretAccessKey" => "SAK_ECS", + "Token" => "TOK_ECS", + "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), + ) + + function ecs_metadata(url::AbstractString) + if startswith(url, "http://169.254.170.2/") + return HTTP.Response(200, JSON.json(ecs_json)) + else + return HTTP.Response(404) + end + end + + function ecs_metadata_localhost(url::AbstractString) + if startswith(url, "http://localhost:8080") + return HTTP.Response(200, JSON.json(ecs_json)) + else + return HTTP.Response(404) + end + end + + function http_request_patcher(funcs) + @patch function HTTP.request(method, url, args...; kwargs...) + local r + for f in funcs + r = f(string(url)) + r.status != 404 && break + end + return r + end + end + + withenv( + [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + "AWS_CONFIG_FILE" => config_file, + ) do + @testset "explicit profile preferred" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + + withenv("AWS_PROFILE" => "profile1") do + creds = AWSCredentials(; profile="profile2") + @test creds.access_key_id == "AKI2" + end + + withenv( + "AWS_ACCESS_KEY_ID" => "AKI0", + "AWS_SECRET_ACCESS_KEY" => "SAK0", + # format trick: using this comment to force use of multiple lines + ) do + creds = AWSCredentials(; profile="profile2") + @test creds.access_key_id == "AKI2" + end + end + + @testset "AWS_ACCESS_KEY_ID preferred over AWS_PROFILE" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + + withenv( + "AWS_PROFILE" => "profile1", + "AWS_ACCESS_KEY_ID" => "AKI0", + "AWS_SECRET_ACCESS_KEY" => "SAK0", + ) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI0" + end + end + + # The AWS CLI used to use `AWS_DEFAULT_PROFILE` to set the AWS profile via the + # command line but this was deprecated in favor of `AWS_PROFILE`. We'll probably + # keeps support for this as long as AWS CLI continues to support it. + # https://github.com/aws/aws-cli/issues/2597 + @testset "AWS_PROFILE preferred over AWS_DEFAULT_PROFILE" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + + withenv( + "AWS_DEFAULT_PROFILE" => "profile1", + "AWS_PROFILE" => "profile2", + # format trick: using this comment to force use of multiple lines + ) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI2" + end + end + + @testset "Web identity preferred over SSO" begin + write( + config_file, + """ + [default] + sso_start_url = https://my-sso-portal.awsapps.com/start + sso_role_name = role1 + """, + ) + isfile(creds_file) && rm(creds_file) + + web_identity_file = joinpath(dir, "web_identity") + write(web_identity_file, "webid") + + patches = [ + Patches._assume_role_patch( + "AssumeRoleWithWebIdentity"; + access_key="AKI_WEB", + secret_key="SAK_WEB", + session_token="TOK_WEB", + ), + Patches.sso_service_patches("AKI_SSO", "SAK_SSO"), + Patches._imds_region_patch(nothing), + ] + + withenv( + "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, + "AWS_ROLE_ARN" => "webid", + ) do + apply(patches) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_WEB" + end + end + end + + # TODO: Additional, precedence tests should be added for IAM Identity Center + # once support has been introduced. + @testset "IAM Identity Center preferred over legacy SSO" begin + write( + config_file, + """ + [sso-session my-sso] + sso_region = us-east-1 + sso_start_url = https://my-sso-portal.awsapps.com/start + + [default] + sso_session = my-sso + sso_start_url = https://my-legacy-sso-portal.awsapps.com/start + sso_role_name = role1 + """, + ) + isfile(creds_file) && rm(creds_file) + + apply(Patches.sso_service_patches("AKI_SSO", "SAK_SSO")) do + @test_throws ErrorException AWSCredentials() + end + end + + @testset "SSO preferred over credentials file" begin + write( + config_file, + """ + [profile profile1] + sso_start_url = https://my-sso-portal.awsapps.com/start + sso_role_name = role1 + """, + ) + write(creds_file, basic_creds_content) + + apply(Patches.sso_service_patches("AKI_SSO", "SAK_SSO")) do + creds = AWSCredentials(; profile="profile1") + @test creds.access_key_id == "AKI_SSO" + end + end + + @testset "Credential file over credential_process" begin + json = Dict( + "Version" => 1, + "AccessKeyId" => "AKI0", + "SecretAccessKey" => "SAK0", + # format trick: using this comment to force use of multiple lines + ) + write( + config_file, + """ + [profile profile1] + credential_process = echo '$(JSON.json(json))' + """, + ) + write(creds_file, basic_creds_content) + + creds = AWSCredentials(; profile="profile1") + @test creds.access_key_id == "AKI1" + end + + @testset "credential_process over config credentials" begin + json = Dict( + "Version" => 1, + "AccessKeyId" => "AKI0", + "SecretAccessKey" => "SAK0", + # format trick: using this comment to force use of multiple lines + ) + write( + config_file, + """ + [profile profile1] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + credential_process = echo '$(JSON.json(json))' + """, + ) + isfile(creds_file) && rm(creds_file) + + creds = AWSCredentials(; profile="profile1") + @test creds.access_key_id == "AKI0" + end + + @testset "default config credentials over ECS container credentials ENV variables" begin + write( + config_file, + """ + [default] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + """, + ) + isfile(creds_file) && rm(creds_file) + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do + apply(http_request_patcher([ecs_metadata])) do + @test isnothing(AWS._aws_get_profile(; default=nothing)) + + creds = AWSCredentials() + @test creds.access_key_id == "AKI1" + end + end + + withenv( + "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost:8080" + ) do + apply(http_request_patcher([ecs_metadata_localhost])) do + @test isnothing(AWS._aws_get_profile(; default=nothing)) + + creds = AWSCredentials() + @test creds.access_key_id == "AKI1" + end + end + end + + @testset "default config credentials over EC2 instance credentials" begin + write( + config_file, + """ + [default] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + """, + ) + isfile(creds_file) && rm(creds_file) + + apply(http_request_patcher([ec2_metadata])) do + @test isnothing(AWS._aws_get_profile(; default=nothing)) + + creds = AWSCredentials() + @test creds.access_key_id == "AKI1" + end + end + + @testset "ECS container credentials ENV variables over EC2 instance credentials" begin + isfile(config_file) && rm(config_file) + isfile(creds_file) && rm(creds_file) + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do + apply(http_request_patcher([ec2_metadata, ecs_metadata])) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_ECS" + end + end + + withenv( + "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost:8080" + ) do + p = http_request_patcher([ec2_metadata, ecs_metadata_localhost]) + apply(p) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_ECS" + end + end + end + + # Note: It appears that the ECS container credentials are only used when + # a `AWS_CONTAINER_*` environmental variable is set. However, this test + # ensures that if we do add implicit support that the documented precedence + # order is not violated. + @testset "EC2 instance credentials over ECS container credentials" begin + isfile(config_file) && rm(config_file) + isfile(creds_file) && rm(creds_file) + + apply(http_request_patcher([ec2_metadata, ecs_metadata])) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_EC2" + end + end + end + end + end +end + +@testset "Retrieving AWS Credentials" begin + test_values = Dict{String,Any}( + "Default-Profile" => "default", + "Test-Profile" => "test", + "Test-Config-Profile" => "test", + "AccessKeyId" => "Default-Key", + "SecretAccessKey" => "Default-Secret", + "Test-AccessKeyId" => "Test-Key", + "Test-SecretAccessKey" => "Test-Secret", + "Token" => "Test-Token", + "InstanceProfileArn" => "Test-Arn", + "RoleArn" => "Test-Arn", + "Expiration" => now(UTC), + "Security-Credentials" => "Test-Security-Credentials", + "Test-SSO-Profile" => "sso-test", + "Test-SSO-start-url" => "https://test-sso.com/start", + "Test-SSO-Role" => "SSORoleName", + ) + + @testset "~/.aws/config - Default Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """, + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + default_profile = dot_aws_config(test_values["Default-Profile"]) + + @test default_profile.access_key_id == test_values["AccessKeyId"] + @test default_profile.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/config - Specified Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [profile $(test_values["Test-Config-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """, + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + specified_result = dot_aws_config(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "~/.aws/config - Specified SSO Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [profile $(test_values["Test-SSO-Profile"])] + sso_start_url=$(test_values["Test-SSO-start-url"]) + sso_role_name=$(test_values["Test-SSO-Role"]) + """, + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + apply( + Patches.sso_service_patches( + test_values["AccessKeyId"], test_values["SecretAccessKey"] + ), + ) do + specified_result = sso_credentials(test_values["Test-SSO-Profile"]) + + @test specified_result.access_key_id == test_values["AccessKeyId"] + @test specified_result.secret_key == test_values["SecretAccessKey"] + end + end + end + end + + @testset "~/.aws/config - Credential Process" begin + mktempdir() do dir + config_file = joinpath(dir, "config") + credential_process_file = joinpath(dir, "cred_process") + open(credential_process_file, "w") do io + println(io, "#!/bin/sh") + println(io, "cat < 1, + "AccessKeyId" => test_values["Test-AccessKeyId"], + "SecretAccessKey" => test_values["Test-SecretAccessKey"], + ) + JSON.print(io, json) + println(io, "\nEOF") + end + chmod(credential_process_file, 0o700) + + withenv("AWS_CONFIG_FILE" => config_file) do + open(config_file, "w") do io + write( + io, + """ + [profile $(test_values["Test-Config-Profile"])] + credential_process = $(abspath(credential_process_file)) + """, + ) + end + + result = dot_aws_config(test_values["Test-Config-Profile"]) + + @test result.access_key_id == test_values["Test-AccessKeyId"] + @test result.secret_key == test_values["Test-SecretAccessKey"] + @test isempty(result.token) + @test result.expiry == typemax(DateTime) + end + end + end + + @testset "~/.aws/creds - Default Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """, + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + specified_result = dot_aws_credentials(test_values["Default-Profile"]) + + @test specified_result.access_key_id == test_values["AccessKeyId"] + @test specified_result.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/creds - Specified Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Test-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """, + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + specified_result = dot_aws_credentials(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "Environment Variables" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"], + ) do + aws_creds = env_var_credentials() + @test aws_creds.access_key_id == test_values["AccessKeyId"] + @test aws_creds.secret_key == test_values["SecretAccessKey"] + end + end + + @testset "Instance - EC2" begin + role_name = "foobar" + role_arn = "arn:aws:sts::1234:assumed-role/$role_name" + access_key = "access-key-$(randstring(6))" + secret_key = "secret-key-$(randstring(6))" + session_token = "session-token-$(randstring(6))" + session_name = "$role_name-session" + + assume_role_patch = Patches._assume_role_patch( + "AssumeRole"; + access_key=access_key, + secret_key=secret_key, + session_token=session_token, + role_arn=role_arn, + ) + ec2_metadata_patch = @patch function HTTP.request(method, url, args...; kwargs...) + url = string(url) + security_credentials = test_values["Security-Credentials"] + + metadata_uri = "http://169.254.169.254/latest/meta-data" + if url == "$metadata_uri/iam/info" + json = JSON.json("InstanceProfileArn" => test_values["InstanceProfileArn"]) + return HTTP.Response(200, json) + elseif url == "$metadata_uri/iam/security-credentials/" + return HTTP.Response(200, security_credentials) + elseif url == "$metadata_uri/iam/security-credentials/$security_credentials" + return HTTP.Response(200, JSON.json(test_values)) + else + return HTTP.Response(404) + end + end + + apply([assume_role_patch, ec2_metadata_patch]) do + result = ec2_instance_credentials("default") + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["InstanceProfileArn"] + @test result.expiry == test_values["Expiration"] + @test result.renew !== nothing + + result = mktemp() do config_file, config_io + write( + config_io, + """ + [profile $role_name] + credential_source = Ec2InstanceMetadata + role_arn = $role_arn + """, + ) + close(config_io) + + withenv( + "AWS_CONFIG_FILE" => config_file, + "AWS_ROLE_SESSION_NAME" => session_name, + ) do + ec2_instance_credentials(role_name) + end + end + + @test result.access_key_id == access_key + @test result.secret_key == secret_key + @test result.token == session_token + @test result.user_arn == "$(role_arn)/$(session_name)" + @test result.renew !== nothing + end + end + + @testset "Instance - ECS" begin + expiration = floor(now(UTC), Second) + rel_uri_json = Dict( + "AccessKeyId" => "AKI_REL_ECS", + "SecretAccessKey" => "SAK_REL_ECS", + "Token" => "TOK_REL_ECS", + "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "RoleArn" => "ROLE_REL_ECS", + ) + + rel_uri_patch = @patch function HTTP.request(::String, url, headers=[]; kwargs...) + url = string(url) + + @test url == "http://169.254.170.2/get-credentials" + @test isempty(headers) + + if url == "http://169.254.170.2/get-credentials" + return HTTP.Response(200, JSON.json(rel_uri_json)) + else + return HTTP.Response(404) + end + end + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-credentials") do + apply(rel_uri_patch) do + result = ecs_instance_credentials() + @test result.access_key_id == rel_uri_json["AccessKeyId"] + @test result.secret_key == rel_uri_json["SecretAccessKey"] + @test result.token == rel_uri_json["Token"] + @test result.user_arn == rel_uri_json["RoleArn"] + @test result.expiry == expiration + @test result.renew == ecs_instance_credentials + end + end + + # When the environmental variable isn't set then the ECS credential provider is + # unavailable. + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing) do + @test ecs_instance_credentials() === nothing + end + + # Specifying the environmental variable results in us attempting to connect to the + # ECS credential provider. + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/invalid") do + # Internally throws a `ConnectError` exception + @test ecs_instance_credentials() === nothing + end + + full_uri_json = Dict( + "AccessKeyId" => "AKI_FULL_ECS", + "SecretAccessKey" => "SAK_FULL_ECS", + "Token" => "TOK_FULL_ECS", + "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "RoleArn" => "ROLE_FULL_ECS", + ) + + full_uri_patch = @patch function HTTP.request(::String, url, headers=[]; kwargs...) + url = string(url) + authorization = http_header(headers, "Authorization") + + @test url == "http://localhost/get-credentials" + @test authorization == "Basic abcd" + + if url == "http://localhost/get-credentials" && authorization == "Basic abcd" + return HTTP.Response(200, JSON.json(full_uri_json)) + else + return HTTP.Response(403) + end + end + + withenv( + "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost/get-credentials", + "AWS_CONTAINER_AUTHORIZATION_TOKEN" => "Basic abcd", + ) do + apply(full_uri_patch) do + result = ecs_instance_credentials() + @test result.access_key_id == full_uri_json["AccessKeyId"] + @test result.secret_key == full_uri_json["SecretAccessKey"] + @test result.token == full_uri_json["Token"] + @test result.user_arn == full_uri_json["RoleArn"] + @test result.expiry == expiration + @test result.renew == ecs_instance_credentials + end + end + + # `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` should be preferred over + # `AWS_CONTAINER_CREDENTIALS_FULL_URI`. + withenv( + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-credentials", + "AWS_CONTAINER_CREDENTIALS_FULL_URI" => "http://localhost/get-credentials", + ) do + apply(rel_uri_patch) do + result = ecs_instance_credentials() + @test result.access_key_id == rel_uri_json["AccessKeyId"] + end + end + end + + @testset "Web Identity File" begin + @test credentials_from_webtoken() == nothing + + mktempdir() do dir + web_identity_file = joinpath(dir, "web_identity") + write(web_identity_file, "foobar") + session_name = "foobar-session" + + access_key = "access-key-$(randstring(6))" + secret_key = "secret-key-$(randstring(6))" + session_token = "session-token-$(randstring(6))" + role_arn = "arn:aws:sts::1234:assumed-role/foobar" + + patch = Patches._assume_role_patch( + "AssumeRoleWithWebIdentity"; + access_key=access_key, + secret_key=secret_key, + session_token=session_token, + role_arn=role_arn, + expiry=duration -> now(UTC), # expire immediately to check renewal + ) + + withenv( + "AWS_ROLE_ARN" => "foobar", + "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, + "AWS_ROLE_SESSION_NAME" => session_name, + ) do + apply(patch) do + result = credentials_from_webtoken() + + @test result.access_key_id == access_key + @test result.secret_key == secret_key + @test result.token == session_token + @test result.user_arn == "$(role_arn)/$(session_name)" + @test result.renew == credentials_from_webtoken + expiry = result.expiry + sleep(0.1) + result = check_credentials(result) + + @test result.access_key_id == access_key + @test result.secret_key == secret_key + @test result.token == session_token + @test result.user_arn == "$(role_arn)/$(session_name)" + @test result.renew == credentials_from_webtoken + @test expiry != result.expiry + end + end + + session_name = "AWS.jl-role-foobar-20210101T000000Z" + patches = [ + patch + @patch Dates.now(::Type{UTC}) = DateTime(2021) + ] + + withenv( + "AWS_ROLE_ARN" => "foobar", + "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, + "AWS_ROLE_SESSION_NAME" => nothing, + ) do + apply(patches) do + result = credentials_from_webtoken() + @test result.user_arn == "$(role_arn)/$(session_name)" + end + end + end + end + + @testset "Credential Process" begin + gen_process(json) = Cmd(["echo", JSON.json(json)]) + + long_term_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + # format trick: using this comment to force use of multiple lines + ) + creds = external_process_credentials(gen_process(long_term_resp)) + @test creds.access_key_id == long_term_resp["AccessKeyId"] + @test creds.secret_key == long_term_resp["SecretAccessKey"] + @test isempty(creds.token) + @test creds.expiry == typemax(DateTime) + + expiration = floor(now(UTC), Second) + temporary_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "SessionToken" => "session-token", + "Expiration" => Dates.format(expiration, EXPIRATION_FMT), + ) + creds = external_process_credentials(gen_process(temporary_resp)) + @test creds.access_key_id == temporary_resp["AccessKeyId"] + @test creds.secret_key == temporary_resp["SecretAccessKey"] + @test creds.token == temporary_resp["SessionToken"] + @test creds.expiry == expiration + + unhandled_version_resp = Dict("Version" => 2) + json = sprint(JSON.print, unhandled_version_resp, 2) + ex = ErrorException("Credential process returned unhandled version 2:\n$json") + @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) + + missing_token_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "Expiration" => Dates.format(expiration, EXPIRATION_FMT), + ) + ex = KeyError("SessionToken") + @test_throws ex external_process_credentials(gen_process(missing_token_resp)) + + missing_expiration_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "SessionToken" => "session-token", + ) + ex = KeyError("Expiration") + @test_throws ex external_process_credentials(gen_process(missing_expiration_resp)) + end + + @testset "Credentials Not Found" begin + patches = [ + @patch function HTTP.request(method::String, url, args...; kwargs...) + throw(HTTP.Exceptions.ConnectError(string(url), "host is unreachable")) + end + Patches._cred_file_patch + Patches._config_file_patch + ] + + withenv( + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing, + ) do + apply(patches) do + @test_throws NoCredentials AWSConfig() + end + end + end + + @testset "Helper functions" begin + @testset "Check Credentials - EnvVars" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"], + ) do + testAWSCredentials = AWSCredentials( + test_values["AccessKeyId"], + test_values["SecretAccessKey"]; + expiry=Dates.now(UTC) - Minute(10), + renew=env_var_credentials, + ) + + result = check_credentials(testAWSCredentials; force_refresh=true) + @test result.access_key_id == testAWSCredentials.access_key_id + @test result.secret_key == testAWSCredentials.secret_key + @test result.expiry == typemax(DateTime) + @test result.renew == testAWSCredentials.renew + end + end + end +end + +@testset "aws_get_region" begin + mktempdir() do dir + config_str = """ + [default] + region = us-west-2 + + [profile test] + region = ap-northeast-1 + """ + config_file = joinpath(dir, "config") + write(config_file, config_str) + ini = read(Inifile(), IOBuffer(config_str)) + + @testset "environmental variable (AWS_REGION)" begin + withenv("AWS_REGION" => "eu-west-1", "AWS_DEFAULT_REGION" => nothing) do + @test aws_get_region(; config=ini, profile="default") == "eu-west-1" + @test aws_get_region() == "eu-west-1" + end + + withenv("AWS_REGION" => nothing, "AWS_DEFAULT_REGION" => "us-gov-east-1") do + @test aws_get_region(; config=ini, profile="default") == "us-gov-east-1" + @test aws_get_region() == "us-gov-east-1" + end + + withenv("AWS_REGION" => "eu-west-1", "AWS_DEFAULT_REGION" => "us-gov-east-1") do + @test aws_get_region(; config=ini, profile="default") == "eu-west-1" + @test aws_get_region() == "eu-west-1" + end + end + + @testset "default profile" begin + withenv("AWS_REGION" => nothing, "AWS_DEFAULT_REGION" => nothing) do + @test aws_get_region(; config=ini, profile="default") == "us-west-2" + @test aws_get_region(; config=config_file, profile="default") == "us-west-2" + end + + withenv( + "AWS_REGION" => nothing, + "AWS_DEFAULT_REGION" => nothing, + "AWS_CONFIG_FILE" => config_file, + "AWS_PROFILE" => nothing, + "AWS_DEFAULT_PROFILE" => nothing, + ) do + @test aws_get_region() == "us-west-2" + end + end + + @testset "specified profile" begin + withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do + @test aws_get_region(; config=ini, profile="test") == "ap-northeast-1" + @test aws_get_region(; config=config_file, profile="test") == + "ap-northeast-1" + end + + withenv( + "AWS_REGION" => nothing, + "AWS_DEFAULT_REGION" => nothing, + "AWS_CONFIG_FILE" => config_file, + "AWS_PROFILE" => "test", + ) do + @test aws_get_region() == "ap-northeast-1" + end + end + + @testset "unknown profile" begin + withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do + apply(Patches._imds_region_patch(nothing)) do + @test aws_get_region(; config=ini, profile="unknown") == + AWS.DEFAULT_REGION + @test aws_get_region(; config=config_file, profile="unknown") == + AWS.DEFAULT_REGION + end + end + + withenv( + "AWS_REGION" => nothing, + "AWS_DEFAULT_REGION" => nothing, + "AWS_CONFIG_FILE" => config_file, + "AWS_PROFILE" => "unknown", + ) do + apply(Patches._imds_region_patch(nothing)) do + @test aws_get_region() == AWS.DEFAULT_REGION + end + end + end + + @testset "default keyword" begin + default = nothing + withenv("AWS_DEFAULT_REGION" => nothing, "AWS_REGION" => nothing) do + apply(Patches._imds_region_patch(nothing)) do + @test aws_get_region(; config=ini, profile="unknown", default) === + default + @test aws_get_region(; + config=config_file, profile="unknown", default + ) === default + end + end + + withenv( + "AWS_REGION" => nothing, + "AWS_DEFAULT_REGION" => nothing, + "AWS_CONFIG_FILE" => config_file, + "AWS_PROFILE" => "unknown", + ) do + apply(Patches._imds_region_patch(nothing)) do + @test aws_get_region(; default=default) === default + end + end + end + + @testset "no such config file" begin + withenv( + "AWS_DEFAULT_REGION" => nothing, + "AWS_REGION" => nothing, + "AWS_CONFIG_FILE" => tempname(), + ) do + apply(Patches._imds_region_patch(nothing)) do + @test aws_get_region() == AWS.DEFAULT_REGION + end + end + end + + @testset "instance profile" begin + withenv( + "AWS_DEFAULT_REGION" => nothing, + "AWS_REGION" => nothing, + "AWS_CONFIG_FILE" => tempname(), + ) do + apply(Patches._imds_region_patch("ap-atlantis-1")) do + @test aws_get_region() == "ap-atlantis-1" + end + end + end + end +end diff --git a/test/AWSExceptions.jl b/test/unit/AWSExceptions.jl similarity index 100% rename from test/AWSExceptions.jl rename to test/unit/AWSExceptions.jl diff --git a/test/AWSMetadataUtilities.jl b/test/unit/AWSMetadataUtilities.jl similarity index 98% rename from test/AWSMetadataUtilities.jl rename to test/unit/AWSMetadataUtilities.jl index d07af2eb3..e138ec996 100644 --- a/test/AWSMetadataUtilities.jl +++ b/test/unit/AWSMetadataUtilities.jl @@ -71,7 +71,7 @@ end end @testset "_generate_low_level_definitions" begin - services = JSON.parsefile(joinpath(@__DIR__, "resources/services.json")) + services = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "services.json")) @testset "rest-xml" begin expected = "const s3 = AWS.RestXMLService(\"s3\", \"s3\", \"2006-03-01\")" @@ -251,7 +251,7 @@ end end @testset "_get_function_parameters" begin - shapes = JSON.parsefile(joinpath(@__DIR__, "resources/shapes.json")) + shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "shapes.json")) @testset "required params" begin input = "RequiredParams" @@ -310,8 +310,8 @@ end @testset "_generate_high_level_definitions" begin service_name = "sample_service" protocol = "rest-xml" - operations = JSON.parsefile(joinpath(@__DIR__, "resources/operations.json")) - shapes = JSON.parsefile(joinpath(@__DIR__, "resources/shapes.json")) + operations = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "operations.json")) + shapes = JSON.parsefile(joinpath(@__DIR__, "..", "resource", "shapes.json")) expected_result = """ \"\"\" diff --git a/test/IMDS.jl b/test/unit/IMDS.jl similarity index 100% rename from test/IMDS.jl rename to test/unit/IMDS.jl diff --git a/test/unit/issues.jl b/test/unit/issues.jl new file mode 100644 index 000000000..60fbc8bfb --- /dev/null +++ b/test/unit/issues.jl @@ -0,0 +1,76 @@ +@service S3 + +# https://github.com/JuliaCloud/AWS.jl/issues/515 +@testset "issue 515" begin + function _incomplete_patch(; data, num_attempts_to_fail=4) + attempt_num = 0 + n = length(data) + + function _downloads_response(content_length) + headers = ["content-length" => string(content_length)] + return Downloads.Response("http", "", 200, "HTTP/1.1 200 OK", headers) + end + + patch = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend + @patch function HTTP.request(args...; response_stream, kwargs...) + attempt_num += 1 + if attempt_num <= num_attempts_to_fail + write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained + throw(HTTP.RequestError(HTTP.Request(), EOFError())) + else + write(response_stream, data) + return HTTP.Response(200, "{\"Location\": \"us-east-1\"}") + end + end + elseif AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend + @patch function Downloads.request(args...; output, kwargs...) + attempt_num += 1 + if attempt_num <= num_attempts_to_fail + write(output, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained + message = "transfer closed with 1 bytes remaining to read" + e = Downloads.RequestError("", 18, message, _downloads_response(n)) + throw(e) + else + write(output, data) + return _downloads_response(n) + end + end + end + + return patch + end + + n = 100 + data = rand(UInt8, n) + bucket = "julialang2" # use public bucket as dummy + key = "bin/versions.json" + config = AWSConfig(; creds=nothing, region="us-east-1") + + @testset "Fail 2 attempts then succeed" begin + apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do + retrieved = S3.get_object(bucket, key; aws_config=config) + + @test length(retrieved) == n + @test retrieved == data + end + end + + @testset "Fail all 4 attempts then throw" begin + err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend + HTTP.RequestError + else + Downloads.RequestError + end + io = IOBuffer() + + apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do + params = Dict("response_stream" => io) + @test_throws err_t S3.get_object(bucket, key, params; aws_config=config) + + seekstart(io) + retrieved = read(io) + @test length(retrieved) == n - 1 + @test retrieved == data[1:(n - 1)] + end + end +end diff --git a/test/test_pkg.jl b/test/unit/test_pkg.jl similarity index 84% rename from test/test_pkg.jl rename to test/unit/test_pkg.jl index 608ef3a7f..ef3584080 100644 --- a/test/test_pkg.jl +++ b/test/unit/test_pkg.jl @@ -1,4 +1,4 @@ -path = joinpath(@__DIR__, "resources", "TestPkg") +path = joinpath(@__DIR__, "..", "resource", "TestPkg") Pkg.develop(; path=path) VERSION >= v"1.8" && Pkg.precompile("TestPkg") diff --git a/test/utilities.jl b/test/unit/utilities.jl similarity index 100% rename from test/utilities.jl rename to test/unit/utilities.jl