Skip to content

Commit 60616e2

Browse files
omuskleinschmidt
omus
and
kleinschmidt
authored
Support scoped AWS configurations (#729)
* Support scoped set AWS configurations * Use `current_aws_config()` as default * Define `default_aws_config` for outside of scope configuration * Set project version to 1.96.0 * fixup! Use `current_aws_config()` as default * Code review typo fixes Co-authored-by: Dave Kleinschmidt <[email protected]> --------- Co-authored-by: Dave Kleinschmidt <[email protected]>
1 parent 2f5bebf commit 60616e2

File tree

400 files changed

+32861
-31229
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

400 files changed

+32861
-31229
lines changed

‎Project.toml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "AWS"
22
uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
33
license = "MIT"
4-
version = "1.95.0"
4+
version = "1.96.0"
55

66
[deps]
77
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -17,6 +17,7 @@ Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
1717
OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
1818
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
1919
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
20+
ScopedValues = "7e506255-f358-4e82-b7e4-beb19740aa63"
2021
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
2122
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
2223
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -32,10 +33,11 @@ JSON = "0.18, 0.19, 0.20, 0.21"
3233
MbedTLS = "0.6, 0.7, 1"
3334
Mocking = "0.7, 0.8"
3435
OrderedCollections = "1.3"
36+
ScopedValues = "1.3"
3537
StableRNGs = "1"
3638
URIs = "1"
3739
XMLDict = "0.3, 0.4"
38-
julia = "1.6"
40+
julia = "1.8"
3941

4042
[extras]
4143
CodecBase = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf"

‎README.md‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,16 @@ S3.DeleteObjects(bucket_name, body) # Delete multiple objects
126126
There are most likely other similar functions which require more intricate details in how the requests are performed, both in the S3 definitions and in other services.
127127

128128
## Modifying Functionality
129+
129130
There are sometimes situations, in which default behavior of AWS.jl might be overridden, for example when this package is used to access S3-compatible object storage of a different cloud service provider, which might have different ways of joining the endpoint url, encoding the region in the signature etc.
131+
130132
In many cases this can be achieved by creating a user-defined subtype of `AbstractAWSConfig` where some of the default methods are overwritten.
131-
For example, if you want to use the S3 high-level interface to access public data from GCS without authorisation, you could define:
133+
For example, if you want to use the S3 high-level interface to access public data from GCS without authorization, you could define:
132134

133135
````julia
134-
struct AnonymousGCS <:AbstractAWSConfig end
136+
struct AnonymousGCS <: AbstractAWSConfig end
135137
struct NoCredentials end
138+
136139
AWS.region(aws::AnonymousGCS) = "" # No region
137140
AWS.credentials(aws::AnonymousGCS) = NoCredentials() # No credentials
138141
AWS.check_credentials(c::NoCredentials) = c # Skip credentials check
@@ -141,7 +144,10 @@ function AWS.generate_service_url(aws::AnonymousGCS, service::String, resource::
141144
service == "s3" || throw(ArgumentError("Can only handle s3 requests to GCS"))
142145
return string("https://storage.googleapis.com.", resource)
143146
end
144-
AWS.global_aws_config(AnonymousGCS())
147+
148+
with_aws_config(AnonymousGCS()) do
149+
...
150+
end
145151
````
146152

147153
which skips some of the signature and credentials checking and modifies the generation of the endpoint url.
@@ -175,7 +181,10 @@ function AWS.generate_service_url(aws::MinioConfig, service::String, resource::S
175181
service == "s3" || throw(ArgumentError("Can only handle s3 requests to Minio"))
176182
return string(aws.endpoint, resource)
177183
end
178-
AWS.global_aws_config(MinioConfig("http://127.0.0.1:9000", "aregion", SimpleCredentials("minio", "minio123", "")))
184+
185+
with_aws_config(MinioConfig("http://127.0.0.1:9000", "aregion", SimpleCredentials("minio", "minio123", ""))) do
186+
...
187+
end
179188
````
180189

181190
Now we are ready to use AWS.jl to do S3-compatible requests to a minio server.

‎src/AWS.jl‎

Lines changed: 7 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ using Mocking
1010
using OrderedCollections: LittleDict, OrderedDict
1111
using Random
1212
using SHA
13+
using ScopedValues: ScopedValues, ScopedValue, @with
1314
using Sockets
1415
using URIs
1516
using UUIDs: UUIDs
@@ -19,7 +20,8 @@ export @service
1920
export _merge
2021
export AbstractAWSConfig, AWSConfig, AWSExceptions, AWSServices, Request
2122
export IMDS
22-
export assume_role, generate_service_url, global_aws_config, set_user_agent
23+
export current_aws_config, default_aws_config, with_aws_config
24+
export assume_role, generate_service_url, set_user_agent
2325
export sign!, sign_aws2!, sign_aws4!
2426
export JSONService, RestJSONService, RestXMLService, QueryService, set_features
2527

@@ -65,41 +67,6 @@ Base.@kwdef struct FeatureSet
6567
use_response_type::Bool = false
6668
end
6769

68-
"""
69-
global_aws_config()
70-
71-
Retrieve the global AWS configuration.
72-
If one is not set, create one with default configuration options.
73-
74-
# Keywords
75-
- `kwargs...`: AWSConfig kwargs to be passed along if the global configuration is not already set
76-
77-
# Returns
78-
- `AWSConfig`: The global AWS configuration
79-
"""
80-
function global_aws_config(; kwargs...)
81-
if !isassigned(aws_config) || !isempty(kwargs)
82-
aws_config[] = AWSConfig(; kwargs...)
83-
end
84-
85-
return aws_config[]
86-
end
87-
88-
"""
89-
global_aws_config(config::AbstractAWSConfig)
90-
91-
Set the global AWSConfig.
92-
93-
# Arguments
94-
- `config::AWSConfig`: The AWSConfig to set in the global state
95-
96-
# Returns
97-
- `AWSConfig`: Global AWSConfig
98-
"""
99-
function global_aws_config(config::AbstractAWSConfig)
100-
return aws_config[] = config
101-
end
102-
10370
"""
10471
set_user_agent(new_user_agent::String)
10572
@@ -317,7 +284,7 @@ function (service::RestXMLService)(
317284
request_method::String,
318285
request_uri::String,
319286
args::AbstractDict{String,<:Any}=Dict{String,Any}();
320-
aws_config::AbstractAWSConfig=global_aws_config(),
287+
aws_config::AbstractAWSConfig=current_aws_config(),
321288
feature_set::FeatureSet=FeatureSet(),
322289
)
323290
feature_set.use_response_type && _delete_legacy_response_kw_args!(args)
@@ -374,7 +341,7 @@ Perform a Query request to AWS.
374341
function (service::QueryService)(
375342
operation::String,
376343
args::AbstractDict{String,<:Any}=Dict{String,Any}();
377-
aws_config::AbstractAWSConfig=global_aws_config(),
344+
aws_config::AbstractAWSConfig=current_aws_config(),
378345
feature_set::FeatureSet=FeatureSet(),
379346
)
380347
feature_set.use_response_type && _delete_legacy_response_kw_args!(args)
@@ -421,7 +388,7 @@ Perform a JSON request to AWS.
421388
function (service::JSONService)(
422389
operation::String,
423390
args::AbstractDict{String,<:Any}=Dict{String,Any}();
424-
aws_config::AbstractAWSConfig=global_aws_config(),
391+
aws_config::AbstractAWSConfig=current_aws_config(),
425392
feature_set::FeatureSet=FeatureSet(),
426393
)
427394
feature_set.use_response_type && _delete_legacy_response_kw_args!(args)
@@ -468,7 +435,7 @@ function (service::RestJSONService)(
468435
request_method::String,
469436
request_uri::String,
470437
args::AbstractDict{String,<:Any}=Dict{String,String}();
471-
aws_config::AbstractAWSConfig=global_aws_config(),
438+
aws_config::AbstractAWSConfig=current_aws_config(),
472439
feature_set::FeatureSet=FeatureSet(),
473440
)
474441
feature_set.use_response_type && _delete_legacy_response_kw_args!(args)

‎src/AWSConfig.jl‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,77 @@ function _update_creds!(aws_config::AWSConfig)
9292

9393
return creds
9494
end
95+
96+
const _DEFAULT_AWS_CONFIG = Ref{AbstractAWSConfig}()
97+
const _DEFAULT_AWS_CONFIG_LOCK = ReentrantLock()
98+
const _SCOPED_AWS_CONFIG = ScopedValue{AbstractAWSConfig}()
99+
100+
"""
101+
default_aws_config(config::AbstractAWSConfig) -> Nothing
102+
103+
Sets the default AWS configuration to use when calling
104+
[`current_aws_config()`](@ref current_aws_config) outside of the scope of any
105+
[`with_aws_config`](@ref) calls.
106+
"""
107+
function default_aws_config(config::AbstractAWSConfig)
108+
@lock _DEFAULT_AWS_CONFIG_LOCK begin
109+
_DEFAULT_AWS_CONFIG[] = config
110+
end
111+
return nothing
112+
end
113+
114+
"""
115+
default_aws_config() -> AbstractAWSConfig
116+
117+
Gets the default AWS configuration to use when calling
118+
[`current_aws_config()`](@ref current_aws_config) outside of the scope of any
119+
[`with_aws_config`](@ref) calls. If no configuration has been set with
120+
`default_aws_config(::AbstractAWSConfig)` then the AWS configuration will be loaded using
121+
`AWSConfig()`.
122+
123+
Users will typically want to use [`current_aws_config()](@ref current_aws_config) instead.
124+
"""
125+
function default_aws_config()
126+
# Load an initial default AWS configuration only if a user hasn't already loaded one.
127+
# We'll make use of a lock to avoid race conditions but only lock when necessary to
128+
# avoid unnecessary overhead.
129+
if !isassigned(_DEFAULT_AWS_CONFIG)
130+
@lock _DEFAULT_AWS_CONFIG_LOCK begin
131+
if !isassigned(_DEFAULT_AWS_CONFIG)
132+
_DEFAULT_AWS_CONFIG[] = @mock AWSConfig()
133+
end
134+
end
135+
end
136+
137+
return _DEFAULT_AWS_CONFIG[]
138+
end
139+
140+
"""
141+
current_aws_config() -> AbstractAWSConfig
142+
143+
Retrieve the current AWS configuration set within the current scope. If no configuration has
144+
been set using [`with_aws_config`](@ref) then the default AWS configuration will be used.
145+
"""
146+
function current_aws_config()
147+
#! format: off
148+
return @something(
149+
ScopedValues.get(_SCOPED_AWS_CONFIG),
150+
@mock(default_aws_config()),
151+
)
152+
#! format: on
153+
end
154+
155+
"""
156+
with_aws_config(f, config::AbstractAWSConfig) -> Any
157+
158+
Executes the function `f` where the result of `current_aws_config()` returns `config`.
159+
Nesting of `with_aws_config` calls is supported.
160+
161+
The user provided function `f` takes no parameters. The `with_aws_config` function returns
162+
the result return from `f`.
163+
"""
164+
function with_aws_config(f, config::AbstractAWSConfig)
165+
return @with _SCOPED_AWS_CONFIG => config begin
166+
f()
167+
end
168+
end

‎src/api_generation/high_level.jl‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -152,28 +152,28 @@ function _generate_high_level_definition(
152152

153153
if required_keys && (idempotent || headers)
154154
return """
155-
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", $params_headers_str; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
156-
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $params_headers_str, params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
155+
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", $params_headers_str; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
156+
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $params_headers_str, params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
157157
"""
158158
elseif !required_keys && (idempotent || headers)
159159
return """
160-
$formatted_function_name(; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", $params_headers_str; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
161-
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $params_headers_str, params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
160+
$formatted_function_name(; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", $params_headers_str; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
161+
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $params_headers_str, params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
162162
"""
163163
elseif required_keys && !isempty(req_kv)
164164
return """
165-
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", $req_str); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
166-
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $req_str), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
165+
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", $req_str); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
166+
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", Dict{String, Any}(mergewith(_merge, $req_str), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
167167
"""
168168
elseif required_keys
169169
return """
170-
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
171-
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
170+
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
171+
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
172172
"""
173173
else
174174
return """
175-
$formatted_function_name(; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
176-
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$method\", \"$request_uri\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
175+
$formatted_function_name(; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
176+
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$method\", \"$request_uri\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
177177
"""
178178
end
179179
end
@@ -201,23 +201,23 @@ function _generate_high_level_definition(
201201

202202
if required && idempotent
203203
return """
204-
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(req_kv, ", ")), $(join(idempotent_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
205-
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(req_kv, ", ")), $(join(idempotent_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
204+
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(req_kv, ", ")), $(join(idempotent_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
205+
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(req_kv, ", ")), $(join(idempotent_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
206206
"""
207207
elseif required
208208
return """
209-
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(req_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
210-
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(req_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
209+
$formatted_function_name($(join(req_keys, ", ")); aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(req_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
210+
$formatted_function_name($(join(req_keys, ", ")), params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(req_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
211211
"""
212212
elseif idempotent
213213
return """
214-
$formatted_function_name(; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(idempotent_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
215-
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(idempotent_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
214+
$formatted_function_name(; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}($(join(idempotent_kv, ", "))); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
215+
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", Dict{String, Any}(mergewith(_merge, Dict{String, Any}($(join(idempotent_kv, ", "))), params)); aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
216216
"""
217217
else
218218
return """
219-
$formatted_function_name(; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
220-
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()) = $service_name(\"$function_name\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
219+
$formatted_function_name(; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
220+
$formatted_function_name(params::AbstractDict{String}; aws_config::AbstractAWSConfig=current_aws_config()) = $service_name(\"$function_name\", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
221221
"""
222222
end
223223
end

0 commit comments

Comments
 (0)