Use single blobstore_configs.json + pre-start to write config files

jobs/cloud_controller_clock/spec

@@ -25,6 +25,10 @@ templates:
   uaa_ca.crt.erb: config/certs/uaa_ca.crt
   db_ca.crt.erb: config/certs/db_ca.crt
   credhub_ca.crt.erb: config/certs/credhub_ca.crt
+  storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json
+  storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json
+  storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json
+  storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json
 
 packages:
   - azure-storage-cli
@@ -172,8 +176,13 @@ properties:
   cc.staging_upload_password:
     description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging"
 
+  cc.resource_pool.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.resource_pool.connection_config:
+      description: "Azure Storage Cli connection hash"
   cc.resource_pool.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.resource_pool.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. Valid keys: ['encryption']."
@@ -218,8 +227,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.packages.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.packages.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.packages.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.packages.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. Valid keys: ['encryption']."
@@ -261,8 +275,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.droplets.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.droplets.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.droplets.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.droplets.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. Valid keys: ['encryption']."
@@ -301,8 +320,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.buildpacks.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.buildpacks.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.buildpacks.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.buildpacks.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. Valid keys: ['encryption']."

jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb

@@ -44,6 +44,11 @@ development_mode: false
 external_protocol: <%= p("cc.external_protocol") %>
 external_domain: <%= p("cc.external_host") %>.<%= p("system_domain") %>
 
+storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_droplets.json
+storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_buildpacks.json
+storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_packages.json
+storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_resource_pool.json
+
 jobs:
   global:
     timeout_in_seconds: <%= p("cc.jobs.global.timeout_in_seconds") %>

jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb

@@ -0,0 +1,48 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+scope = "cc.buildpacks.connection_config"
+provider = p("cc.buildpacks.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = p("#{scope}.azure_storage_account_name")
+  options["container_name"] = p("#{scope}.container_name")
+  add(options, "account_key",        p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb

@@ -0,0 +1,48 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+scope = "cc.droplets.connection_config"
+provider = p("cc.droplets.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = p("#{scope}.azure_storage_account_name")
+  options["container_name"] = p("#{scope}.container_name")
+  add(options, "account_key",        p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = p("cc.droplets.connection_config.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb

@@ -0,0 +1,48 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+scope = "cc.packages.connection_config"
+provider = p("cc.packages.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = p("#{scope}.azure_storage_account_name")
+  options["container_name"] = p("#{scope}.container_name")
+  add(options, "account_key",        p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb

@@ -0,0 +1,48 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+scope = "cc.resource_pool.connection_config"
+provider = p("cc.resource_pool.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = p("#{scope}.azure_storage_account_name")
+  options["container_name"] = p("#{scope}.container_name")
+  add(options, "account_key",        p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cloud_controller_ng/spec

@@ -56,6 +56,10 @@ templates:
   ruby_version.sh.erb: bin/ruby_version.sh
   seed_db.sh.erb: bin/seed_db
   stack_check.sh.erb: bin/stack_check
+  storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json
+  storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json
+  storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json
+  storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json
   setup_local_blobstore.sh.erb: bin/setup_local_blobstore.sh
   stacks.yml.erb: config/stacks.yml
   uaa_ca.crt.erb: config/certs/uaa_ca.crt
@@ -115,6 +119,7 @@ provides:
 - name: cloud_controller_internal
   type: cloud_controller_internal
   properties:
+  - cc.buildpacks.blobstore_provider
   - cc.buildpacks.blobstore_type
   - cc.buildpacks.buildpack_directory_key
   - cc.buildpacks.cdn.key_pair_id
@@ -143,6 +148,7 @@ provides:
   - cc.default_stack
   - cc.default_app_lifecycle
   - cc.disable_private_domain_cross_space_context_path_route_sharing
+  - cc.droplets.blobstore_provider
   - cc.droplets.blobstore_type
   - cc.droplets.cdn.key_pair_id
   - cc.droplets.cdn.private_key
@@ -172,6 +178,7 @@ provides:
   - cc.max_annotations_per_resource
   - cc.maximum_health_check_timeout
   - cc.packages.app_package_directory_key
+  - cc.packages.blobstore_provider
   - cc.packages.blobstore_type
   - cc.packages.cdn.key_pair_id
   - cc.packages.cdn.private_key
@@ -187,6 +194,7 @@ provides:
   - cc.packages.webdav_config.private_endpoint
   - cc.packages.webdav_config.public_endpoint
   - cc.packages.webdav_config.username
+  - cc.resource_pool.blobstore_provider
   - cc.resource_pool.blobstore_type
   - cc.resource_pool.cdn.key_pair_id
   - cc.resource_pool.cdn.private_key
@@ -231,7 +239,11 @@ provides:
   - cc.temporary_enable_deprecated_thin_webserver
   - cc.custom_root_links
   - cc.feature_flag_overrides
-
+  - cc.resource_pool.connection_config
+  - cc.packages.connection_config
+  - cc.droplets.connection_config
+  - cc.buildpacks.connection_config
+
 consumes:
 - name: database
   type: database
@@ -512,8 +524,13 @@ properties:
     default: default
     description: "The name of the quota definition CC will fallback on for org and space limits from the list of quota definitions."
 
+  cc.resource_pool.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.resource_pool.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.resource_pool.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.resource_pool.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. See http://docs.cloudfoundry.org/deploying/common/cc-blobstore-config.html#fog-aws-sse for example configuration."
@@ -560,8 +577,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.packages.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.packages.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.packages.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.packages.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. See http://docs.cloudfoundry.org/deploying/common/cc-blobstore-config.html#fog-aws-sse for example configuration."
@@ -608,8 +630,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.droplets.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.droplets.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.droplets.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.droplets.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. See http://docs.cloudfoundry.org/deploying/common/cc-blobstore-config.html#fog-aws-sse for example configuration."
@@ -653,8 +680,13 @@ properties:
     description: "Key pair name for signed download URIs"
     default: ""
 
+  cc.buildpacks.blobstore_provider:
+    description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']"
+    default: ~
+  cc.buildpacks.connection_config:
+    description: "Azure Storage Cli connection hash"
   cc.buildpacks.blobstore_type:
-    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']"
+    description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav', 'storage-cli']"
     default: "fog"
   cc.buildpacks.fog_aws_storage_options:
     description: "Storage options passed to fog for aws blobstores. See http://docs.cloudfoundry.org/deploying/common/cc-blobstore-config.html#fog-aws-sse for example configuration."