chore: update connector registry and secrets mask nightly (#16614)

Author: abuchanan-airbyte

.gitignore

@@ -77,11 +77,6 @@ resources/examples/airflow/logs/*
 # Ignore docs folder, since we're using it to temporarily copy files into on CI
 /docs
 
-# Ignore downloaded connector registry
-**/specs_secrets_mask.yaml
-**/oss_registry.json
-**/local_oss_registry.json
-
 # Files generated for uploading to GCS
 airbyte-config/**/resources/seed/oss_catalog.json
 

airbyte-commons/build.gradle.kts

@@ -33,16 +33,12 @@ airbyte {
   }
 }
 
-val downloadSpecSecretMask =
-  tasks.register<Download>("downloadSpecSecretMask") {
-    src("https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml")
-    dest(File(projectDir, "src/main/resources/seed/specs_secrets_mask.yaml"))
-    overwrite(true)
-    onlyIfModified(true)
-  }
-
-tasks.named("processResources") {
-  dependsOn(downloadSpecSecretMask)
+tasks.register<Download>("downloadSpecSecretMask") {
+  src("https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml")
+  dest(File(projectDir, "src/main/resources/seed/specs_secrets_mask.yaml"))
+  overwrite(true)
+  useETag(true)
+  onlyIfModified(true)
 }
 
 tasks.named<Test>("test") {

airbyte-commons/readme.md

@@ -9,3 +9,10 @@ the `build.gradle` of `airbyte-commons` should always be empty.
 
 For other common java code that needs to be shared across modules that requires additional dependencies, we follow this
 convention: `airbyte-commons-<name of lib>`. See for example `airbyte-commons-micronaut`.
+
+# Updating the secrets mask
+
+This module contains a secrets mask configuration file, which is baked into the JAR.
+This is used by logging systems to mask secrets from log messages.
+
+Run ./gradlew :downloadSpecSecretMask to manually update this file.

airbyte-commons/src/main/resources/seed/specs_secrets_mask.yaml

@@ -0,0 +1,140 @@
+properties:
+- X-Postmark-Account-Token
+- X-Postmark-Server-Token
+- access_key
+- access_key_id
+- access_key_secret
+- access_token
+- account_access_token
+- account_id
+- account_sid
+- account_token
+- agreement_grant_token
+- apiId
+- apiKey
+- apiKeySecret
+- api_id
+- api_key
+- api_key_2
+- api_key_id
+- api_key_secret
+- api_password
+- api_secret
+- api_token
+- apikey
+- app_api_key
+- app_client_id
+- app_client_secret
+- app_id
+- app_secret_token
+- app_tenant_id
+- appid
+- application_key
+- application_secret
+- astra_db_app_token
+- auth_hash
+- auth_ssh_key
+- auth_token
+- auth_user_password
+- aws_access_key_id
+- aws_key_id
+- aws_key_secret
+- aws_secret_access_key
+- aws_secret_key
+- azure_blob_storage_account_key
+- basic_auth_header
+- bearer_token
+- ca_certificate
+- certificate
+- client_access_token
+- client_certificate
+- client_id
+- client_id_2
+- client_key
+- client_key_password
+- client_refresh_token
+- client_refresh_token_2
+- client_secret
+- client_secret_2
+- client_secret_value
+- cohere_key
+- company_id
+- consumer_key
+- consumer_secret
+- credentials_json
+- developer_token
+- domain
+- domain_url
+- email
+- extension_id
+- fastly_api_token
+- google_application_credentials
+- hmac_key_access_id
+- hmac_key_secret
+- hoorayhrpassword
+- key
+- key_id
+- key_store_password
+- lwa_app_id
+- lwa_client_secret
+- management_token
+- motherduck_api_key
+- oauth_access_token
+- openai_key
+- org_id
+- password
+- personal_access_token
+- pinecone_key
+- pipeline_access_token
+- privateKey
+- private_key
+- private_key_password
+- private_token
+- project_access_token
+- public_key
+- realm
+- realm_id
+- redirect_uri
+- refresh_token
+- reserved_attribute_names
+- restid
+- s3_access_key_id
+- s3_secret_access_key
+- sas_token
+- sasl_jaas_config
+- schema_registry_password
+- secret
+- secret_access_key
+- secret_api_key
+- secret_key
+- service_account
+- service_account_info
+- service_account_json
+- session_token
+- shared_access_signature
+- shared_key
+- site_api_key
+- site_id
+- ssh_key
+- ssl_ca_certificate
+- ssl_certificate
+- storage_access_key
+- survey_group_id
+- survey_id
+- tenant_id
+- token
+- token_key
+- token_secret
+- trustStorePassword
+- tunnel_user_password
+- uid
+- user_api_key
+- user_auth_key
+- user_id
+- user_name
+- user_principal_name
+- user_secret
+- user_token
+- username
+- value
+- x-api-key

airbyte-config/specs/README.md

@@ -1,7 +1,7 @@
 # Downloading Local Connector Registry
 
-The local connector registry is downloaded from the metadata service using the `downloadConnectorRegistry` task defined in this module.
+This module contains a version of the connector registry, which is baked into the JAR. 
+The registry is shipped with Airbyte to account for air-gapped airbyte deployments. 
+This also allows Airbyte to start with pre-populated connectors. 
 
-This task is automatically run when Airbyte is built. The downloaded specs are shipped with Airbyte to account for air-gapped airbyte
-deployments. This also allows Airbyte to start with pre-populated connectors. After initialization, we expect most deployments to query the remote
-metadata service for the most up-to-date connector versions.
+Run ./gradlew :downloadConnectorRegistry to manually update this file.

airbyte-config/specs/build.gradle.kts

@@ -40,14 +40,10 @@ airbyte {
   }
 }
 
-val downloadConnectorRegistry =
-  tasks.register<Download>("downloadConnectorRegistry") {
-    src("https://connectors.airbyte.com/files/registries/v0/oss_registry.json")
-    dest(File(projectDir, "src/main/resources/seed/local_oss_registry.json"))
-    overwrite(true)
-    onlyIfModified(true)
-  }
-
-tasks.processResources {
-  dependsOn(downloadConnectorRegistry)
+tasks.register<Download>("downloadConnectorRegistry") {
+  src("https://connectors.airbyte.com/files/registries/v0/oss_registry.json")
+  dest(File(projectDir, "src/main/resources/seed/local_oss_registry.json"))
+  overwrite(true)
+  useETag(true)
+  onlyIfModified(true)
 }