Merge branch 'main' into db-creds-leakage

Author: saifsafsf

.github/workflows/gradle.yml

@@ -4,8 +4,8 @@
 name: Airbyte Platform OSS Developer Build
 
 env:
-  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.PLATFORM_BUILD_CACHE_ACCESS_KEY_ID }}
+  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.PLATFORM_BUILD_CACHE_SECRET_KEY }}
 
 on:
   #ability to start task manually in Web UI

.github/workflows/terminate-zombie-build-instances.yml

@@ -15,8 +15,8 @@ jobs:
     steps:
       - name: List and Terminate Instances Older Than 4 Hours
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.PLATFORM_BUILD_CACHE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.PLATFORM_BUILD_CACHE_SECRET_KEY }}
           # See https://github.com/aws/aws-cli/issues/5623
           AWS_EC2_METADATA_DISABLED: true
         run: |

airbyte-api/server-api/src/main/openapi/config.yaml

@@ -9213,6 +9213,8 @@ components:
           type: object
           additionalProperties:
             $ref: "#/components/schemas/ConnectorRolloutActorSyncInfo"
+        tier:
+          $ref: "#/components/schemas/CustomerTier"
 
     ConnectorRolloutState:
       type: string
@@ -9241,6 +9243,26 @@ components:
         - automated
         - overridden
 
+    ConnectorRolloutFilters:
+      type: object
+      properties:
+        tierFilter:
+          type: object
+          $ref: "#/components/schemas/ConnectorRolloutTierFilter"
+
+    ConnectorRolloutTierFilter:
+      type: object
+      properties:
+        tier:
+          $ref: "#/components/schemas/CustomerTier"
+
+    CustomerTier:
+      type: string
+      enum:
+        - TIER_0
+        - TIER_1
+        - TIER_2
+
     ConnectorRolloutListRequestBody:
       type: object
       required:
@@ -9305,6 +9327,8 @@ components:
         expires_at:
           type: string
           format: date-time
+        tier:
+          $ref: "#/components/schemas/CustomerTier"
 
     ConnectorRolloutCreateResponse:
       type: object
@@ -9557,6 +9581,8 @@ components:
         migrate_pins:
           type: boolean
           default: true
+        filters:
+          $ref: "#/components/schemas/ConnectorRolloutFilters"
 
     ConnectorRolloutManualRolloutRequestBody:
       type: object
@@ -9592,6 +9618,8 @@ components:
         migrate_pins:
           type: boolean
           default: true
+        filters:
+          $ref: "#/components/schemas/ConnectorRolloutFilters"
 
     ConnectorRolloutManualFinalizeRequestBody:
       type: object
@@ -10550,10 +10578,6 @@ components:
       required:
         - manifest
       properties:
-        formGeneratedManifest:
-          type: boolean
-          description: Indicates if the manifest was auto-generated from the form-based Connector Builder UI
-          default: false
         workspaceId:
           $ref: "#/components/schemas/WorkspaceId"
         builderProjectId:

airbyte-base-java-image/build.gradle.kts

@@ -7,5 +7,5 @@ plugins {
 tasks.register<DockerBuildxTask>("dockerJavaBaseImage") {
   inputDir = project.projectDir
   tag = "3.3.4"
-  imageName = "airbyte/airbyte-base-java-image"
-}
+  imageName = "airbyte-base-java-image"
+}

airbyte-base-java-python-image/build.gradle.kts

@@ -8,5 +8,5 @@ tasks.register<DockerBuildxTask>("dockerJavaPythonBaseImage") {
   inputDir = project.projectDir
   tag = "2.2.4"
   buildArgs.put("AIRBYTE_BASE_JAVA_IMAGE_TAG", "3.3.4")
-  imageName = "airbyte/airbyte-base-java-python-image"
-}
+  imageName = "airbyte-base-java-python-image"
+}

airbyte-bootloader/src/test-integration/kotlin/io/airbyte/bootloader/BootloaderTest.kt

@@ -46,7 +46,7 @@ import io.airbyte.db.factory.FlywayFactory
 import io.airbyte.db.instance.DatabaseConstants
 import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator
 import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider
-import io.airbyte.db.instance.configs.migrations.V1_1_1_027__AddOriginIndexToScopedConfiguration
+import io.airbyte.db.instance.configs.migrations.V1_1_1_028__AddFiltersToConnectorRollout
 import io.airbyte.db.instance.jobs.JobsDatabaseMigrator
 import io.airbyte.db.instance.jobs.JobsDatabaseTestProvider
 import io.airbyte.db.instance.jobs.migrations.V1_1_0_001__AddIsScheduledToJobTable
@@ -773,7 +773,7 @@ internal class BootloaderTest {
 
     // ⚠️ This line should change with every new migration to show that you meant to make a new
     // migration to the prod database
-    private val CURRENT_CONFIGS_MIGRATION = V1_1_1_027__AddOriginIndexToScopedConfiguration::class.java
+    private val CURRENT_CONFIGS_MIGRATION = V1_1_1_028__AddFiltersToConnectorRollout::class.java
     private val CURRENT_JOBS_MIGRATION = V1_1_0_001__AddIsScheduledToJobTable::class.java
 
     private fun getMigrationVersion(cls: Class<*>): String =

airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java

@@ -544,7 +544,6 @@ public ConnectorBuilderProjectFullResolveResponse fullResolveManifestBuilderProj
     final FullResolveManifestRequestBody fullResolveManifestRequestBody =
         new FullResolveManifestRequestBody(existingHydratedTestingValues,
             requestBody.getManifest(),
-            requestBody.getFormGeneratedManifest(),
             requestBody.getStreamLimit(),
             requestBody.getBuilderProjectId().toString(),
             requestBody.getWorkspaceId().toString());

airbyte-commons-server/src/main/kotlin/io/airbyte/commons/server/handlers/ConnectorRolloutHandler.kt

@@ -7,6 +7,7 @@ package io.airbyte.commons.server.handlers
 import com.google.common.annotations.VisibleForTesting
 import io.airbyte.api.model.generated.ConnectorRolloutActorSelectionInfo
 import io.airbyte.api.model.generated.ConnectorRolloutActorSyncInfo
+import io.airbyte.api.model.generated.ConnectorRolloutFilters
 import io.airbyte.api.model.generated.ConnectorRolloutFinalizeRequestBody
 import io.airbyte.api.model.generated.ConnectorRolloutManualFinalizeRequestBody
 import io.airbyte.api.model.generated.ConnectorRolloutManualFinalizeResponse
@@ -24,10 +25,14 @@ import io.airbyte.api.problems.model.generated.ProblemMessageData
 import io.airbyte.api.problems.throwable.generated.ConnectorRolloutInvalidRequestProblem
 import io.airbyte.api.problems.throwable.generated.ConnectorRolloutMaximumRolloutPercentageReachedProblem
 import io.airbyte.api.problems.throwable.generated.ConnectorRolloutNotEnoughActorsProblem
+import io.airbyte.config.AttributeName
 import io.airbyte.config.ConnectorEnumRolloutState
 import io.airbyte.config.ConnectorEnumRolloutStrategy
 import io.airbyte.config.ConnectorRollout
 import io.airbyte.config.ConnectorRolloutFinalState
+import io.airbyte.config.CustomerTier
+import io.airbyte.config.CustomerTierFilter
+import io.airbyte.config.Operator
 import io.airbyte.config.persistence.UserPersistence
 import io.airbyte.connector.rollout.client.ConnectorRolloutClient
 import io.airbyte.connector.rollout.shared.ActorSelectionInfo
@@ -161,10 +166,11 @@ open class ConnectorRolloutHandler
       dockerRepository: String,
       actorDefinitionId: UUID,
       dockerImageTag: String,
-      updatedBy: UUID,
+      updatedBy: UUID?,
       rolloutStrategy: ConnectorRolloutStrategy,
       initialRolloutPct: Int?,
       finalTargetRolloutPct: Int?,
+      requestFilters: ConnectorRolloutFilters?,
     ): ConnectorRollout {
       val actorDefinitionVersion =
         actorDefinitionService.getActorDefinitionVersion(
@@ -199,6 +205,8 @@ open class ConnectorRolloutHandler
             ProblemMessageData().message("Could not find initial version for actor definition id: $actorDefinitionId"),
           )
 
+      val filters = createFiltersFromRequest(requestFilters)
+
       if (initializedRollouts.isEmpty()) {
         val currentTime = OffsetDateTime.now(ZoneOffset.UTC).toEpochSecond()
 
@@ -216,6 +224,7 @@ open class ConnectorRolloutHandler
             rolloutStrategy = getRolloutStrategyForManualStart(rolloutStrategy),
             initialRolloutPct = initialRolloutPct,
             finalTargetRolloutPct = finalTargetRolloutPct,
+            filters = filters,
           )
         connectorRolloutService.writeConnectorRollout(connectorRollout)
         return connectorRollout
@@ -250,6 +259,7 @@ open class ConnectorRolloutHandler
       connectorRollout.rolloutStrategy = getRolloutStrategyForManualStart(rolloutStrategy)
       connectorRollout.initialRolloutPct = initialRolloutPct
       connectorRollout.finalTargetRolloutPct = finalTargetRolloutPct
+      connectorRollout.filters = filters
 
       connectorRolloutService.writeConnectorRollout(connectorRollout)
       return connectorRollout
@@ -580,7 +590,7 @@ open class ConnectorRolloutHandler
     fun getPinnedActorInfo(id: UUID): ConnectorRolloutActorSelectionInfo {
       val rollout = connectorRolloutService.getConnectorRollout(id)
       logger.info { "getPinnedActorInfo: rollout=$rollout" }
-      val actorSelectionInfo = rolloutActorFinder.getActorSelectionInfo(rollout, null)
+      val actorSelectionInfo = rolloutActorFinder.getActorSelectionInfo(rollout, null, rollout.filters)
       logger.info { "getPinnedActorInfo: actorSelectionInfo=$actorSelectionInfo" }
 
       return ConnectorRolloutActorSelectionInfo()
@@ -599,6 +609,7 @@ open class ConnectorRolloutHandler
           connectorRolloutManualStart.rolloutStrategy,
           connectorRolloutManualStart.initialRolloutPct,
           connectorRolloutManualStart.finalTargetRolloutPct,
+          connectorRolloutManualStart.filters,
         )
 
       try {
@@ -629,9 +640,21 @@ open class ConnectorRolloutHandler
       return buildConnectorRolloutRead(connectorRolloutService.getConnectorRollout(rollout.id), false)
     }
 
-    open fun manualDoConnectorRolloutUpdate(connectorRolloutUpdate: ConnectorRolloutManualRolloutRequestBody): ConnectorRolloutManualRolloutResponse {
-      val connectorRollout = connectorRolloutService.getConnectorRollout(connectorRolloutUpdate.id)
+    open fun manualDoConnectorRollout(connectorRolloutUpdate: ConnectorRolloutManualRolloutRequestBody): ConnectorRolloutManualRolloutResponse {
+      var connectorRollout = connectorRolloutService.getConnectorRollout(connectorRolloutUpdate.id)
+
       if (connectorRollout.state == ConnectorEnumRolloutState.INITIALIZED) {
+        connectorRollout =
+          getOrCreateAndValidateManualStartInput(
+            connectorRolloutUpdate.dockerRepository,
+            connectorRolloutUpdate.actorDefinitionId,
+            connectorRolloutUpdate.dockerImageTag,
+            connectorRolloutUpdate.updatedBy,
+            ConnectorRolloutStrategy.MANUAL,
+            null,
+            null,
+            connectorRolloutUpdate.filters,
+          )
         try {
           connectorRolloutClient.startRollout(
             ConnectorRolloutWorkflowInput(
@@ -654,6 +677,10 @@ open class ConnectorRolloutHandler
         } catch (e: WorkflowUpdateException) {
           throw throwAirbyteApiClientExceptionIfExists("startWorkflow", e)
         }
+      } else {
+        if (connectorRolloutUpdate.filters != null) {
+          throw RuntimeException("Cannot modify filters in a running rollout.")
+        }
       }
       try {
         connectorRolloutClient.doRollout(
@@ -805,7 +832,7 @@ open class ConnectorRolloutHandler
       connectorRollout: ConnectorRollout,
       targetPercent: Int,
     ): ActorSelectionInfo {
-      val actorSelectionInfo = rolloutActorFinder.getActorSelectionInfo(connectorRollout, targetPercent)
+      val actorSelectionInfo = rolloutActorFinder.getActorSelectionInfo(connectorRollout, targetPercent, connectorRollout.filters)
       if (targetPercent > 0 && actorSelectionInfo.actorIdsToPin.isEmpty() && actorSelectionInfo.nPreviouslyPinned == 0) {
         throw ConnectorRolloutNotEnoughActorsProblem(
           ProblemMessageData().message(
@@ -867,6 +894,32 @@ open class ConnectorRolloutHandler
       return rollouts.first()
     }
 
+    private fun createFiltersFromRequest(filters: ConnectorRolloutFilters?): io.airbyte.config.ConnectorRolloutFilters {
+      if (filters?.tierFilter == null) {
+        return io.airbyte.config.ConnectorRolloutFilters(
+          customerTierFilters =
+            listOf(
+              CustomerTierFilter(
+                name = AttributeName.TIER,
+                operator = Operator.IN,
+                value = listOf(CustomerTier.TIER_2),
+              ),
+            ),
+        )
+      } else {
+        return io.airbyte.config.ConnectorRolloutFilters(
+          customerTierFilters =
+            listOf(
+              CustomerTierFilter(
+                name = AttributeName.TIER,
+                operator = Operator.IN,
+                value = listOf(CustomerTier.valueOf(filters.tierFilter!!.tier.toString())),
+              ),
+            ),
+        )
+      }
+    }
+
     private fun throwAirbyteApiClientExceptionIfExists(
       handlerName: String,
       e: WorkflowUpdateException,