Merge pull request #6836 from devtron-labs/ci-artifact-linked-copy

feat: linked ci should have artifacts of parent ci

Author: kartik-579

env_gen.json

@@ -187,6 +187,7 @@
  | ECR_REPO_NAME_PREFIX | string |test/ | Prefix for ECR repo to be created in does not exist |  | false |
  | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of gitops application |  | false |
  | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of no-gitops application |  | false |
+ | ENABLE_LINKED_CI_ARTIFACT_COPY | bool |false | Enable copying artifacts from parent CI pipeline to linked CI pipeline during creation |  | false |
  | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | ephemeral containers support version regex that is compared with k8sServerVersion |  | false |
  | EVENT_URL | string |http://localhost:3000/notify | Notifier service url |  | false |
  | EXECUTE_WIRE_NIL_CHECKER | bool |false | checks for any nil pointer in wire.go |  | false |
@@ -225,6 +226,7 @@
  | LENS_URL | string |http://lens-milandevtron-service:80 | Lens micro-service URL |  | false |
  | LIMIT_CI_CPU | string |0.5 |  |  | false |
  | LIMIT_CI_MEM | string |3G |  |  | false |
+ | LINKED_CI_ARTIFACT_COPY_LIMIT | int |10 | Maximum number of artifacts to copy from parent CI pipeline to linked CI pipeline |  | false |
  | LOGGER_DEV_MODE | bool |false | Enables a different logger theme. |  | false |
  | LOG_LEVEL | int |-1 |  |  | false |
  | MAX_SESSION_PER_USER | int |5 | max no of cluster terminal pods can be created by an user |  | false |

env_gen.md

@@ -124,6 +124,7 @@ type CiArtifactRepository interface {
 	GetArtifactsByCDPipelineAndRunnerType(cdPipelineId int, runnerType bean.WorkflowType) ([]CiArtifact, error)
 	SaveAll(artifacts []*CiArtifact) ([]*CiArtifact, error)
 	GetArtifactsByCiPipelineId(ciPipelineId int) ([]CiArtifact, error)
+	GetLatestArtifactsByCiPipelineId(ciPipelineId, limit int) ([]CiArtifact, error)
 	GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error)
 	FinDByParentCiArtifactAndCiId(parentCiArtifact int, ciPipelineIds []int) ([]*CiArtifact, error)
 	GetLatest(cdPipelineId int) (int, error)
@@ -680,6 +681,19 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineId(ciPipelineId int
 	return artifacts, err
 }
 
+func (impl CiArtifactRepositoryImpl) GetLatestArtifactsByCiPipelineId(ciPipelineId, limit int) ([]CiArtifact, error) {
+	var artifacts []CiArtifact
+	err := impl.dbConnection.
+		Model(&artifacts).
+		Column("ci_artifact.*").
+		Join("INNER JOIN ci_pipeline cp on cp.id=ci_artifact.pipeline_id").
+		Where("ci_artifact.pipeline_id = ?", ciPipelineId).
+		Where("cp.deleted = ?", false).
+		Order("ci_artifact.id DESC").Limit(limit).
+		Select()
+	return artifacts, err
+}
+
 func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error) {
 	var artifacts []CiArtifact
 	if len(ciPipelineIds) == 0 {

internal/sql/repository/CiArtifactRepository.go

@@ -154,6 +154,7 @@ type CiCdPipelineOrchestratorImpl struct {
 	deploymentConfigReadService   read.DeploymentConfigReadService
 	workflowCacheConfig           types.WorkflowCacheConfig
 	chartReadService              read2.ChartReadService
+	globalEnvVariables            *util2.GlobalEnvVariables
 }
 
 func NewCiCdPipelineOrchestrator(
@@ -185,7 +186,8 @@ func NewCiCdPipelineOrchestrator(
 	gitOpsConfigReadService config.GitOpsConfigReadService,
 	deploymentConfigService common.DeploymentConfigService,
 	deploymentConfigReadService read.DeploymentConfigReadService,
-	chartReadService read2.ChartReadService) *CiCdPipelineOrchestratorImpl {
+	chartReadService read2.ChartReadService,
+	envVariables *util2.EnvironmentVariables) *CiCdPipelineOrchestratorImpl {
 	_, workflowCacheConfig, err := types.GetCiConfigWithWorkflowCacheConfig()
 	if err != nil {
 		logger.Errorw("Error in getting workflow cache config, continuing with default values", "err", err)
@@ -223,6 +225,7 @@ func NewCiCdPipelineOrchestrator(
 		deploymentConfigReadService:   deploymentConfigReadService,
 		workflowCacheConfig:           workflowCacheConfig,
 		chartReadService:              chartReadService,
+		globalEnvVariables:            envVariables.GlobalEnvVariables,
 	}
 }
 
@@ -1160,6 +1163,19 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf
 				r.GitMaterialName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:]
 			}
 		}
+
+		// Copy artifacts from parent CI pipeline if this is a linked CI pipeline
+		if ciPipeline.ParentCiPipeline > 0 && ciPipeline.PipelineType == buildCommonBean.LINKED {
+			go func() {
+				err = impl.copyArtifactsFromParentCiPipeline(ciPipeline.ParentCiPipeline, ciPipeline.Id, createRequest.UserId)
+				if err != nil {
+					impl.logger.Errorw("error in copying artifacts from parent CI pipeline",
+						"parentCiPipelineId", ciPipeline.ParentCiPipeline,
+						"childCiPipelineId", ciPipeline.Id,
+						"err", err)
+				}
+			}()
+		}
 	}
 	return createRequest, nil
 }
@@ -2511,3 +2527,63 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper.
 		}
 	}
 }
+
+// copyArtifactsFromParentCiPipeline copies existing artifacts from parent CI pipeline to linked CI pipeline during pipeline creation
+func (impl *CiCdPipelineOrchestratorImpl) copyArtifactsFromParentCiPipeline(parentCiPipelineId, childCiPipelineId int, userId int32) error {
+	if !impl.globalEnvVariables.EnableLinkedCiArtifactCopy {
+		impl.logger.Debugw("Linked CI artifact copying is disabled", "parentCiPipelineId", parentCiPipelineId, "childCiPipelineId", childCiPipelineId)
+		return nil
+	}
+	limit := impl.globalEnvVariables.LinkedCiArtifactCopyLimit
+	if limit <= 0 {
+		return nil
+	}
+
+	impl.logger.Infow("Starting, copyArtifactsFromParentCiPipeline", "parentCiPipelineId", parentCiPipelineId,
+		"childCiPipelineId", childCiPipelineId, "limit", limit)
+
+	parentArtifacts, err := impl.CiArtifactRepository.GetLatestArtifactsByCiPipelineId(parentCiPipelineId, limit)
+	if err != nil && !util.IsErrNoRows(err) {
+		impl.logger.Errorw("error in fetching artifacts from parent CI pipeline", "parentCiPipelineId", parentCiPipelineId, "err", err)
+		return err
+	}
+
+	if len(parentArtifacts) == 0 {
+		impl.logger.Infow("No artifacts found in parent CI pipeline", "parentCiPipelineId", parentCiPipelineId)
+		return nil
+	}
+
+	var childArtifacts []*repository.CiArtifact
+	for i := len(parentArtifacts); i > 0; i-- {
+		parentArtifact := parentArtifacts[i-1] //reversing order into new array because postgres will save the array as it is thus causing our actual order to reverse
+		childArtifact := &repository.CiArtifact{
+			Image:              parentArtifact.Image,
+			ImageDigest:        parentArtifact.ImageDigest,
+			MaterialInfo:       parentArtifact.MaterialInfo,
+			DataSource:         parentArtifact.DataSource,
+			PipelineId:         childCiPipelineId,
+			ParentCiArtifact:   parentArtifact.Id,
+			IsArtifactUploaded: parentArtifact.IsArtifactUploaded, // for backward compatibility
+			ScanEnabled:        parentArtifact.ScanEnabled,
+			TargetPlatforms:    parentArtifact.TargetPlatforms,
+			AuditLog: sql.AuditLog{
+				CreatedOn: time.Now(),
+				CreatedBy: userId,
+				UpdatedOn: time.Now(),
+				UpdatedBy: userId,
+			},
+		}
+		if parentArtifact.ScanEnabled {
+			childArtifact.Scanned = parentArtifact.Scanned
+		}
+		childArtifacts = append(childArtifacts, childArtifact)
+	}
+	if len(childArtifacts) > 0 {
+		_, err = impl.CiArtifactRepository.SaveAll(childArtifacts)
+		if err != nil {
+			impl.logger.Errorw("error in saving copied artifacts for child ci pipeline", "childCiPipelineId", childCiPipelineId, "err", err)
+			return err
+		}
+	}
+	return nil
+}

pkg/pipeline/CiCdPipelineOrchestrator.go

@@ -65,6 +65,8 @@ type GlobalEnvVariables struct {
 	ExposeCiMetrics                   bool `env:"EXPOSE_CI_METRICS" envDefault:"false" description:"To expose CI metrics"`
 	ExecuteWireNilChecker             bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false" description:"checks for any nil pointer in wire.go"`
 	IsAirGapEnvironment               bool `json:"isAirGapEnvironment" env:"IS_AIR_GAP_ENVIRONMENT" envDefault:"false"`
+	EnableLinkedCiArtifactCopy        bool `env:"ENABLE_LINKED_CI_ARTIFACT_COPY" envDefault:"false" description:"Enable copying artifacts from parent CI pipeline to linked CI pipeline during creation"`
+	LinkedCiArtifactCopyLimit         int  `env:"LINKED_CI_ARTIFACT_COPY_LIMIT" envDefault:"10" description:"Maximum number of artifacts to copy from parent CI pipeline to linked CI pipeline"`
 }
 
 type GlobalClusterConfig struct {